null/BillTracker
null
/
BillTracker
1
0
Fork 0
Code Issues 4 Pull Requests Packages Projects Releases 1 Wiki Activity

refactor(server): migrate csvTransactionImportService to TypeScript (.cts) 

The CSV parser + import primitives (also shared by the OFX importer).

Behavior-preserving. Verified: typecheck:server 0, check:server 0, suite 226/226.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
null 2026-07-06 11:20:29 -05:00
parent 310eb07b9b
commit 99e4927640
4 changed files with 60 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
routes/import.js
View File

@ -15,7 +15,7 @@ const {
const { const {
previewCsvTransactions, previewCsvTransactions,
commitCsvTransactions, commitCsvTransactions,
} = require('../services/csvTransactionImportService'); } = require('../services/csvTransactionImportService.cts');
const { const {
previewOfxTransactions, previewOfxTransactions,
commitOfxTransactions, commitOfxTransactions,

113
services/csvTransactionImportService.js → services/csvTransactionImportService.cts
View File

@ -1,14 +1,18 @@
'use strict'; 'use strict';
import type { Db } from '../types/db';
const crypto = require('crypto'); const crypto = require('crypto');
const { getDb } = require('../db/database'); const { getDb } = require('../db/database');
const { decorateTransaction, ensureManualDataSource } = require('./transactionService.cts'); const { decorateTransaction, ensureManualDataSource } = require('./transactionService.cts');
type Row = Record<string, any>;
const SESSION_TTL_MS = 24 * 60 * 60 * 1000; const SESSION_TTL_MS = 24 * 60 * 60 * 1000;
const MAX_ROWS = 25000; const MAX_ROWS = 25000;
const SAMPLE_SIZE = 10; const SAMPLE_SIZE = 10;
const FIELD_LABELS = { const FIELD_LABELS: Record<string, string> = {
posted_date: 'Posted date', posted_date: 'Posted date',
transacted_at: 'Transaction date/time', transacted_at: 'Transaction date/time',
amount: 'Amount', amount: 'Amount',
@ -24,32 +28,32 @@ const FIELD_LABELS = {
currency: 'Currency', currency: 'Currency',
}; };
function importError(status, message, code, details = []) { function importError(status: number, message: string, code: string, details: any[] = []): Error {
const err = new Error(message); const err = new Error(message) as any;
err.status = status; err.status = status;
err.code = code; err.code = code;
err.details = details; err.details = details;
return err; return err;
} }
function makeSessionId() { function makeSessionId(): string {
return crypto.randomBytes(16).toString('hex'); return crypto.randomBytes(16).toString('hex');
} }
function cleanFilename(value) { function cleanFilename(value: any): string | null {
return value return value
? String(value).replace(/[^a-zA-Z0-9._\-\s]/g, '').trim().slice(0, 255) ? String(value).replace(/[^a-zA-Z0-9._\-\s]/g, '').trim().slice(0, 255)
: null; : null;
} }
function normalizeHeader(value, index) { function normalizeHeader(value: any, index: number): string {
const text = String(value ?? '').trim(); const text = String(value ?? '').trim();
return text || `Column ${index + 1}`; return text || `Column ${index + 1}`;
} }
function parseCsv(text) { function parseCsv(text: string): any[] {
const rows = []; const rows: any[] = [];
let row = []; let row: any[] = [];
let cell = ''; let cell = '';
let inQuotes = false; let inQuotes = false;
@ -98,38 +102,38 @@ function parseCsv(text) {
rows.push(row); rows.push(row);
} }
return rows.filter(r => r.some(c => String(c ?? '').trim() !== '')); return rows.filter((r: any[]) => r.some((c: any) => String(c ?? '').trim() !== ''));
} }
function csvBufferToText(buffer) { function csvBufferToText(buffer: any): string {
if (!Buffer.isBuffer(buffer) || buffer.length === 0) { if (!Buffer.isBuffer(buffer) || buffer.length === 0) {
throw importError(400, 'CSV file is required.', 'CSV_REQUIRED'); throw importError(400, 'CSV file is required.', 'CSV_REQUIRED');
} }
return buffer.toString('utf8').replace(/^\uFEFF/, ''); return buffer.toString('utf8').replace(/^/, '');
} }
function rowToObject(headers, row) { function rowToObject(headers: string[], row: any[]): Row {
const out = {}; const out: Row = {};
headers.forEach((header, index) => { headers.forEach((header: string, index: number) => {
out[header] = String(row[index] ?? '').trim(); out[header] = String(row[index] ?? '').trim();
}); });
return out; return out;
} }
function normalizeHeaderToken(value) { function normalizeHeaderToken(value: any): string {
return String(value || '').toLowerCase().replace(/[^a-z0-9]+/g, ' ').trim(); return String(value || '').toLowerCase().replace(/[^a-z0-9]+/g, ' ').trim();
} }
function headerMatches(header, patterns) { function headerMatches(header: any, patterns: any[]): boolean {
const token = normalizeHeaderToken(header); const token = normalizeHeaderToken(header);
return patterns.some(pattern => ( return patterns.some((pattern: any) => (
pattern instanceof RegExp ? pattern.test(token) : token === pattern pattern instanceof RegExp ? pattern.test(token) : token === pattern
)); ));
} }
function suggestMapping(headers) { function suggestMapping(headers: string[]): Row {
const mapping = {}; const mapping: Row = {};
const candidates = [ const candidates: [string, any[]][] = [
['posted_date', [/^date$/, 'posted date', 'post date', 'posting date', 'transaction date', 'trans date']], ['posted_date', [/^date$/, 'posted date', 'post date', 'posting date', 'transaction date', 'trans date']],
['transacted_at', ['authorized date', 'authorization date', 'datetime', 'date time', 'timestamp']], ['transacted_at', ['authorized date', 'authorization date', 'datetime', 'date time', 'timestamp']],
['transaction_id', ['transaction id', 'transaction number', 'trans id', 'id', 'fitid', 'reference', 'reference number']], ['transaction_id', ['transaction id', 'transaction number', 'trans id', 'id', 'fitid', 'reference', 'reference number']],
@ -146,19 +150,19 @@ function suggestMapping(headers) {
]; ];
for (const [field, patterns] of candidates) { for (const [field, patterns] of candidates) {
const found = headers.find(header => !Object.values(mapping).includes(header) && headerMatches(header, patterns)); const found = headers.find((header: string) => !Object.values(mapping).includes(header) && headerMatches(header, patterns));
if (found) mapping[field] = found; if (found) mapping[field] = found;
} }
if (!mapping.payee && mapping.description) { if (!mapping.payee && mapping.description) {
const payee = headers.find(header => header !== mapping.description && headerMatches(header, ['name', 'merchant name'])); const payee = headers.find((header: string) => header !== mapping.description && headerMatches(header, ['name', 'merchant name']));
if (payee) mapping.payee = payee; if (payee) mapping.payee = payee;
} }
return mapping; return mapping;
} }
function parseCsvPreview(buffer, options = {}) { function parseCsvPreview(buffer: any, options: any = {}): any {
const text = csvBufferToText(buffer); const text = csvBufferToText(buffer);
const parsed = parseCsv(text); const parsed = parseCsv(text);
if (parsed.length < 2) { if (parsed.length < 2) {
@ -166,8 +170,8 @@ function parseCsvPreview(buffer, options = {}) {
} }
const headers = parsed[0].map(normalizeHeader); const headers = parsed[0].map(normalizeHeader);
const seenHeaders = new Set(); const seenHeaders = new Set<string>();
const duplicateHeaders = []; const duplicateHeaders: any[] = [];
for (const header of headers) { for (const header of headers) {
const key = header.toLowerCase(); const key = header.toLowerCase();
if (seenHeaders.has(key)) duplicateHeaders.push(header); if (seenHeaders.has(key)) duplicateHeaders.push(header);
@ -177,10 +181,10 @@ function parseCsvPreview(buffer, options = {}) {
throw importError(400, `CSV contains duplicate headers: ${duplicateHeaders.join(', ')}`, 'CSV_DUPLICATE_HEADERS'); throw importError(400, `CSV contains duplicate headers: ${duplicateHeaders.join(', ')}`, 'CSV_DUPLICATE_HEADERS');
} }
const dataRows = parsed.slice(1).slice(0, MAX_ROWS).map(row => rowToObject(headers, row)); const dataRows = parsed.slice(1).slice(0, MAX_ROWS).map((row: any[]) => rowToObject(headers, row));
const truncated = parsed.length - 1 > MAX_ROWS; const truncated = parsed.length - 1 > MAX_ROWS;
const suggestedMapping = suggestMapping(headers); const suggestedMapping = suggestMapping(headers);
const errors = []; const errors: any[] = [];
if (!suggestedMapping.posted_date) { if (!suggestedMapping.posted_date) {
errors.push({ field: 'posted_date', message: 'Could not detect a posted date column.' }); errors.push({ field: 'posted_date', message: 'Could not detect a posted date column.' });
@ -206,11 +210,11 @@ function parseCsvPreview(buffer, options = {}) {
}; };
} }
function pruneExpiredSessions(db) { function pruneExpiredSessions(db: Db): void {
db.prepare('DELETE FROM import_sessions WHERE expires_at <= ?').run(new Date().toISOString()); db.prepare('DELETE FROM import_sessions WHERE expires_at <= ?').run(new Date().toISOString());
} }
function saveImportSession(db, userId, sessionData) { function saveImportSession(db: Db, userId: number, sessionData: any): string {
const id = makeSessionId(); const id = makeSessionId();
const now = new Date().toISOString(); const now = new Date().toISOString();
const expiresAt = new Date(Date.now() + SESSION_TTL_MS).toISOString(); const expiresAt = new Date(Date.now() + SESSION_TTL_MS).toISOString();
@ -221,7 +225,7 @@ function saveImportSession(db, userId, sessionData) {
return id; return id;
} }
function loadImportSession(db, userId, sessionId) { function loadImportSession(db: Db, userId: number, sessionId: any): any {
const row = db.prepare(` const row = db.prepare(`
SELECT preview_json SELECT preview_json
FROM import_sessions FROM import_sessions
@ -234,12 +238,12 @@ function loadImportSession(db, userId, sessionId) {
return JSON.parse(row.preview_json); return JSON.parse(row.preview_json);
} }
function deleteImportSession(db, sessionId) { function deleteImportSession(db: Db, sessionId: any): void {
db.prepare('DELETE FROM import_sessions WHERE id = ?').run(sessionId); db.prepare('DELETE FROM import_sessions WHERE id = ?').run(sessionId);
} }
function previewCsvTransactions(userId, buffer, options = {}) { function previewCsvTransactions(userId: number, buffer: any, options: any = {}): any {
const db = getDb(); const db: Db = getDb();
pruneExpiredSessions(db); pruneExpiredSessions(db);
const preview = parseCsvPreview(buffer, options); const preview = parseCsvPreview(buffer, options);
const sessionId = saveImportSession(db, userId, { const sessionId = saveImportSession(db, userId, {
@ -258,13 +262,13 @@ function previewCsvTransactions(userId, buffer, options = {}) {
}; };
} }
function headerValue(row, mapping, field) { function headerValue(row: Row, mapping: any, field: string): string {
const header = mapping?.[field]; const header = mapping?.[field];
if (!header) return ''; if (!header) return '';
return String(row[header] ?? '').trim(); return String(row[header] ?? '').trim();
} }
function parseDateValue(value, field, rowNumber) { function parseDateValue(value: any, field: string, rowNumber: number): string {
const text = String(value || '').trim(); const text = String(value || '').trim();
if (!text) { if (!text) {
throw importError(400, `${FIELD_LABELS[field] || field} is required`, 'CSV_ROW_VALIDATION', [ throw importError(400, `${FIELD_LABELS[field] || field} is required`, 'CSV_ROW_VALIDATION', [
@ -290,7 +294,7 @@ function parseDateValue(value, field, rowNumber) {
]); ]);
} }
function isRealDate(value) { function isRealDate(value: any): boolean {
const [year, month, day] = String(value).split('-').map(Number); const [year, month, day] = String(value).split('-').map(Number);
const date = new Date(Date.UTC(year, month - 1, day)); const date = new Date(Date.UTC(year, month - 1, day));
return date.getUTCFullYear() === year return date.getUTCFullYear() === year
@ -298,7 +302,7 @@ function isRealDate(value) {
&& date.getUTCDate() === day; && date.getUTCDate() === day;
} }
function parseCents(value, { negative = false } = {}) { function parseCents(value: any, { negative = false }: { negative?: boolean } = {}): number | null {
const text = String(value ?? '').trim(); const text = String(value ?? '').trim();
if (!text) return null; if (!text) return null;
const parenNegative = /^\(.*\)$/.test(text); const parenNegative = /^\(.*\)$/.test(text);
@ -311,7 +315,7 @@ function parseCents(value, { negative = false } = {}) {
return Math.round(Math.abs(number) * 100) * sign; return Math.round(Math.abs(number) * 100) * sign;
} }
function parseMappedAmount(row, mapping) { function parseMappedAmount(row: Row, mapping: any): number | null {
const amount = parseCents(headerValue(row, mapping, 'amount')); const amount = parseCents(headerValue(row, mapping, 'amount'));
if (amount !== null) return amount; if (amount !== null) return amount;
@ -324,15 +328,15 @@ function parseMappedAmount(row, mapping) {
return null; return null;
} }
function stableHash(parts) { function stableHash(parts: any[]): string {
return crypto return crypto
.createHash('sha256') .createHash('sha256')
.update(parts.map(part => String(part || '').trim().toLowerCase()).join('\u001f')) .update(parts.map((part: any) => String(part || '').trim().toLowerCase()).join(''))
.digest('hex') .digest('hex')
.slice(0, 48); .slice(0, 48);
} }
function getOrCreateCsvDataSource(db, userId) { function getOrCreateCsvDataSource(db: Db, userId: number): Row {
ensureManualDataSource(db, userId); ensureManualDataSource(db, userId);
const existing = db.prepare(` const existing = db.prepare(`
SELECT * SELECT *
@ -350,7 +354,7 @@ function getOrCreateCsvDataSource(db, userId) {
return db.prepare('SELECT * FROM data_sources WHERE id = ? AND user_id = ?').get(result.lastInsertRowid, userId); return db.prepare('SELECT * FROM data_sources WHERE id = ? AND user_id = ?').get(result.lastInsertRowid, userId);
} }
function getOrCreateAccount(db, userId, dataSourceId, name) { function getOrCreateAccount(db: Db, userId: number, dataSourceId: any, name: any): Row | null {
const accountName = String(name || '').trim(); const accountName = String(name || '').trim();
if (!accountName) return null; if (!accountName) return null;
const providerAccountId = stableHash([accountName]).slice(0, 32); const providerAccountId = stableHash([accountName]).slice(0, 32);
@ -369,13 +373,13 @@ function getOrCreateAccount(db, userId, dataSourceId, name) {
return db.prepare('SELECT * FROM financial_accounts WHERE id = ? AND user_id = ?').get(result.lastInsertRowid, userId); return db.prepare('SELECT * FROM financial_accounts WHERE id = ? AND user_id = ?').get(result.lastInsertRowid, userId);
} }
function validateMapping(headers, mapping = {}) { function validateMapping(headers: any[], mapping: any = {}): void {
const headerSet = new Set(headers); const headerSet = new Set(headers);
const required = []; const required: any[] = [];
if (!mapping.posted_date) required.push('posted_date'); if (!mapping.posted_date) required.push('posted_date');
if (!mapping.amount && !(mapping.debit_amount || mapping.credit_amount)) required.push('amount'); if (!mapping.amount && !(mapping.debit_amount || mapping.credit_amount)) required.push('amount');
if (required.length) { if (required.length) {
throw importError(400, `Missing required mapping: ${required.map(f => FIELD_LABELS[f] || f).join(', ')}`, 'CSV_MAPPING_REQUIRED'); throw importError(400, `Missing required mapping: ${required.map((f: string) => FIELD_LABELS[f] || f).join(', ')}`, 'CSV_MAPPING_REQUIRED');
} }
for (const [field, header] of Object.entries(mapping)) { for (const [field, header] of Object.entries(mapping)) {
@ -388,7 +392,7 @@ function validateMapping(headers, mapping = {}) {
} }
} }
function parseOptionalDateTimeValue(value, field, rowNumber) { function parseOptionalDateTimeValue(value: any, field: string, rowNumber: number): string | null {
const text = String(value || '').trim(); const text = String(value || '').trim();
if (!text) return null; if (!text) return null;
@ -401,7 +405,7 @@ function parseOptionalDateTimeValue(value, field, rowNumber) {
return text; return text;
} }
function normalizeCsvTransaction(row, mapping, rowNumber) { function normalizeCsvTransaction(row: Row, mapping: any, rowNumber: number): Row {
const postedDate = parseDateValue(headerValue(row, mapping, 'posted_date'), 'posted_date', rowNumber); const postedDate = parseDateValue(headerValue(row, mapping, 'posted_date'), 'posted_date', rowNumber);
const transactedAt = parseOptionalDateTimeValue(headerValue(row, mapping, 'transacted_at'), 'transacted_at', rowNumber); const transactedAt = parseOptionalDateTimeValue(headerValue(row, mapping, 'transacted_at'), 'transacted_at', rowNumber);
const amount = parseMappedAmount(row, mapping); const amount = parseMappedAmount(row, mapping);
@ -436,8 +440,8 @@ function normalizeCsvTransaction(row, mapping, rowNumber) {
}; };
} }
function commitCsvTransactions(userId, importSessionId, mapping, options = {}) { function commitCsvTransactions(userId: number, importSessionId: any, mapping: any, options: any = {}): any {
const db = getDb(); const db: Db = getDb();
const session = loadImportSession(db, userId, importSessionId); const session = loadImportSession(db, userId, importSessionId);
if (session.kind !== 'csv_transactions') { if (session.kind !== 'csv_transactions') {
throw importError(400, 'Import session is not a CSV transaction preview.', 'CSV_SESSION_INVALID'); throw importError(400, 'Import session is not a CSV transaction preview.', 'CSV_SESSION_INVALID');
@ -445,7 +449,7 @@ function commitCsvTransactions(userId, importSessionId, mapping, options = {}) {
validateMapping(session.headers, mapping); validateMapping(session.headers, mapping);
const dataSource = getOrCreateCsvDataSource(db, userId); const dataSource = getOrCreateCsvDataSource(db, userId);
const details = []; const details: any[] = [];
const counts = { imported: 0, skipped: 0, failed: 0 }; const counts = { imported: 0, skipped: 0, failed: 0 };
const insert = db.prepare(` const insert = db.prepare(`
INSERT INTO transactions INSERT INTO transactions
@ -461,7 +465,7 @@ function commitCsvTransactions(userId, importSessionId, mapping, options = {}) {
`); `);
const run = db.transaction(() => { const run = db.transaction(() => {
session.rows.forEach((row, index) => { session.rows.forEach((row: Row, index: number) => {
const rowNumber = index + 2; const rowNumber = index + 2;
try { try {
const tx = normalizeCsvTransaction(row, mapping, rowNumber); const tx = normalizeCsvTransaction(row, mapping, rowNumber);
@ -508,7 +512,7 @@ function commitCsvTransactions(userId, importSessionId, mapping, options = {}) {
ignored: 0, ignored: 0,
}), }),
}); });
} catch (err) { } catch (err: any) {
counts.failed++; counts.failed++;
details.push({ row: rowNumber, result: 'failed', message: err.message, details: err.details || [] }); details.push({ row: rowNumber, result: 'failed', message: err.message, details: err.details || [] });
} }
@ -553,9 +557,6 @@ module.exports = {
FIELD_LABELS, FIELD_LABELS,
commitCsvTransactions, commitCsvTransactions,
previewCsvTransactions, previewCsvTransactions,
// Reusable transaction-import primitives (shared by the OFX/QFX importer so it
// dedupes and sessions identically — same import_sessions table, same
// (user_id, data_source_id, provider_transaction_id) dedupe scope).
saveImportSession, saveImportSession,
loadImportSession, loadImportSession,
deleteImportSession, deleteImportSession,

2
services/ofxImportService.cts
View File

@ -21,7 +21,7 @@ const {
getOrCreateAccount, getOrCreateAccount,
stableHash, stableHash,
parseCents, parseCents,
} = require('./csvTransactionImportService'); } = require('./csvTransactionImportService.cts');
type Row = Record<string, any>; type Row = Record<string, any>;

2
tests/csvTransactionImportService.test.js
View File

@ -11,7 +11,7 @@ const { getDb, closeDb } = require('../db/database');
const { const {
commitCsvTransactions, commitCsvTransactions,
previewCsvTransactions, previewCsvTransactions,
} = require('../services/csvTransactionImportService'); } = require('../services/csvTransactionImportService.cts');
function createUser(db) { function createUser(db) {
return db.prepare(` return db.prepare(`