refactor(server): migrate csvTransactionImportService to TypeScript (.cts)
The CSV parser + import primitives (also shared by the OFX importer). Behavior-preserving. Verified: typecheck:server 0, check:server 0, suite 226/226. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
null
parent
310eb07b9b
commit
99e4927640
|
|
@ -15,7 +15,7 @@ const {
|
|||
const {
|
||||
previewCsvTransactions,
|
||||
commitCsvTransactions,
|
||||
} = require('../services/csvTransactionImportService');
|
||||
} = require('../services/csvTransactionImportService.cts');
|
||||
const {
|
||||
previewOfxTransactions,
|
||||
commitOfxTransactions,
|
||||
|
|
|
|||
|
|
@ -1,14 +1,18 @@
|
|||
'use strict';
|
||||
|
||||
import type { Db } from '../types/db';
|
||||
|
||||
const crypto = require('crypto');
|
||||
const { getDb } = require('../db/database');
|
||||
const { decorateTransaction, ensureManualDataSource } = require('./transactionService.cts');
|
||||
|
||||
type Row = Record<string, any>;
|
||||
|
||||
const SESSION_TTL_MS = 24 * 60 * 60 * 1000;
|
||||
const MAX_ROWS = 25000;
|
||||
const SAMPLE_SIZE = 10;
|
||||
|
||||
const FIELD_LABELS = {
|
||||
const FIELD_LABELS: Record<string, string> = {
|
||||
posted_date: 'Posted date',
|
||||
transacted_at: 'Transaction date/time',
|
||||
amount: 'Amount',
|
||||
|
|
@ -24,32 +28,32 @@ const FIELD_LABELS = {
|
|||
currency: 'Currency',
|
||||
};
|
||||
|
||||
function importError(status, message, code, details = []) {
|
||||
const err = new Error(message);
|
||||
function importError(status: number, message: string, code: string, details: any[] = []): Error {
|
||||
const err = new Error(message) as any;
|
||||
err.status = status;
|
||||
err.code = code;
|
||||
err.details = details;
|
||||
return err;
|
||||
}
|
||||
|
||||
function makeSessionId() {
|
||||
function makeSessionId(): string {
|
||||
return crypto.randomBytes(16).toString('hex');
|
||||
}
|
||||
|
||||
function cleanFilename(value) {
|
||||
function cleanFilename(value: any): string | null {
|
||||
return value
|
||||
? String(value).replace(/[^a-zA-Z0-9._\-\s]/g, '').trim().slice(0, 255)
|
||||
: null;
|
||||
}
|
||||
|
||||
function normalizeHeader(value, index) {
|
||||
function normalizeHeader(value: any, index: number): string {
|
||||
const text = String(value ?? '').trim();
|
||||
return text || `Column ${index + 1}`;
|
||||
}
|
||||
|
||||
function parseCsv(text) {
|
||||
const rows = [];
|
||||
let row = [];
|
||||
function parseCsv(text: string): any[] {
|
||||
const rows: any[] = [];
|
||||
let row: any[] = [];
|
||||
let cell = '';
|
||||
let inQuotes = false;
|
||||
|
||||
|
|
@ -98,38 +102,38 @@ function parseCsv(text) {
|
|||
rows.push(row);
|
||||
}
|
||||
|
||||
return rows.filter(r => r.some(c => String(c ?? '').trim() !== ''));
|
||||
return rows.filter((r: any[]) => r.some((c: any) => String(c ?? '').trim() !== ''));
|
||||
}
|
||||
|
||||
function csvBufferToText(buffer) {
|
||||
function csvBufferToText(buffer: any): string {
|
||||
if (!Buffer.isBuffer(buffer) || buffer.length === 0) {
|
||||
throw importError(400, 'CSV file is required.', 'CSV_REQUIRED');
|
||||
}
|
||||
return buffer.toString('utf8').replace(/^\uFEFF/, '');
|
||||
return buffer.toString('utf8').replace(/^/, '');
|
||||
}
|
||||
|
||||
function rowToObject(headers, row) {
|
||||
const out = {};
|
||||
headers.forEach((header, index) => {
|
||||
function rowToObject(headers: string[], row: any[]): Row {
|
||||
const out: Row = {};
|
||||
headers.forEach((header: string, index: number) => {
|
||||
out[header] = String(row[index] ?? '').trim();
|
||||
});
|
||||
return out;
|
||||
}
|
||||
|
||||
function normalizeHeaderToken(value) {
|
||||
function normalizeHeaderToken(value: any): string {
|
||||
return String(value || '').toLowerCase().replace(/[^a-z0-9]+/g, ' ').trim();
|
||||
}
|
||||
|
||||
function headerMatches(header, patterns) {
|
||||
function headerMatches(header: any, patterns: any[]): boolean {
|
||||
const token = normalizeHeaderToken(header);
|
||||
return patterns.some(pattern => (
|
||||
return patterns.some((pattern: any) => (
|
||||
pattern instanceof RegExp ? pattern.test(token) : token === pattern
|
||||
));
|
||||
}
|
||||
|
||||
function suggestMapping(headers) {
|
||||
const mapping = {};
|
||||
const candidates = [
|
||||
function suggestMapping(headers: string[]): Row {
|
||||
const mapping: Row = {};
|
||||
const candidates: [string, any[]][] = [
|
||||
['posted_date', [/^date$/, 'posted date', 'post date', 'posting date', 'transaction date', 'trans date']],
|
||||
['transacted_at', ['authorized date', 'authorization date', 'datetime', 'date time', 'timestamp']],
|
||||
['transaction_id', ['transaction id', 'transaction number', 'trans id', 'id', 'fitid', 'reference', 'reference number']],
|
||||
|
|
@ -146,19 +150,19 @@ function suggestMapping(headers) {
|
|||
];
|
||||
|
||||
for (const [field, patterns] of candidates) {
|
||||
const found = headers.find(header => !Object.values(mapping).includes(header) && headerMatches(header, patterns));
|
||||
const found = headers.find((header: string) => !Object.values(mapping).includes(header) && headerMatches(header, patterns));
|
||||
if (found) mapping[field] = found;
|
||||
}
|
||||
|
||||
if (!mapping.payee && mapping.description) {
|
||||
const payee = headers.find(header => header !== mapping.description && headerMatches(header, ['name', 'merchant name']));
|
||||
const payee = headers.find((header: string) => header !== mapping.description && headerMatches(header, ['name', 'merchant name']));
|
||||
if (payee) mapping.payee = payee;
|
||||
}
|
||||
|
||||
return mapping;
|
||||
}
|
||||
|
||||
function parseCsvPreview(buffer, options = {}) {
|
||||
function parseCsvPreview(buffer: any, options: any = {}): any {
|
||||
const text = csvBufferToText(buffer);
|
||||
const parsed = parseCsv(text);
|
||||
if (parsed.length < 2) {
|
||||
|
|
@ -166,8 +170,8 @@ function parseCsvPreview(buffer, options = {}) {
|
|||
}
|
||||
|
||||
const headers = parsed[0].map(normalizeHeader);
|
||||
const seenHeaders = new Set();
|
||||
const duplicateHeaders = [];
|
||||
const seenHeaders = new Set<string>();
|
||||
const duplicateHeaders: any[] = [];
|
||||
for (const header of headers) {
|
||||
const key = header.toLowerCase();
|
||||
if (seenHeaders.has(key)) duplicateHeaders.push(header);
|
||||
|
|
@ -177,10 +181,10 @@ function parseCsvPreview(buffer, options = {}) {
|
|||
throw importError(400, `CSV contains duplicate headers: ${duplicateHeaders.join(', ')}`, 'CSV_DUPLICATE_HEADERS');
|
||||
}
|
||||
|
||||
const dataRows = parsed.slice(1).slice(0, MAX_ROWS).map(row => rowToObject(headers, row));
|
||||
const dataRows = parsed.slice(1).slice(0, MAX_ROWS).map((row: any[]) => rowToObject(headers, row));
|
||||
const truncated = parsed.length - 1 > MAX_ROWS;
|
||||
const suggestedMapping = suggestMapping(headers);
|
||||
const errors = [];
|
||||
const errors: any[] = [];
|
||||
|
||||
if (!suggestedMapping.posted_date) {
|
||||
errors.push({ field: 'posted_date', message: 'Could not detect a posted date column.' });
|
||||
|
|
@ -206,11 +210,11 @@ function parseCsvPreview(buffer, options = {}) {
|
|||
};
|
||||
}
|
||||
|
||||
function pruneExpiredSessions(db) {
|
||||
function pruneExpiredSessions(db: Db): void {
|
||||
db.prepare('DELETE FROM import_sessions WHERE expires_at <= ?').run(new Date().toISOString());
|
||||
}
|
||||
|
||||
function saveImportSession(db, userId, sessionData) {
|
||||
function saveImportSession(db: Db, userId: number, sessionData: any): string {
|
||||
const id = makeSessionId();
|
||||
const now = new Date().toISOString();
|
||||
const expiresAt = new Date(Date.now() + SESSION_TTL_MS).toISOString();
|
||||
|
|
@ -221,7 +225,7 @@ function saveImportSession(db, userId, sessionData) {
|
|||
return id;
|
||||
}
|
||||
|
||||
function loadImportSession(db, userId, sessionId) {
|
||||
function loadImportSession(db: Db, userId: number, sessionId: any): any {
|
||||
const row = db.prepare(`
|
||||
SELECT preview_json
|
||||
FROM import_sessions
|
||||
|
|
@ -234,12 +238,12 @@ function loadImportSession(db, userId, sessionId) {
|
|||
return JSON.parse(row.preview_json);
|
||||
}
|
||||
|
||||
function deleteImportSession(db, sessionId) {
|
||||
function deleteImportSession(db: Db, sessionId: any): void {
|
||||
db.prepare('DELETE FROM import_sessions WHERE id = ?').run(sessionId);
|
||||
}
|
||||
|
||||
function previewCsvTransactions(userId, buffer, options = {}) {
|
||||
const db = getDb();
|
||||
function previewCsvTransactions(userId: number, buffer: any, options: any = {}): any {
|
||||
const db: Db = getDb();
|
||||
pruneExpiredSessions(db);
|
||||
const preview = parseCsvPreview(buffer, options);
|
||||
const sessionId = saveImportSession(db, userId, {
|
||||
|
|
@ -258,13 +262,13 @@ function previewCsvTransactions(userId, buffer, options = {}) {
|
|||
};
|
||||
}
|
||||
|
||||
function headerValue(row, mapping, field) {
|
||||
function headerValue(row: Row, mapping: any, field: string): string {
|
||||
const header = mapping?.[field];
|
||||
if (!header) return '';
|
||||
return String(row[header] ?? '').trim();
|
||||
}
|
||||
|
||||
function parseDateValue(value, field, rowNumber) {
|
||||
function parseDateValue(value: any, field: string, rowNumber: number): string {
|
||||
const text = String(value || '').trim();
|
||||
if (!text) {
|
||||
throw importError(400, `${FIELD_LABELS[field] || field} is required`, 'CSV_ROW_VALIDATION', [
|
||||
|
|
@ -290,7 +294,7 @@ function parseDateValue(value, field, rowNumber) {
|
|||
]);
|
||||
}
|
||||
|
||||
function isRealDate(value) {
|
||||
function isRealDate(value: any): boolean {
|
||||
const [year, month, day] = String(value).split('-').map(Number);
|
||||
const date = new Date(Date.UTC(year, month - 1, day));
|
||||
return date.getUTCFullYear() === year
|
||||
|
|
@ -298,7 +302,7 @@ function isRealDate(value) {
|
|||
&& date.getUTCDate() === day;
|
||||
}
|
||||
|
||||
function parseCents(value, { negative = false } = {}) {
|
||||
function parseCents(value: any, { negative = false }: { negative?: boolean } = {}): number | null {
|
||||
const text = String(value ?? '').trim();
|
||||
if (!text) return null;
|
||||
const parenNegative = /^\(.*\)$/.test(text);
|
||||
|
|
@ -311,7 +315,7 @@ function parseCents(value, { negative = false } = {}) {
|
|||
return Math.round(Math.abs(number) * 100) * sign;
|
||||
}
|
||||
|
||||
function parseMappedAmount(row, mapping) {
|
||||
function parseMappedAmount(row: Row, mapping: any): number | null {
|
||||
const amount = parseCents(headerValue(row, mapping, 'amount'));
|
||||
if (amount !== null) return amount;
|
||||
|
||||
|
|
@ -324,15 +328,15 @@ function parseMappedAmount(row, mapping) {
|
|||
return null;
|
||||
}
|
||||
|
||||
function stableHash(parts) {
|
||||
function stableHash(parts: any[]): string {
|
||||
return crypto
|
||||
.createHash('sha256')
|
||||
.update(parts.map(part => String(part || '').trim().toLowerCase()).join('\u001f'))
|
||||
.update(parts.map((part: any) => String(part || '').trim().toLowerCase()).join(' '))
|
||||
.digest('hex')
|
||||
.slice(0, 48);
|
||||
}
|
||||
|
||||
function getOrCreateCsvDataSource(db, userId) {
|
||||
function getOrCreateCsvDataSource(db: Db, userId: number): Row {
|
||||
ensureManualDataSource(db, userId);
|
||||
const existing = db.prepare(`
|
||||
SELECT *
|
||||
|
|
@ -350,7 +354,7 @@ function getOrCreateCsvDataSource(db, userId) {
|
|||
return db.prepare('SELECT * FROM data_sources WHERE id = ? AND user_id = ?').get(result.lastInsertRowid, userId);
|
||||
}
|
||||
|
||||
function getOrCreateAccount(db, userId, dataSourceId, name) {
|
||||
function getOrCreateAccount(db: Db, userId: number, dataSourceId: any, name: any): Row | null {
|
||||
const accountName = String(name || '').trim();
|
||||
if (!accountName) return null;
|
||||
const providerAccountId = stableHash([accountName]).slice(0, 32);
|
||||
|
|
@ -369,13 +373,13 @@ function getOrCreateAccount(db, userId, dataSourceId, name) {
|
|||
return db.prepare('SELECT * FROM financial_accounts WHERE id = ? AND user_id = ?').get(result.lastInsertRowid, userId);
|
||||
}
|
||||
|
||||
function validateMapping(headers, mapping = {}) {
|
||||
function validateMapping(headers: any[], mapping: any = {}): void {
|
||||
const headerSet = new Set(headers);
|
||||
const required = [];
|
||||
const required: any[] = [];
|
||||
if (!mapping.posted_date) required.push('posted_date');
|
||||
if (!mapping.amount && !(mapping.debit_amount || mapping.credit_amount)) required.push('amount');
|
||||
if (required.length) {
|
||||
throw importError(400, `Missing required mapping: ${required.map(f => FIELD_LABELS[f] || f).join(', ')}`, 'CSV_MAPPING_REQUIRED');
|
||||
throw importError(400, `Missing required mapping: ${required.map((f: string) => FIELD_LABELS[f] || f).join(', ')}`, 'CSV_MAPPING_REQUIRED');
|
||||
}
|
||||
|
||||
for (const [field, header] of Object.entries(mapping)) {
|
||||
|
|
@ -388,7 +392,7 @@ function validateMapping(headers, mapping = {}) {
|
|||
}
|
||||
}
|
||||
|
||||
function parseOptionalDateTimeValue(value, field, rowNumber) {
|
||||
function parseOptionalDateTimeValue(value: any, field: string, rowNumber: number): string | null {
|
||||
const text = String(value || '').trim();
|
||||
if (!text) return null;
|
||||
|
||||
|
|
@ -401,7 +405,7 @@ function parseOptionalDateTimeValue(value, field, rowNumber) {
|
|||
return text;
|
||||
}
|
||||
|
||||
function normalizeCsvTransaction(row, mapping, rowNumber) {
|
||||
function normalizeCsvTransaction(row: Row, mapping: any, rowNumber: number): Row {
|
||||
const postedDate = parseDateValue(headerValue(row, mapping, 'posted_date'), 'posted_date', rowNumber);
|
||||
const transactedAt = parseOptionalDateTimeValue(headerValue(row, mapping, 'transacted_at'), 'transacted_at', rowNumber);
|
||||
const amount = parseMappedAmount(row, mapping);
|
||||
|
|
@ -436,8 +440,8 @@ function normalizeCsvTransaction(row, mapping, rowNumber) {
|
|||
};
|
||||
}
|
||||
|
||||
function commitCsvTransactions(userId, importSessionId, mapping, options = {}) {
|
||||
const db = getDb();
|
||||
function commitCsvTransactions(userId: number, importSessionId: any, mapping: any, options: any = {}): any {
|
||||
const db: Db = getDb();
|
||||
const session = loadImportSession(db, userId, importSessionId);
|
||||
if (session.kind !== 'csv_transactions') {
|
||||
throw importError(400, 'Import session is not a CSV transaction preview.', 'CSV_SESSION_INVALID');
|
||||
|
|
@ -445,7 +449,7 @@ function commitCsvTransactions(userId, importSessionId, mapping, options = {}) {
|
|||
|
||||
validateMapping(session.headers, mapping);
|
||||
const dataSource = getOrCreateCsvDataSource(db, userId);
|
||||
const details = [];
|
||||
const details: any[] = [];
|
||||
const counts = { imported: 0, skipped: 0, failed: 0 };
|
||||
const insert = db.prepare(`
|
||||
INSERT INTO transactions
|
||||
|
|
@ -461,7 +465,7 @@ function commitCsvTransactions(userId, importSessionId, mapping, options = {}) {
|
|||
`);
|
||||
|
||||
const run = db.transaction(() => {
|
||||
session.rows.forEach((row, index) => {
|
||||
session.rows.forEach((row: Row, index: number) => {
|
||||
const rowNumber = index + 2;
|
||||
try {
|
||||
const tx = normalizeCsvTransaction(row, mapping, rowNumber);
|
||||
|
|
@ -508,7 +512,7 @@ function commitCsvTransactions(userId, importSessionId, mapping, options = {}) {
|
|||
ignored: 0,
|
||||
}),
|
||||
});
|
||||
} catch (err) {
|
||||
} catch (err: any) {
|
||||
counts.failed++;
|
||||
details.push({ row: rowNumber, result: 'failed', message: err.message, details: err.details || [] });
|
||||
}
|
||||
|
|
@ -553,9 +557,6 @@ module.exports = {
|
|||
FIELD_LABELS,
|
||||
commitCsvTransactions,
|
||||
previewCsvTransactions,
|
||||
// Reusable transaction-import primitives (shared by the OFX/QFX importer so it
|
||||
// dedupes and sessions identically — same import_sessions table, same
|
||||
// (user_id, data_source_id, provider_transaction_id) dedupe scope).
|
||||
saveImportSession,
|
||||
loadImportSession,
|
||||
deleteImportSession,
|
||||
|
|
@ -21,7 +21,7 @@ const {
|
|||
getOrCreateAccount,
|
||||
stableHash,
|
||||
parseCents,
|
||||
} = require('./csvTransactionImportService');
|
||||
} = require('./csvTransactionImportService.cts');
|
||||
|
||||
type Row = Record<string, any>;
|
||||
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ const { getDb, closeDb } = require('../db/database');
|
|||
const {
|
||||
commitCsvTransactions,
|
||||
previewCsvTransactions,
|
||||
} = require('../services/csvTransactionImportService');
|
||||
} = require('../services/csvTransactionImportService.cts');
|
||||
|
||||
function createUser(db) {
|
||||
return db.prepare(`
|
||||
|
|
|
|||
Loading…
Reference in New Issue