BillTracker/services/ofxImportService.cts

'use strict';

// OFX / QFX transaction import. OFX 1.x is SGML (leaf tags often unclosed); OFX
// 2.x is XML; QFX is OFX plus Intuit-specific tags we can ignore. We extract the
// <STMTTRN> blocks and read each leaf tag's value up to the next '<' — which works
// for both the closed (XML) and unclosed (SGML) forms.
//
// Normalized transactions are written through the SAME session + dedupe + insert
// path as the CSV importer (shared primitives from csvTransactionImportService),
// so dedupe scope, the import_sessions table, and import_history are identical.

import type { Db } from '../types/db';

const { getDb } = require('../db/database.cts');
const { decorateTransaction, ensureManualDataSource } = require('./transactionService.cts');
const {
  saveImportSession,
  loadImportSession,
  deleteImportSession,
  pruneExpiredSessions,
  getOrCreateAccount,
  stableHash,
  parseCents,
} = require('./csvTransactionImportService.cts');

type Row = Record<string, any>;

const MAX_TX = 25000;

function importError(status: number, message: string, code: string, details: any[] = []): Error {
  const err = new Error(message) as any;
  err.status = status;
  err.code = code;
  err.details = details;
  return err;
}

// Read the first value of <TAG> in `block`, up to the next '<' or line end.
function tagValue(block: string, tag: string): string {
  const m = new RegExp(`<${tag}>([^<\\r\\n]*)`, 'i').exec(block);
  return m ? m[1].trim() : '';
}

// OFX date: YYYYMMDD[HHMMSS[.XXX]][ tz ] → 'YYYY-MM-DD' (posted date; tz dropped).
function ofxDate(value: unknown): string | null {
  const m = /^(\d{4})(\d{2})(\d{2})/.exec(String(value || '').trim());
  if (!m) return null;
  const [, y, mo, d] = m;
  const month = Number(mo);
  const day = Number(d);
  if (month < 1 || month > 12 || day < 1 || day > 31) return null;
  return `${y}-${mo}-${d}`;
}

function decodeEntities(s: unknown): string {
  return String(s || '')
    .replace(/&amp;/gi, '&').replace(/&lt;/gi, '<').replace(/&gt;/gi, '>')
    .replace(/&#(\d+);/g, (_: string, n: string) => String.fromCharCode(Number(n)))
    .trim();
}

/**
 * Parse an OFX/QFX buffer into normalized transactions (same shape the CSV path
 * produces). Throws importError on a file with no parsable transactions.
 */
function parseOfx(buffer: any): Row[] {
  const text = Buffer.isBuffer(buffer) ? buffer.toString('utf8') : String(buffer || '');
  if (!/<OFX>/i.test(text) && !/<STMTTRN>/i.test(text)) {
    throw importError(400, 'This does not look like an OFX/QFX file.', 'OFX_INVALID');
  }

  // Account id (best effort) → a stable account name.
  const acctId = tagValue(text, 'ACCTID');
  const curdef = (tagValue(text, 'CURDEF') || 'USD').toUpperCase().slice(0, 3) || 'USD';
  const accountName = acctId ? `OFX ${acctId.slice(-4).padStart(4, '•')}` : 'OFX import';

  const blocks = text.match(/<STMTTRN>[\s\S]*?<\/STMTTRN>/gi)
    || text.match(/<STMTTRN>[\s\S]*?(?=<STMTTRN>|<\/BANKTRANLIST>|<\/OFX>)/gi)
    || [];

  const transactions: Row[] = [];
  for (const raw of blocks) {
    if (transactions.length >= MAX_TX) break;
    const postedDate = ofxDate(tagValue(raw, 'DTPOSTED'));
    const amount = parseCents(tagValue(raw, 'TRNAMT'));
    if (!postedDate || amount === null || amount === 0) continue; // skip incomplete rows

    const fitid = tagValue(raw, 'FITID');
    const name = decodeEntities(tagValue(raw, 'NAME'));
    const memo = decodeEntities(tagValue(raw, 'MEMO'));
    const trntype = tagValue(raw, 'TRNTYPE') || null;
    const providerTransactionId = fitid
      ? `ofx:id:${fitid}`
      : `ofx:hash:${stableHash([postedDate, amount, name, memo])}`;

    transactions.push({
      provider_transaction_id: providerTransactionId,
      transaction_type: trntype,
      posted_date: postedDate,
      transacted_at: null,
      amount,
      currency: curdef,
      description: name || memo || null,
      payee: name || null,
      memo: memo || null,
      category: null,
      account_name: accountName,
      raw_data: null,
    });
  }

  if (transactions.length === 0) {
    throw importError(400, 'No transactions found in the OFX/QFX file.', 'OFX_EMPTY');
  }
  return transactions;
}

function getOrCreateOfxDataSource(db: Db, userId: number): Row {
  ensureManualDataSource(db, userId);
  const existing = db.prepare(`
    SELECT * FROM data_sources
    WHERE user_id = ? AND type = 'file_import' AND provider = 'ofx' AND name = 'OFX Import'
    ORDER BY id ASC LIMIT 1
  `).get(userId);
  if (existing) return existing;
  const result = db.prepare(`
    INSERT INTO data_sources (user_id, type, provider, name, status)
    VALUES (?, 'file_import', 'ofx', 'OFX Import', 'active')
  `).run(userId);
  return db.prepare('SELECT * FROM data_sources WHERE id = ? AND user_id = ?').get(result.lastInsertRowid, userId);
}

function previewOfxTransactions(userId: number, buffer: any, options: any = {}) {
  const db: Db = getDb();
  pruneExpiredSessions(db);
  const transactions = parseOfx(buffer);
  const sessionId = saveImportSession(db, userId, {
    kind: 'ofx_transactions',
    original_filename: options.original_filename || null,
    transactions,
  });
  return {
    import_session_id: sessionId,
    count: transactions.length,
    // A small sample for the confirm screen (money kept in cents; client formats).
    sample: transactions.slice(0, 12),
  };
}

function commitOfxTransactions(userId: number, importSessionId: any, options: any = {}) {
  const db: Db = getDb();
  const session = loadImportSession(db, userId, importSessionId);
  if (session.kind !== 'ofx_transactions') {
    throw importError(400, 'Import session is not an OFX/QFX preview.', 'OFX_SESSION_INVALID');
  }

  const dataSource = getOrCreateOfxDataSource(db, userId);
  const counts = { imported: 0, skipped: 0, failed: 0 };
  const details: any[] = [];
  const insert = db.prepare(`
    INSERT INTO transactions
      (user_id, data_source_id, account_id, provider_transaction_id, source_type,
       transaction_type, posted_date, transacted_at, amount, currency, description,
       payee, memo, category, raw_data, match_status, ignored)
    VALUES (?, ?, ?, ?, 'file_import', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'unmatched', 0)
  `);
  const existing = db.prepare(
    'SELECT id FROM transactions WHERE user_id = ? AND data_source_id = ? AND provider_transaction_id = ?',
  );

  const run = db.transaction(() => {
    (session.transactions || []).forEach((tx: Row, index: number) => {
      try {
        if (existing.get(userId, dataSource.id, tx.provider_transaction_id)) {
          counts.skipped++;
          details.push({ row: index + 1, result: 'skipped_duplicate', provider_transaction_id: tx.provider_transaction_id });
          return;
        }
        const account = getOrCreateAccount(db, userId, dataSource.id, tx.account_name);
        const result = insert.run(
          userId, dataSource.id, account?.id ?? null, tx.provider_transaction_id,
          tx.transaction_type, tx.posted_date, tx.transacted_at, tx.amount, tx.currency,
          tx.description, tx.payee, tx.memo, tx.category, tx.raw_data,
        );
        counts.imported++;
        details.push({
          row: index + 1,
          result: 'imported',
          transaction: decorateTransaction({
            ...tx, id: result.lastInsertRowid, user_id: userId, data_source_id: dataSource.id,
            source_type: 'file_import', account_id: account?.id ?? null, match_status: 'unmatched', ignored: 0,
          }),
        });
      } catch (err: any) {
        counts.failed++;
        details.push({ row: index + 1, result: 'failed', message: err.message });
      }
    });

    db.prepare(`
      INSERT INTO import_history (
        user_id, imported_at, source_filename, file_type, sheet_name,
        rows_parsed, rows_created, rows_updated, rows_skipped, rows_ambiguous,
        rows_errored, options_json, summary_json
      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
    `).run(
      userId, new Date().toISOString(), session.original_filename, 'ofx_transactions', null,
      (session.transactions || []).length, counts.imported, 0, counts.skipped, 0, counts.failed,
      JSON.stringify({ options }), JSON.stringify(details.slice(0, 500)),
    );
  });

  run();
  deleteImportSession(db, importSessionId);
  return { success: true, imported: counts.imported, skipped: counts.skipped, failed: counts.failed, details };
}

module.exports = { parseOfx, previewOfxTransactions, commitOfxTransactions };