BillTracker/services/bankSyncService.cts

'use strict';

import type { Db } from '../types/db';

const { assertEncryptionReady, encryptSecret, decryptSecret } = require('./encryptionService.cts');
const {
  claimSetupToken,
  fetchAccountsAndTransactions,
  normalizeAccount,
  normalizeTransaction,
  sanitizeErrorMessage,
} = require('./simplefinService.cts');
const {
  getBankSyncConfig,
  SYNC_DAYS_EFFECTIVE,
  SYNC_DAYS_DEFAULT,
} = require('./bankSyncConfigService.cts');
const { decorateDataSource } = require('./transactionService.cts');
const { applyMerchantRules } = require('./billMerchantRuleService.cts');
const { applySpendingCategoryRules } = require('./spendingService.cts');
const { applyMerchantStoreMatches } = require('./merchantStoreMatchService.cts');
const { autoMatchForUser } = require('./matchSuggestionService.cts');
const { getUserSettings } = require('./userSettings.cts');

type Row = Record<string, any>;

function sinceEpochDays(days: number): number {
  return Math.floor((Date.now() - days * 86400 * 1000) / 1000);
}

function safeErrorMessage(err: any): string {
  return sanitizeErrorMessage(err?.message || String(err || 'Sync failed'));
}

// Upsert a single financial account, return the local row.
function upsertAccount(db: Db, accountRow: Row): Row {
  const existing = db
    .prepare(
      `
    SELECT id, monitored FROM financial_accounts
    WHERE data_source_id = ? AND provider_account_id = ? AND user_id = ?
  `,
    )
    .get(accountRow.data_source_id, accountRow.provider_account_id, accountRow.user_id);

  if (existing) {
    db.prepare(
      `
      UPDATE financial_accounts
      SET name = ?, org_name = ?, currency = ?, balance = ?, available_balance = ?,
          raw_data = ?, updated_at = datetime('now')
      WHERE id = ?
    `,
    ).run(
      accountRow.name,
      accountRow.org_name,
      accountRow.currency,
      accountRow.balance,
      accountRow.available_balance,
      accountRow.raw_data,
      existing.id,
    );
    return existing;
  }

  const result = db
    .prepare(
      `
    INSERT INTO financial_accounts
      (user_id, data_source_id, provider_account_id, name, org_name, account_type,
       currency, balance, available_balance, raw_data)
    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
  `,
    )
    .run(
      accountRow.user_id,
      accountRow.data_source_id,
      accountRow.provider_account_id,
      accountRow.name,
      accountRow.org_name,
      accountRow.account_type,
      accountRow.currency,
      accountRow.balance,
      accountRow.available_balance,
      accountRow.raw_data,
    );
  return { id: result.lastInsertRowid, monitored: 1 };
}

// Insert a transaction, or update one we previously stored as PENDING.
// Returns: 'inserted' | 'posted' (pending→settled) | 'updated' (pending refreshed) | 'skipped'.
function upsertTransaction(db: Db, txRow: Row): string {
  const existing = db
    .prepare(
      `
    SELECT id, pending, match_status FROM transactions
    WHERE user_id = ? AND provider_transaction_id = ?
  `,
    )
    .get(txRow.user_id, txRow.provider_transaction_id);

  if (!existing) {
    try {
      db.prepare(
        `
        INSERT INTO transactions
          (user_id, data_source_id, account_id, provider_transaction_id,
           source_type, posted_date, transacted_at, amount, currency,
           description, payee, memo, match_status, ignored, pending, raw_data)
        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
      `,
      ).run(
        txRow.user_id,
        txRow.data_source_id,
        txRow.account_id,
        txRow.provider_transaction_id,
        txRow.source_type,
        txRow.posted_date,
        txRow.transacted_at,
        txRow.amount,
        txRow.currency,
        txRow.description,
        txRow.payee,
        txRow.memo,
        txRow.match_status,
        txRow.ignored,
        txRow.pending,
        txRow.raw_data,
      );
      return 'inserted';
    } catch (err: any) {
      if (err.code === 'SQLITE_CONSTRAINT_UNIQUE' || (err.message || '').includes('UNIQUE')) {
        return 'skipped';
      }
      throw err;
    }
  }

  // Only refresh rows still pending in our DB and not yet acted on by the user.
  if (existing.pending === 1 && existing.match_status === 'unmatched') {
    db.prepare(
      `
      UPDATE transactions
      SET posted_date = ?, transacted_at = ?, amount = ?, currency = ?,
          description = ?, payee = ?, memo = ?, pending = ?, raw_data = ?,
          updated_at = datetime('now')
      WHERE id = ?
    `,
    ).run(
      txRow.posted_date,
      txRow.transacted_at,
      txRow.amount,
      txRow.currency,
      txRow.description,
      txRow.payee,
      txRow.memo,
      txRow.pending,
      txRow.raw_data,
      existing.id,
    );
    return txRow.pending === 0 ? 'posted' : 'updated';
  }

  return 'skipped';
}

async function runSync(
  db: Db,
  userId: number,
  dataSource: Row,
  { days, debug = false }: { days?: number; debug?: boolean } = {},
): Promise<any> {
  const accessUrl = decryptSecret(dataSource.encrypted_secret);
  const isFirstSync = !dataSource.last_sync_at;
  const config = getBankSyncConfig();
  const syncDays =
    days ?? (isFirstSync ? SYNC_DAYS_EFFECTIVE : config.sync_days || SYNC_DAYS_DEFAULT);
  const since = sinceEpochDays(syncDays);

  if (debug)
    console.log(
      `[bankSync:debug] Source #${dataSource.id} user ${userId}: fetching ${syncDays} days from SimpleFIN (since epoch ${since})`,
    );

  const raw = await fetchAccountsAndTransactions(accessUrl, since);
  const accounts = Array.isArray(raw.accounts) ? raw.accounts : [];

  if (debug)
    console.log(
      `[bankSync:debug] Source #${dataSource.id}: SimpleFIN returned ${accounts.length} account(s)`,
    );

  if (raw._errlistSummary) {
    console.warn(`[bankSync] errlist for source ${dataSource.id}: ${raw._errlistSummary}`);
  }

  let accountsUpserted = 0;
  let transactionsNew = 0;
  let transactionsSkip = 0;
  let transactionsPosted = 0; // pending → settled this cycle
  let pendingCleared = 0; // stale pending rows pruned (re-posted under new id / dropped)

  const pruneOrphanPending = db.prepare(`
    DELETE FROM transactions
    WHERE account_id = ? AND user_id = ? AND pending = 1 AND match_status = 'unmatched'
      AND provider_transaction_id NOT IN (SELECT value FROM json_each(?))
  `);

  const partialError = raw._errlistSummary
    ? sanitizeErrorMessage(`Partial sync — some connections failed: ${raw._errlistSummary}`)
    : null;

  const updateSource = db.prepare(`
    UPDATE data_sources
    SET last_sync_at = datetime('now'), last_error = ?, status = 'active', updated_at = datetime('now')
    WHERE id = ? AND user_id = ?
  `);

  const ingest = db.transaction(() => {
    for (const rawAccount of accounts) {
      const accountRow = normalizeAccount(rawAccount, dataSource.id, userId);
      const localAccount = upsertAccount(db, accountRow);
      accountsUpserted += 1;

      const txList = rawAccount.transactions || [];
      if (debug)
        console.log(
          `[bankSync:debug]   Account "${rawAccount.name}" (monitored=${localAccount.monitored}): ${txList.length} transaction(s)`,
        );

      if (localAccount.monitored === 0) continue;

      const seenTxIds: any[] = [];
      for (const rawTx of txList) {
        const txRow = normalizeTransaction(
          rawTx,
          localAccount.id,
          dataSource.id,
          userId,
          rawAccount.id,
          rawAccount.currency,
        );
        seenTxIds.push(txRow.provider_transaction_id);
        const outcome = upsertTransaction(db, txRow);
        if (outcome === 'inserted') {
          transactionsNew += 1;
          if (debug)
            console.log(
              `[bankSync:debug]     tx ${txRow.provider_transaction_id}: inserted${txRow.pending ? ' (pending)' : ''} (${rawTx.description || rawTx.payee || '—'}, ${txRow.amount}¢)`,
            );
        } else if (outcome === 'posted') {
          transactionsPosted += 1;
          if (debug)
            console.log(
              `[bankSync:debug]     tx ${txRow.provider_transaction_id}: pending → posted`,
            );
        } else if (outcome === 'updated') {
          if (debug)
            console.log(
              `[bankSync:debug]     tx ${txRow.provider_transaction_id}: pending amount refreshed`,
            );
        } else {
          transactionsSkip += 1;
          if (debug)
            console.log(
              `[bankSync:debug]     tx ${txRow.provider_transaction_id}: duplicate — skipped`,
            );
        }
      }

      const orphans = pruneOrphanPending.run(localAccount.id, userId, JSON.stringify(seenTxIds));
      if (orphans.changes > 0) {
        pendingCleared += orphans.changes;
        if (debug)
          console.log(
            `[bankSync:debug]   Account "${rawAccount.name}": pruned ${orphans.changes} stale pending row(s)`,
          );
      }
    }

    updateSource.run(partialError, dataSource.id, userId);
  });
  ingest();

  if (debug)
    console.log(`[bankSync:debug] Source #${dataSource.id}: applying merchant rules + auto-match`);

  // Apply stored merchant→bill rules, then spending category rules, then score-based auto-match
  const {
    matched: autoMatched,
    matched_bills: matchedBills,
    late_attributions: lateAttributions,
  } = applyMerchantRules(db, userId);
  try {
    applySpendingCategoryRules(db, userId);
  } catch {
    /* non-blocking */
  }
  try {
    if (getUserSettings(userId).bank_auto_categorize_merchants !== 'false') {
      applyMerchantStoreMatches(db, userId);
    }
  } catch {
    /* non-blocking */
  }
  try {
    autoMatchForUser(userId);
  } catch {
    /* non-blocking */
  }

  if (debug)
    console.log(
      `[bankSync:debug] Source #${dataSource.id}: auto-matched ${autoMatched} transaction(s)`,
    );

  return {
    accountsUpserted,
    transactionsNew,
    transactionsSkip,
    transactionsPosted,
    pendingCleared,
    autoMatched,
    matched_bills: matchedBills || [],
    late_attributions: lateAttributions || [],
    errlist: raw._errlistSummary || null,
  };
}

// ─── Public API ───────────────────────────────────────────────────────────────

async function connectSimplefin(db: Db, userId: number, setupToken: any): Promise<any> {
  assertEncryptionReady();

  const accessUrl = await claimSetupToken(setupToken);
  const encrypted = encryptSecret(accessUrl);

  const result = db
    .prepare(
      `
    INSERT INTO data_sources (user_id, type, provider, name, status, encrypted_secret)
    VALUES (?, 'provider_sync', 'simplefin', 'SimpleFIN', 'active', ?)
  `,
    )
    .run(userId, encrypted);

  const dataSourceId = result.lastInsertRowid;
  const dataSource = db
    .prepare('SELECT * FROM data_sources WHERE id = ? AND user_id = ?')
    .get(dataSourceId, userId);

  let syncResult: any = { accountsUpserted: 0, transactionsNew: 0, transactionsSkip: 0 };
  try {
    syncResult = await runSync(db, userId, dataSource);
  } catch (err) {
    const msg = safeErrorMessage(err);
    db.prepare(
      `
      UPDATE data_sources SET last_error = ?, status = 'error', updated_at = datetime('now')
      WHERE id = ?
    `,
    ).run(msg, dataSourceId);
  }

  const fresh = db
    .prepare('SELECT * FROM data_sources WHERE id = ? AND user_id = ?')
    .get(dataSourceId, userId);
  return { dataSource: decorateDataSource(fresh), ...syncResult };
}

async function syncDataSource(
  db: Db,
  userId: number,
  dataSourceId: any,
  { debug }: { debug?: boolean } = {},
): Promise<any> {
  assertEncryptionReady();

  const dataSource = db
    .prepare(
      `
    SELECT * FROM data_sources
    WHERE id = ? AND user_id = ? AND type = 'provider_sync' AND provider = 'simplefin'
  `,
    )
    .get(dataSourceId, userId);

  if (!dataSource)
    throw Object.assign(new Error('SimpleFIN connection not found'), { status: 404 });
  if (!dataSource.encrypted_secret) throw new Error('No stored credentials for this connection');

  const useDebug = debug ?? getBankSyncConfig().debug_logging;

  let syncResult;
  try {
    syncResult = await runSync(db, userId, dataSource, { debug: useDebug });
  } catch (err) {
    const msg = safeErrorMessage(err);
    db.prepare(
      `
      UPDATE data_sources SET last_error = ?, status = 'error', updated_at = datetime('now')
      WHERE id = ?
    `,
    ).run(msg, dataSourceId);
    // Re-throw so the route can surface a meaningful error
    throw err;
  }

  const fresh = db
    .prepare('SELECT * FROM data_sources WHERE id = ? AND user_id = ?')
    .get(dataSourceId, userId);
  return { dataSource: decorateDataSource(fresh), ...syncResult };
}

async function backfillDataSource(db: Db, userId: number, dataSourceId: any): Promise<any> {
  assertEncryptionReady();

  const dataSource = db
    .prepare(
      `
    SELECT * FROM data_sources
    WHERE id = ? AND user_id = ? AND type = 'provider_sync' AND provider = 'simplefin'
  `,
    )
    .get(dataSourceId, userId);

  if (!dataSource)
    throw Object.assign(new Error('SimpleFIN connection not found'), { status: 404 });
  if (!dataSource.encrypted_secret) throw new Error('No stored credentials for this connection');

  let syncResult;
  try {
    syncResult = await runSync(db, userId, dataSource, { days: SYNC_DAYS_EFFECTIVE });
  } catch (err) {
    const msg = safeErrorMessage(err);
    db.prepare(
      `
      UPDATE data_sources SET last_error = ?, status = 'error', updated_at = datetime('now')
      WHERE id = ?
    `,
    ).run(msg, dataSourceId);
    throw err;
  }

  const fresh = db
    .prepare('SELECT * FROM data_sources WHERE id = ? AND user_id = ?')
    .get(dataSourceId, userId);
  return { dataSource: decorateDataSource(fresh), ...syncResult };
}

function disconnectDataSource(db: Db, userId: number, dataSourceId: any): void {
  const row = db
    .prepare(
      `
    SELECT id FROM data_sources WHERE id = ? AND user_id = ? AND provider = 'simplefin'
  `,
    )
    .get(dataSourceId, userId);

  if (!row) throw Object.assign(new Error('SimpleFIN connection not found'), { status: 404 });

  // Financial accounts cascade-delete. Transactions get data_source_id = NULL (SET NULL FK).
  db.prepare('DELETE FROM data_sources WHERE id = ? AND user_id = ?').run(dataSourceId, userId);
}

module.exports = { connectSimplefin, syncDataSource, backfillDataSource, disconnectDataSource };