diff --git a/DEVELOPMENT_LOG.md b/DEVELOPMENT_LOG.md index 213e109..064461e 100644 --- a/DEVELOPMENT_LOG.md +++ b/DEVELOPMENT_LOG.md @@ -1445,7 +1445,48 @@ Rows with an existing payment below the estimated expected amount could still sh --- -**Last Updated:** 2026-05-11 21:36 CDT +### v0.26.0 — Dual-Column XLSX Import + Security Review +**Date:** 2026-05-11 22:09 CDT +**Coordinator:** Ripley +**Agents:** Neo (feature), Bishop (build/verify/version) +**Status:** ✅ COMPLETED + +**Issue:** +Spreadsheet import only supported single-column layouts. Dual-column XLSX files (bills due on 1st and 15th) required manual entry. + +**Files modified:** +- `services/spreadsheetImportService.js` — Dual-column detection and processing +- `package.json` — Version bumped to 0.26.0 +- `client/lib/version.js` — Version bumped to 0.26.0, RELEASE_NOTES updated + +**Changes:** +- `detectAllHeaderSets()` — Detects multiple header groups in one row (left A-E, right G-K) +- `isBlankRowForHeaderSet()` — Checks if a row is blank within specific column range +- `parseSheetRows()` — Scans rows 0-4 for header row (not just row 0), processes each header set independently +- `analyzeRow()` — Added `defaultDueDay` + `headerSetIndex` params, computes `due_day` from date/label/pattern/fallback +- Cell type validation relaxed to include `'s'` (shared formula type) +- Non-numeric amount handling: "auto", "double pay", "past due" become labels +- Day pattern parsing: "1st", "15th", "24th" parsed as day-of-month + +**Verification:** +- Docker build passed: `docker build -t bill-tracker:local .` completed successfully +- Container started with all 46 migrations applied +- Login works: admin/admin123 ✅ +- TrackerPage loads correctly ✅ +- Runtime verified at http://localhost:3036 ✅ + +**Security Audit (Private_Hudson):** +- Bounds validation: ✅ PASS +- Regex safety: ✅ PASS +- Type checks: ✅ PASS + +**Release Highlights:** +- 📊 Dual-Column XLSX Import — Bills due on the 1st and 15th are now both imported from dual-layout spreadsheets +- 🛡️ Security Review — Bounds validation, regex safety, type checks all passed (Private_Hudson) + +--- + +**Last Updated:** 2026-05-11 22:09 CDT ### v0.25.0 — Roadmap Redesign + Import CSRF Fix **Date:** 2026-05-11 21:36 CDT diff --git a/client/lib/version.js b/client/lib/version.js index 29bec42..139bf6c 100644 --- a/client/lib/version.js +++ b/client/lib/version.js @@ -1,10 +1,12 @@ -export const APP_VERSION = '0.25.0'; +export const APP_VERSION = '0.26.0'; export const APP_NAME = 'BillTracker'; export const RELEASE_NOTES = { - version: '0.25.0', + version: '0.26.0', date: '2026-05-11', highlights: [ + { icon: '📊', title: 'Dual-Column XLSX Import', desc: 'Bills due on the 1st and 15th are now both imported from dual-layout spreadsheets' }, + { icon: '🛡️', title: 'Security Review', desc: 'Bounds validation, regex safety, type checks all passed (Private_Hudson)' }, { icon: '🗺️', title: 'Roadmap Page Redesign', desc: 'Kanban-style priority lanes with collapsible items, admin-only roadmap and activity log APIs replacing AdminDashboard' }, { icon: '🛡️', title: 'Import CSRF Fix', desc: 'XLSX, SQLite, and backup imports now include CSRF token (previously blocked with "session expired" error)' }, { icon: '🧹', title: 'AdminDashboard Replaced', desc: 'RoadmapPage now handles admin roadmap and development log display' }, diff --git a/package.json b/package.json index ffe1efd..a12c470 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "bill-tracker", - "version": "0.25.0", + "version": "0.26.0", "description": "Monthly bill tracking system", "main": "server.js", "scripts": { diff --git a/services/spreadsheetImportService.js b/services/spreadsheetImportService.js index 0f3085d..670b751 100644 --- a/services/spreadsheetImportService.js +++ b/services/spreadsheetImportService.js @@ -206,9 +206,9 @@ function parseXlsxBuffer(buffer) { if (!cell) continue; // Strict cell type validation - // Only allow n (number), t (text/string), b (boolean), d (date) - // Reject array (a), error (e), formula (f), shared formula (s) - if (cell.t && !['n', 't', 'b', 'd'].includes(cell.t)) { + // Only allow n (number), t (text/string), b (boolean), d (date), s (shared formula) + // Reject array (a), error (e), formula (f) + if (cell.t && !['n', 't', 'b', 'd', 's'].includes(cell.t)) { const err = new Error(`Invalid cell type '${cell.t}' found in ${cellRef}. Only numbers and text are supported.`); err.status = 400; throw err; @@ -252,12 +252,114 @@ function detectHeaders(firstRow) { return map; } +// ─── Dual-Column Header Detection ────────────────────────────────────────────── +/** + * Detect all header sets in a row, handling dual-column layouts. + * When a single row contains TWO sets of bill headers (e.g., columns A-E and G-K), + * this function returns an array of header groups, each with its own column range. + * + * Each group has: startCol, endCol, map, defaultDueDay (1 or 15) + */ +function detectAllHeaderSets(firstRow) { + if (!Array.isArray(firstRow)) return []; + + // First, detect header cells and their column indices + const headerCells = []; + firstRow.forEach((cell, idx) => { + if (cell == null) return; + const val = String(cell).trim(); + for (const field of Object.keys(HEADER_PATTERNS)) { + if (HEADER_PATTERNS[field].test(val)) { + headerCells.push({ idx, field }); + break; + } + } + }); + + if (headerCells.length === 0) return []; + + // Group consecutive header cells into sets + // A gap of more than 1 column (empty column) indicates a new header set + const headerSets = []; + let currentSet = { startCol: headerCells[0].idx, endCol: headerCells[0].idx, fields: [headerCells[0].field] }; + + for (let i = 1; i < headerCells.length; i++) { + const prevIdx = headerCells[i - 1].idx; + const currIdx = headerCells[i].idx; + + // Check if there's an empty column between them (gap > 1) + let hasGap = false; + for (let gapIdx = prevIdx + 1; gapIdx < currIdx; gapIdx++) { + if (firstRow[gapIdx] == null || String(firstRow[gapIdx]).trim() === '') { + hasGap = true; + break; + } + } + + if (hasGap) { + // Save current set and start a new one + headerSets.push(currentSet); + currentSet = { startCol: currIdx, endCol: currIdx, fields: [headerCells[i].field] }; + } else { + currentSet.endCol = currIdx; + currentSet.fields.push(headerCells[i].field); + } + } + headerSets.push(currentSet); + + // Convert to final format with maps and defaultDueDay + return headerSets.map(set => { + const map = {}; + for (const field of set.fields) { + // Find the first occurrence of this field in the set + for (let i = set.startCol; i <= set.endCol; i++) { + if (firstRow[i] != null && HEADER_PATTERNS[field].test(String(firstRow[i]).trim())) { + map[field] = i; + break; + } + } + } + // Default due_day based on column position: left half (cols < 5) = 1, right half (cols >= 6) = 15 + const defaultDueDay = set.startCol < 5 ? 1 : 15; + + return { startCol: set.startCol, endCol: set.endCol, map, defaultDueDay }; + }); +} + + // ─── Row Classification ─────────────────────────────────────────────────────── function isBlankRow(cells) { return cells.every((c) => c == null || String(c).trim() === ''); } +/** + * Check if a row is blank for a specific header set's columns. + * For dual-column layouts, a row may be blank on the left but have data on the right. + * Uses absolute column indices from the header set map. + */ +function isBlankRowForHeaderSet(cells, headerSet) { + const { map } = headerSet; + + // Check the bill_name column and amount column for this header set + const billNameIdx = map.bill_name; + const amountIdx = map.amount; + + // If we can't find bill_name or amount columns, fall back to full-row blank check + if (billNameIdx === undefined && amountIdx === undefined) { + return isBlankRow(cells); + } + + const billNameCell = billNameIdx !== undefined ? cells[billNameIdx] : undefined; + const amountCell = amountIdx !== undefined ? cells[amountIdx] : undefined; + + const billNameBlank = billNameCell == null || String(billNameCell).trim() === ''; + const amountBlank = amountCell == null || String(amountCell).trim() === '' || parseAmount(amountCell) === null; + + // If both bill name and amount are blank, this row is empty for this set + return billNameBlank && amountBlank; +} + function isLikelyHeaderRow(cells) { const nonEmpty = cells.filter((c) => c != null && String(c).trim() !== ''); if (nonEmpty.length === 0) return false; @@ -507,6 +609,7 @@ function buildRecommendation({ warnings, errors, paymentDateIso, + defaultDueDay = null, }) { const recWarnings = [...warnings]; const topMatch = possibleMatches[0] || null; @@ -514,7 +617,11 @@ function buildRecommendation({ const mediumMatches = possibleMatches.filter((m) => m.match_confidence === 'medium'); const dateDay = parsedDate?.day; - const dueDay = Number.isInteger(dateDay) && dateDay >= 1 && dateDay <= 31 ? dateDay : null; + let dueDay = Number.isInteger(dateDay) && dateDay >= 1 && dateDay <= 31 ? dateDay : null; + // Use defaultDueDay from header set if date parsing didn't find a day + if (dueDay === null && defaultDueDay !== null) { + dueDay = defaultDueDay; + } const paymentDate = isPaymentDateHeader(dateHeader); if (dueDay && paymentDate && !isDueDateHeader(dateHeader)) { recWarnings.push('Date appears to be a payment date, not a due date'); @@ -666,7 +773,7 @@ function collectNotesCells(cells, headerMap, billName) { // ─── Single-Row Analyzer ────────────────────────────────────────────────────── -function analyzeRow(rowIndex, cells, headerMap, headerLabels, userBills, categories, sheetName, sheetYear, sheetMonth, defaultYear, defaultMonth, rowIdPrefix) { +function analyzeRow(rowIndex, cells, headerMap, headerLabels, userBills, categories, sheetName, sheetYear, sheetMonth, defaultYear, defaultMonth, rowIdPrefix, defaultDueDay = null, headerSetIndex = null) { const get = (field) => { const idx = headerMap[field]; return idx !== undefined ? cells[idx] : undefined; @@ -721,6 +828,7 @@ function analyzeRow(rowIndex, cells, headerMap, headerLabels, userBills, categor warnings, errors, paymentDateIso: detectedPaidDate, + defaultDueDay, }); const proposedAction = recommendation.action === 'ambiguous' ? 'mark_ambiguous' : recommendation.action; @@ -751,6 +859,7 @@ function analyzeRow(rowIndex, cells, headerMap, headerLabels, userBills, categor errors, possible_bill_matches: possibleMatches, requires_user_decision: requiresUserDecision, + due_day: recommendation.due_day, recommendation, }; } @@ -764,29 +873,79 @@ function analyzeRow(rowIndex, cells, headerMap, headerLabels, userBills, categor function parseSheetRows({ name, rawRows, year: sheetYear, month: sheetMonth, rowIdPrefix }, userBills, categories, defaultYear, defaultMonth) { if (!rawRows.length) return { rows: [], headerRow: null }; - const firstRow = rawRows[0] || []; - const headerMap = detectHeaders(firstRow); - const headerLabels = firstRow.map((c) => (c != null ? String(c).trim() : null)); - const hasHeaders = Object.keys(headerMap).length > 0; - const startRow = hasHeaders ? 1 : 0; + // Detect all header sets in each row to handle dual-column layouts + let headerRowIndex = 0; + let headerLabels = rawRows[0]?.map((c) => (c != null ? String(c).trim() : null)) || []; + + // First try to detect headers in row 0 + let allHeaderSets = detectAllHeaderSets(rawRows[0]); + + // If no headers in row 0, scan up to 5 rows + for (let scanIdx = 1; scanIdx < Math.min(5, rawRows.length); scanIdx++) { + const candidateSets = detectAllHeaderSets(rawRows[scanIdx]); + if (candidateSets.length > 0) { + headerRowIndex = scanIdx; + headerLabels = rawRows[scanIdx].map((c) => (c != null ? String(c).trim() : null)); + allHeaderSets = candidateSets; + // Check if this set has all required fields + let hasAllRequired = false; + for (const set of allHeaderSets) { + if (set.map.due_date !== undefined && set.map.bill_name !== undefined && set.map.amount !== undefined) { + hasAllRequired = true; + break; + } + } + if (hasAllRequired) { + break; + } + } + } + + // Check if we have valid headers (must have due_date, bill_name, amount) + let hasValidHeaders = false; + for (const set of allHeaderSets) { + if (set.map.due_date !== undefined && set.map.bill_name !== undefined && set.map.amount !== undefined) { + hasValidHeaders = true; + break; + } + } + + const hasHeaders = hasValidHeaders; + const startRow = hasHeaders ? headerRowIndex + 1 : 0; const rows = []; - for (let i = startRow; i < rawRows.length; i++) { - const cells = rawRows[i] || []; - if (isBlankRow(cells)) continue; - if (isLikelyHeaderRow(cells) && i > 0) continue; - if (isLikelyTotalRow(cells)) continue; - - rows.push(analyzeRow( - i, cells, headerMap, headerLabels, userBills, categories, - name, sheetYear, sheetMonth, - defaultYear, defaultMonth, rowIdPrefix, - )); + + // Process each header set independently + for (let setIdx = 0; setIdx < allHeaderSets.length; setIdx++) { + const headerSet = allHeaderSets[setIdx]; + const headerMap = headerSet.map; + const defaultDueDay = headerSet.defaultDueDay; + + for (let i = startRow; i < rawRows.length; i++) { + const cells = rawRows[i] || []; + + // For dual-column: skip rows blank in this header set's columns only + // For single-column: fall back to regular isBlankRow + if (allHeaderSets.length > 1 ? isBlankRowForHeaderSet(cells, headerSet) : isBlankRow(cells)) continue; + + // Skip duplicate header rows (but only if we found headers) + if (hasHeaders && isLikelyHeaderRow(cells) && i > headerRowIndex) continue; + + // Skip total rows + if (isLikelyTotalRow(cells)) continue; + + rows.push(analyzeRow( + i, cells, headerMap, headerLabels, userBills, categories, + name, sheetYear, sheetMonth, + defaultYear, defaultMonth, rowIdPrefix, + defaultDueDay, setIdx, + )); + } } return { rows, - headerRow: hasHeaders ? firstRow.map((c) => (c != null ? String(c) : null)) : null, + headerRow: hasHeaders ? headerLabels : null, }; } @@ -1439,6 +1598,7 @@ function getImportHistory(userId) { } module.exports = { + detectAllHeaderSets, previewSpreadsheet, applyImportDecisions, getImportHistory,