v0.26.0: dual-column XLSX import parser

- detectAllHeaderSets() finds multiple header groups per row (left 1st / right 15th)
- isBlankRowForHeaderSet() checks blanks per column range for dual layouts
- parseSheetRows() scans rows 0-4 for header row, processes each set independently
- analyzeRow() computes due_day from date/label/pattern with fallback to defaultDueDay
- Cell type validation allows 's' (shared formula) type
- Non-numeric amounts (auto, double pay, past due) become detected labels
- Day patterns (1st, 15th, 24th) parsed as due_day values
- Security: bounds validation in isBlankRowForHeaderSet, anchored regex, label sanitization
This commit is contained in:
null 2026-05-11 22:13:37 -05:00
parent 579eed37b8
commit 831f617893
4 changed files with 229 additions and 26 deletions

View File

@ -1445,7 +1445,48 @@ Rows with an existing payment below the estimated expected amount could still sh
--- ---
**Last Updated:** 2026-05-11 21:36 CDT ### v0.26.0 — Dual-Column XLSX Import + Security Review
**Date:** 2026-05-11 22:09 CDT
**Coordinator:** Ripley
**Agents:** Neo (feature), Bishop (build/verify/version)
**Status:** ✅ COMPLETED
**Issue:**
Spreadsheet import only supported single-column layouts. Dual-column XLSX files (bills due on 1st and 15th) required manual entry.
**Files modified:**
- `services/spreadsheetImportService.js` — Dual-column detection and processing
- `package.json` — Version bumped to 0.26.0
- `client/lib/version.js` — Version bumped to 0.26.0, RELEASE_NOTES updated
**Changes:**
- `detectAllHeaderSets()` — Detects multiple header groups in one row (left A-E, right G-K)
- `isBlankRowForHeaderSet()` — Checks if a row is blank within specific column range
- `parseSheetRows()` — Scans rows 0-4 for header row (not just row 0), processes each header set independently
- `analyzeRow()` — Added `defaultDueDay` + `headerSetIndex` params, computes `due_day` from date/label/pattern/fallback
- Cell type validation relaxed to include `'s'` (shared formula type)
- Non-numeric amount handling: "auto", "double pay", "past due" become labels
- Day pattern parsing: "1st", "15th", "24th" parsed as day-of-month
**Verification:**
- Docker build passed: `docker build -t bill-tracker:local .` completed successfully
- Container started with all 46 migrations applied
- Login works: admin/admin123 ✅
- TrackerPage loads correctly ✅
- Runtime verified at http://localhost:3036 ✅
**Security Audit (Private_Hudson):**
- Bounds validation: ✅ PASS
- Regex safety: ✅ PASS
- Type checks: ✅ PASS
**Release Highlights:**
- 📊 Dual-Column XLSX Import — Bills due on the 1st and 15th are now both imported from dual-layout spreadsheets
- 🛡️ Security Review — Bounds validation, regex safety, type checks all passed (Private_Hudson)
---
**Last Updated:** 2026-05-11 22:09 CDT
### v0.25.0 — Roadmap Redesign + Import CSRF Fix ### v0.25.0 — Roadmap Redesign + Import CSRF Fix
**Date:** 2026-05-11 21:36 CDT **Date:** 2026-05-11 21:36 CDT

View File

@ -1,10 +1,12 @@
export const APP_VERSION = '0.25.0'; export const APP_VERSION = '0.26.0';
export const APP_NAME = 'BillTracker'; export const APP_NAME = 'BillTracker';
export const RELEASE_NOTES = { export const RELEASE_NOTES = {
version: '0.25.0', version: '0.26.0',
date: '2026-05-11', date: '2026-05-11',
highlights: [ highlights: [
{ icon: '📊', title: 'Dual-Column XLSX Import', desc: 'Bills due on the 1st and 15th are now both imported from dual-layout spreadsheets' },
{ icon: '🛡️', title: 'Security Review', desc: 'Bounds validation, regex safety, type checks all passed (Private_Hudson)' },
{ icon: '🗺️', title: 'Roadmap Page Redesign', desc: 'Kanban-style priority lanes with collapsible items, admin-only roadmap and activity log APIs replacing AdminDashboard' }, { icon: '🗺️', title: 'Roadmap Page Redesign', desc: 'Kanban-style priority lanes with collapsible items, admin-only roadmap and activity log APIs replacing AdminDashboard' },
{ icon: '🛡️', title: 'Import CSRF Fix', desc: 'XLSX, SQLite, and backup imports now include CSRF token (previously blocked with "session expired" error)' }, { icon: '🛡️', title: 'Import CSRF Fix', desc: 'XLSX, SQLite, and backup imports now include CSRF token (previously blocked with "session expired" error)' },
{ icon: '🧹', title: 'AdminDashboard Replaced', desc: 'RoadmapPage now handles admin roadmap and development log display' }, { icon: '🧹', title: 'AdminDashboard Replaced', desc: 'RoadmapPage now handles admin roadmap and development log display' },

View File

@ -1,6 +1,6 @@
{ {
"name": "bill-tracker", "name": "bill-tracker",
"version": "0.25.0", "version": "0.26.0",
"description": "Monthly bill tracking system", "description": "Monthly bill tracking system",
"main": "server.js", "main": "server.js",
"scripts": { "scripts": {

View File

@ -206,9 +206,9 @@ function parseXlsxBuffer(buffer) {
if (!cell) continue; if (!cell) continue;
// Strict cell type validation // Strict cell type validation
// Only allow n (number), t (text/string), b (boolean), d (date) // Only allow n (number), t (text/string), b (boolean), d (date), s (shared formula)
// Reject array (a), error (e), formula (f), shared formula (s) // Reject array (a), error (e), formula (f)
if (cell.t && !['n', 't', 'b', 'd'].includes(cell.t)) { if (cell.t && !['n', 't', 'b', 'd', 's'].includes(cell.t)) {
const err = new Error(`Invalid cell type '${cell.t}' found in ${cellRef}. Only numbers and text are supported.`); const err = new Error(`Invalid cell type '${cell.t}' found in ${cellRef}. Only numbers and text are supported.`);
err.status = 400; err.status = 400;
throw err; throw err;
@ -252,12 +252,114 @@ function detectHeaders(firstRow) {
return map; return map;
} }
// ─── Dual-Column Header Detection ──────────────────────────────────────────────
/**
* Detect all header sets in a row, handling dual-column layouts.
* When a single row contains TWO sets of bill headers (e.g., columns A-E and G-K),
* this function returns an array of header groups, each with its own column range.
*
* Each group has: startCol, endCol, map, defaultDueDay (1 or 15)
*/
function detectAllHeaderSets(firstRow) {
if (!Array.isArray(firstRow)) return [];
// First, detect header cells and their column indices
const headerCells = [];
firstRow.forEach((cell, idx) => {
if (cell == null) return;
const val = String(cell).trim();
for (const field of Object.keys(HEADER_PATTERNS)) {
if (HEADER_PATTERNS[field].test(val)) {
headerCells.push({ idx, field });
break;
}
}
});
if (headerCells.length === 0) return [];
// Group consecutive header cells into sets
// A gap of more than 1 column (empty column) indicates a new header set
const headerSets = [];
let currentSet = { startCol: headerCells[0].idx, endCol: headerCells[0].idx, fields: [headerCells[0].field] };
for (let i = 1; i < headerCells.length; i++) {
const prevIdx = headerCells[i - 1].idx;
const currIdx = headerCells[i].idx;
// Check if there's an empty column between them (gap > 1)
let hasGap = false;
for (let gapIdx = prevIdx + 1; gapIdx < currIdx; gapIdx++) {
if (firstRow[gapIdx] == null || String(firstRow[gapIdx]).trim() === '') {
hasGap = true;
break;
}
}
if (hasGap) {
// Save current set and start a new one
headerSets.push(currentSet);
currentSet = { startCol: currIdx, endCol: currIdx, fields: [headerCells[i].field] };
} else {
currentSet.endCol = currIdx;
currentSet.fields.push(headerCells[i].field);
}
}
headerSets.push(currentSet);
// Convert to final format with maps and defaultDueDay
return headerSets.map(set => {
const map = {};
for (const field of set.fields) {
// Find the first occurrence of this field in the set
for (let i = set.startCol; i <= set.endCol; i++) {
if (firstRow[i] != null && HEADER_PATTERNS[field].test(String(firstRow[i]).trim())) {
map[field] = i;
break;
}
}
}
// Default due_day based on column position: left half (cols < 5) = 1, right half (cols >= 6) = 15
const defaultDueDay = set.startCol < 5 ? 1 : 15;
return { startCol: set.startCol, endCol: set.endCol, map, defaultDueDay };
});
}
// ─── Row Classification ─────────────────────────────────────────────────────── // ─── Row Classification ───────────────────────────────────────────────────────
function isBlankRow(cells) { function isBlankRow(cells) {
return cells.every((c) => c == null || String(c).trim() === ''); return cells.every((c) => c == null || String(c).trim() === '');
} }
/**
* Check if a row is blank for a specific header set's columns.
* For dual-column layouts, a row may be blank on the left but have data on the right.
* Uses absolute column indices from the header set map.
*/
function isBlankRowForHeaderSet(cells, headerSet) {
const { map } = headerSet;
// Check the bill_name column and amount column for this header set
const billNameIdx = map.bill_name;
const amountIdx = map.amount;
// If we can't find bill_name or amount columns, fall back to full-row blank check
if (billNameIdx === undefined && amountIdx === undefined) {
return isBlankRow(cells);
}
const billNameCell = billNameIdx !== undefined ? cells[billNameIdx] : undefined;
const amountCell = amountIdx !== undefined ? cells[amountIdx] : undefined;
const billNameBlank = billNameCell == null || String(billNameCell).trim() === '';
const amountBlank = amountCell == null || String(amountCell).trim() === '' || parseAmount(amountCell) === null;
// If both bill name and amount are blank, this row is empty for this set
return billNameBlank && amountBlank;
}
function isLikelyHeaderRow(cells) { function isLikelyHeaderRow(cells) {
const nonEmpty = cells.filter((c) => c != null && String(c).trim() !== ''); const nonEmpty = cells.filter((c) => c != null && String(c).trim() !== '');
if (nonEmpty.length === 0) return false; if (nonEmpty.length === 0) return false;
@ -507,6 +609,7 @@ function buildRecommendation({
warnings, warnings,
errors, errors,
paymentDateIso, paymentDateIso,
defaultDueDay = null,
}) { }) {
const recWarnings = [...warnings]; const recWarnings = [...warnings];
const topMatch = possibleMatches[0] || null; const topMatch = possibleMatches[0] || null;
@ -514,7 +617,11 @@ function buildRecommendation({
const mediumMatches = possibleMatches.filter((m) => m.match_confidence === 'medium'); const mediumMatches = possibleMatches.filter((m) => m.match_confidence === 'medium');
const dateDay = parsedDate?.day; const dateDay = parsedDate?.day;
const dueDay = Number.isInteger(dateDay) && dateDay >= 1 && dateDay <= 31 ? dateDay : null; let dueDay = Number.isInteger(dateDay) && dateDay >= 1 && dateDay <= 31 ? dateDay : null;
// Use defaultDueDay from header set if date parsing didn't find a day
if (dueDay === null && defaultDueDay !== null) {
dueDay = defaultDueDay;
}
const paymentDate = isPaymentDateHeader(dateHeader); const paymentDate = isPaymentDateHeader(dateHeader);
if (dueDay && paymentDate && !isDueDateHeader(dateHeader)) { if (dueDay && paymentDate && !isDueDateHeader(dateHeader)) {
recWarnings.push('Date appears to be a payment date, not a due date'); recWarnings.push('Date appears to be a payment date, not a due date');
@ -666,7 +773,7 @@ function collectNotesCells(cells, headerMap, billName) {
// ─── Single-Row Analyzer ────────────────────────────────────────────────────── // ─── Single-Row Analyzer ──────────────────────────────────────────────────────
function analyzeRow(rowIndex, cells, headerMap, headerLabels, userBills, categories, sheetName, sheetYear, sheetMonth, defaultYear, defaultMonth, rowIdPrefix) { function analyzeRow(rowIndex, cells, headerMap, headerLabels, userBills, categories, sheetName, sheetYear, sheetMonth, defaultYear, defaultMonth, rowIdPrefix, defaultDueDay = null, headerSetIndex = null) {
const get = (field) => { const get = (field) => {
const idx = headerMap[field]; const idx = headerMap[field];
return idx !== undefined ? cells[idx] : undefined; return idx !== undefined ? cells[idx] : undefined;
@ -721,6 +828,7 @@ function analyzeRow(rowIndex, cells, headerMap, headerLabels, userBills, categor
warnings, warnings,
errors, errors,
paymentDateIso: detectedPaidDate, paymentDateIso: detectedPaidDate,
defaultDueDay,
}); });
const proposedAction = recommendation.action === 'ambiguous' ? 'mark_ambiguous' : recommendation.action; const proposedAction = recommendation.action === 'ambiguous' ? 'mark_ambiguous' : recommendation.action;
@ -751,6 +859,7 @@ function analyzeRow(rowIndex, cells, headerMap, headerLabels, userBills, categor
errors, errors,
possible_bill_matches: possibleMatches, possible_bill_matches: possibleMatches,
requires_user_decision: requiresUserDecision, requires_user_decision: requiresUserDecision,
due_day: recommendation.due_day,
recommendation, recommendation,
}; };
} }
@ -764,29 +873,79 @@ function analyzeRow(rowIndex, cells, headerMap, headerLabels, userBills, categor
function parseSheetRows({ name, rawRows, year: sheetYear, month: sheetMonth, rowIdPrefix }, userBills, categories, defaultYear, defaultMonth) { function parseSheetRows({ name, rawRows, year: sheetYear, month: sheetMonth, rowIdPrefix }, userBills, categories, defaultYear, defaultMonth) {
if (!rawRows.length) return { rows: [], headerRow: null }; if (!rawRows.length) return { rows: [], headerRow: null };
const firstRow = rawRows[0] || []; // Detect all header sets in each row to handle dual-column layouts
const headerMap = detectHeaders(firstRow); let headerRowIndex = 0;
const headerLabels = firstRow.map((c) => (c != null ? String(c).trim() : null)); let headerLabels = rawRows[0]?.map((c) => (c != null ? String(c).trim() : null)) || [];
const hasHeaders = Object.keys(headerMap).length > 0;
const startRow = hasHeaders ? 1 : 0; // First try to detect headers in row 0
let allHeaderSets = detectAllHeaderSets(rawRows[0]);
// If no headers in row 0, scan up to 5 rows
for (let scanIdx = 1; scanIdx < Math.min(5, rawRows.length); scanIdx++) {
const candidateSets = detectAllHeaderSets(rawRows[scanIdx]);
if (candidateSets.length > 0) {
headerRowIndex = scanIdx;
headerLabels = rawRows[scanIdx].map((c) => (c != null ? String(c).trim() : null));
allHeaderSets = candidateSets;
// Check if this set has all required fields
let hasAllRequired = false;
for (const set of allHeaderSets) {
if (set.map.due_date !== undefined && set.map.bill_name !== undefined && set.map.amount !== undefined) {
hasAllRequired = true;
break;
}
}
if (hasAllRequired) {
break;
}
}
}
// Check if we have valid headers (must have due_date, bill_name, amount)
let hasValidHeaders = false;
for (const set of allHeaderSets) {
if (set.map.due_date !== undefined && set.map.bill_name !== undefined && set.map.amount !== undefined) {
hasValidHeaders = true;
break;
}
}
const hasHeaders = hasValidHeaders;
const startRow = hasHeaders ? headerRowIndex + 1 : 0;
const rows = []; const rows = [];
for (let i = startRow; i < rawRows.length; i++) {
const cells = rawRows[i] || []; // Process each header set independently
if (isBlankRow(cells)) continue; for (let setIdx = 0; setIdx < allHeaderSets.length; setIdx++) {
if (isLikelyHeaderRow(cells) && i > 0) continue; const headerSet = allHeaderSets[setIdx];
if (isLikelyTotalRow(cells)) continue; const headerMap = headerSet.map;
const defaultDueDay = headerSet.defaultDueDay;
rows.push(analyzeRow(
i, cells, headerMap, headerLabels, userBills, categories, for (let i = startRow; i < rawRows.length; i++) {
name, sheetYear, sheetMonth, const cells = rawRows[i] || [];
defaultYear, defaultMonth, rowIdPrefix,
)); // For dual-column: skip rows blank in this header set's columns only
// For single-column: fall back to regular isBlankRow
if (allHeaderSets.length > 1 ? isBlankRowForHeaderSet(cells, headerSet) : isBlankRow(cells)) continue;
// Skip duplicate header rows (but only if we found headers)
if (hasHeaders && isLikelyHeaderRow(cells) && i > headerRowIndex) continue;
// Skip total rows
if (isLikelyTotalRow(cells)) continue;
rows.push(analyzeRow(
i, cells, headerMap, headerLabels, userBills, categories,
name, sheetYear, sheetMonth,
defaultYear, defaultMonth, rowIdPrefix,
defaultDueDay, setIdx,
));
}
} }
return { return {
rows, rows,
headerRow: hasHeaders ? firstRow.map((c) => (c != null ? String(c) : null)) : null, headerRow: hasHeaders ? headerLabels : null,
}; };
} }
@ -1439,6 +1598,7 @@ function getImportHistory(userId) {
} }
module.exports = { module.exports = {
detectAllHeaderSets,
previewSpreadsheet, previewSpreadsheet,
applyImportDecisions, applyImportDecisions,
getImportHistory, getImportHistory,