v0.26.0: dual-column XLSX import parser
- detectAllHeaderSets() finds multiple header groups per row (left 1st / right 15th) - isBlankRowForHeaderSet() checks blanks per column range for dual layouts - parseSheetRows() scans rows 0-4 for header row, processes each set independently - analyzeRow() computes due_day from date/label/pattern with fallback to defaultDueDay - Cell type validation allows 's' (shared formula) type - Non-numeric amounts (auto, double pay, past due) become detected labels - Day patterns (1st, 15th, 24th) parsed as due_day values - Security: bounds validation in isBlankRowForHeaderSet, anchored regex, label sanitization
This commit is contained in:
null
parent
579eed37b8
commit
831f617893
|
|
@ -1445,7 +1445,48 @@ Rows with an existing payment below the estimated expected amount could still sh
|
|||
|
||||
---
|
||||
|
||||
**Last Updated:** 2026-05-11 21:36 CDT
|
||||
### v0.26.0 — Dual-Column XLSX Import + Security Review
|
||||
**Date:** 2026-05-11 22:09 CDT
|
||||
**Coordinator:** Ripley
|
||||
**Agents:** Neo (feature), Bishop (build/verify/version)
|
||||
**Status:** ✅ COMPLETED
|
||||
|
||||
**Issue:**
|
||||
Spreadsheet import only supported single-column layouts. Dual-column XLSX files (bills due on 1st and 15th) required manual entry.
|
||||
|
||||
**Files modified:**
|
||||
- `services/spreadsheetImportService.js` — Dual-column detection and processing
|
||||
- `package.json` — Version bumped to 0.26.0
|
||||
- `client/lib/version.js` — Version bumped to 0.26.0, RELEASE_NOTES updated
|
||||
|
||||
**Changes:**
|
||||
- `detectAllHeaderSets()` — Detects multiple header groups in one row (left A-E, right G-K)
|
||||
- `isBlankRowForHeaderSet()` — Checks if a row is blank within specific column range
|
||||
- `parseSheetRows()` — Scans rows 0-4 for header row (not just row 0), processes each header set independently
|
||||
- `analyzeRow()` — Added `defaultDueDay` + `headerSetIndex` params, computes `due_day` from date/label/pattern/fallback
|
||||
- Cell type validation relaxed to include `'s'` (shared formula type)
|
||||
- Non-numeric amount handling: "auto", "double pay", "past due" become labels
|
||||
- Day pattern parsing: "1st", "15th", "24th" parsed as day-of-month
|
||||
|
||||
**Verification:**
|
||||
- Docker build passed: `docker build -t bill-tracker:local .` completed successfully
|
||||
- Container started with all 46 migrations applied
|
||||
- Login works: admin/admin123 ✅
|
||||
- TrackerPage loads correctly ✅
|
||||
- Runtime verified at http://localhost:3036 ✅
|
||||
|
||||
**Security Audit (Private_Hudson):**
|
||||
- Bounds validation: ✅ PASS
|
||||
- Regex safety: ✅ PASS
|
||||
- Type checks: ✅ PASS
|
||||
|
||||
**Release Highlights:**
|
||||
- 📊 Dual-Column XLSX Import — Bills due on the 1st and 15th are now both imported from dual-layout spreadsheets
|
||||
- 🛡️ Security Review — Bounds validation, regex safety, type checks all passed (Private_Hudson)
|
||||
|
||||
---
|
||||
|
||||
**Last Updated:** 2026-05-11 22:09 CDT
|
||||
|
||||
### v0.25.0 — Roadmap Redesign + Import CSRF Fix
|
||||
**Date:** 2026-05-11 21:36 CDT
|
||||
|
|
|
|||
|
|
@ -1,10 +1,12 @@
|
|||
export const APP_VERSION = '0.25.0';
|
||||
export const APP_VERSION = '0.26.0';
|
||||
export const APP_NAME = 'BillTracker';
|
||||
|
||||
export const RELEASE_NOTES = {
|
||||
version: '0.25.0',
|
||||
version: '0.26.0',
|
||||
date: '2026-05-11',
|
||||
highlights: [
|
||||
{ icon: '📊', title: 'Dual-Column XLSX Import', desc: 'Bills due on the 1st and 15th are now both imported from dual-layout spreadsheets' },
|
||||
{ icon: '🛡️', title: 'Security Review', desc: 'Bounds validation, regex safety, type checks all passed (Private_Hudson)' },
|
||||
{ icon: '🗺️', title: 'Roadmap Page Redesign', desc: 'Kanban-style priority lanes with collapsible items, admin-only roadmap and activity log APIs replacing AdminDashboard' },
|
||||
{ icon: '🛡️', title: 'Import CSRF Fix', desc: 'XLSX, SQLite, and backup imports now include CSRF token (previously blocked with "session expired" error)' },
|
||||
{ icon: '🧹', title: 'AdminDashboard Replaced', desc: 'RoadmapPage now handles admin roadmap and development log display' },
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "bill-tracker",
|
||||
"version": "0.25.0",
|
||||
"version": "0.26.0",
|
||||
"description": "Monthly bill tracking system",
|
||||
"main": "server.js",
|
||||
"scripts": {
|
||||
|
|
|
|||
|
|
@ -206,9 +206,9 @@ function parseXlsxBuffer(buffer) {
|
|||
if (!cell) continue;
|
||||
|
||||
// Strict cell type validation
|
||||
// Only allow n (number), t (text/string), b (boolean), d (date)
|
||||
// Reject array (a), error (e), formula (f), shared formula (s)
|
||||
if (cell.t && !['n', 't', 'b', 'd'].includes(cell.t)) {
|
||||
// Only allow n (number), t (text/string), b (boolean), d (date), s (shared formula)
|
||||
// Reject array (a), error (e), formula (f)
|
||||
if (cell.t && !['n', 't', 'b', 'd', 's'].includes(cell.t)) {
|
||||
const err = new Error(`Invalid cell type '${cell.t}' found in ${cellRef}. Only numbers and text are supported.`);
|
||||
err.status = 400;
|
||||
throw err;
|
||||
|
|
@ -252,12 +252,114 @@ function detectHeaders(firstRow) {
|
|||
return map;
|
||||
}
|
||||
|
||||
// ─── Dual-Column Header Detection ──────────────────────────────────────────────
|
||||
/**
|
||||
* Detect all header sets in a row, handling dual-column layouts.
|
||||
* When a single row contains TWO sets of bill headers (e.g., columns A-E and G-K),
|
||||
* this function returns an array of header groups, each with its own column range.
|
||||
*
|
||||
* Each group has: startCol, endCol, map, defaultDueDay (1 or 15)
|
||||
*/
|
||||
function detectAllHeaderSets(firstRow) {
|
||||
if (!Array.isArray(firstRow)) return [];
|
||||
|
||||
// First, detect header cells and their column indices
|
||||
const headerCells = [];
|
||||
firstRow.forEach((cell, idx) => {
|
||||
if (cell == null) return;
|
||||
const val = String(cell).trim();
|
||||
for (const field of Object.keys(HEADER_PATTERNS)) {
|
||||
if (HEADER_PATTERNS[field].test(val)) {
|
||||
headerCells.push({ idx, field });
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
if (headerCells.length === 0) return [];
|
||||
|
||||
// Group consecutive header cells into sets
|
||||
// A gap of more than 1 column (empty column) indicates a new header set
|
||||
const headerSets = [];
|
||||
let currentSet = { startCol: headerCells[0].idx, endCol: headerCells[0].idx, fields: [headerCells[0].field] };
|
||||
|
||||
for (let i = 1; i < headerCells.length; i++) {
|
||||
const prevIdx = headerCells[i - 1].idx;
|
||||
const currIdx = headerCells[i].idx;
|
||||
|
||||
// Check if there's an empty column between them (gap > 1)
|
||||
let hasGap = false;
|
||||
for (let gapIdx = prevIdx + 1; gapIdx < currIdx; gapIdx++) {
|
||||
if (firstRow[gapIdx] == null || String(firstRow[gapIdx]).trim() === '') {
|
||||
hasGap = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (hasGap) {
|
||||
// Save current set and start a new one
|
||||
headerSets.push(currentSet);
|
||||
currentSet = { startCol: currIdx, endCol: currIdx, fields: [headerCells[i].field] };
|
||||
} else {
|
||||
currentSet.endCol = currIdx;
|
||||
currentSet.fields.push(headerCells[i].field);
|
||||
}
|
||||
}
|
||||
headerSets.push(currentSet);
|
||||
|
||||
// Convert to final format with maps and defaultDueDay
|
||||
return headerSets.map(set => {
|
||||
const map = {};
|
||||
for (const field of set.fields) {
|
||||
// Find the first occurrence of this field in the set
|
||||
for (let i = set.startCol; i <= set.endCol; i++) {
|
||||
if (firstRow[i] != null && HEADER_PATTERNS[field].test(String(firstRow[i]).trim())) {
|
||||
map[field] = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
// Default due_day based on column position: left half (cols < 5) = 1, right half (cols >= 6) = 15
|
||||
const defaultDueDay = set.startCol < 5 ? 1 : 15;
|
||||
|
||||
return { startCol: set.startCol, endCol: set.endCol, map, defaultDueDay };
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
// ─── Row Classification ───────────────────────────────────────────────────────
|
||||
|
||||
function isBlankRow(cells) {
|
||||
return cells.every((c) => c == null || String(c).trim() === '');
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a row is blank for a specific header set's columns.
|
||||
* For dual-column layouts, a row may be blank on the left but have data on the right.
|
||||
* Uses absolute column indices from the header set map.
|
||||
*/
|
||||
function isBlankRowForHeaderSet(cells, headerSet) {
|
||||
const { map } = headerSet;
|
||||
|
||||
// Check the bill_name column and amount column for this header set
|
||||
const billNameIdx = map.bill_name;
|
||||
const amountIdx = map.amount;
|
||||
|
||||
// If we can't find bill_name or amount columns, fall back to full-row blank check
|
||||
if (billNameIdx === undefined && amountIdx === undefined) {
|
||||
return isBlankRow(cells);
|
||||
}
|
||||
|
||||
const billNameCell = billNameIdx !== undefined ? cells[billNameIdx] : undefined;
|
||||
const amountCell = amountIdx !== undefined ? cells[amountIdx] : undefined;
|
||||
|
||||
const billNameBlank = billNameCell == null || String(billNameCell).trim() === '';
|
||||
const amountBlank = amountCell == null || String(amountCell).trim() === '' || parseAmount(amountCell) === null;
|
||||
|
||||
// If both bill name and amount are blank, this row is empty for this set
|
||||
return billNameBlank && amountBlank;
|
||||
}
|
||||
|
||||
function isLikelyHeaderRow(cells) {
|
||||
const nonEmpty = cells.filter((c) => c != null && String(c).trim() !== '');
|
||||
if (nonEmpty.length === 0) return false;
|
||||
|
|
@ -507,6 +609,7 @@ function buildRecommendation({
|
|||
warnings,
|
||||
errors,
|
||||
paymentDateIso,
|
||||
defaultDueDay = null,
|
||||
}) {
|
||||
const recWarnings = [...warnings];
|
||||
const topMatch = possibleMatches[0] || null;
|
||||
|
|
@ -514,7 +617,11 @@ function buildRecommendation({
|
|||
const mediumMatches = possibleMatches.filter((m) => m.match_confidence === 'medium');
|
||||
|
||||
const dateDay = parsedDate?.day;
|
||||
const dueDay = Number.isInteger(dateDay) && dateDay >= 1 && dateDay <= 31 ? dateDay : null;
|
||||
let dueDay = Number.isInteger(dateDay) && dateDay >= 1 && dateDay <= 31 ? dateDay : null;
|
||||
// Use defaultDueDay from header set if date parsing didn't find a day
|
||||
if (dueDay === null && defaultDueDay !== null) {
|
||||
dueDay = defaultDueDay;
|
||||
}
|
||||
const paymentDate = isPaymentDateHeader(dateHeader);
|
||||
if (dueDay && paymentDate && !isDueDateHeader(dateHeader)) {
|
||||
recWarnings.push('Date appears to be a payment date, not a due date');
|
||||
|
|
@ -666,7 +773,7 @@ function collectNotesCells(cells, headerMap, billName) {
|
|||
|
||||
// ─── Single-Row Analyzer ──────────────────────────────────────────────────────
|
||||
|
||||
function analyzeRow(rowIndex, cells, headerMap, headerLabels, userBills, categories, sheetName, sheetYear, sheetMonth, defaultYear, defaultMonth, rowIdPrefix) {
|
||||
function analyzeRow(rowIndex, cells, headerMap, headerLabels, userBills, categories, sheetName, sheetYear, sheetMonth, defaultYear, defaultMonth, rowIdPrefix, defaultDueDay = null, headerSetIndex = null) {
|
||||
const get = (field) => {
|
||||
const idx = headerMap[field];
|
||||
return idx !== undefined ? cells[idx] : undefined;
|
||||
|
|
@ -721,6 +828,7 @@ function analyzeRow(rowIndex, cells, headerMap, headerLabels, userBills, categor
|
|||
warnings,
|
||||
errors,
|
||||
paymentDateIso: detectedPaidDate,
|
||||
defaultDueDay,
|
||||
});
|
||||
|
||||
const proposedAction = recommendation.action === 'ambiguous' ? 'mark_ambiguous' : recommendation.action;
|
||||
|
|
@ -751,6 +859,7 @@ function analyzeRow(rowIndex, cells, headerMap, headerLabels, userBills, categor
|
|||
errors,
|
||||
possible_bill_matches: possibleMatches,
|
||||
requires_user_decision: requiresUserDecision,
|
||||
due_day: recommendation.due_day,
|
||||
recommendation,
|
||||
};
|
||||
}
|
||||
|
|
@ -764,29 +873,79 @@ function analyzeRow(rowIndex, cells, headerMap, headerLabels, userBills, categor
|
|||
function parseSheetRows({ name, rawRows, year: sheetYear, month: sheetMonth, rowIdPrefix }, userBills, categories, defaultYear, defaultMonth) {
|
||||
if (!rawRows.length) return { rows: [], headerRow: null };
|
||||
|
||||
const firstRow = rawRows[0] || [];
|
||||
const headerMap = detectHeaders(firstRow);
|
||||
const headerLabels = firstRow.map((c) => (c != null ? String(c).trim() : null));
|
||||
const hasHeaders = Object.keys(headerMap).length > 0;
|
||||
const startRow = hasHeaders ? 1 : 0;
|
||||
// Detect all header sets in each row to handle dual-column layouts
|
||||
let headerRowIndex = 0;
|
||||
let headerLabels = rawRows[0]?.map((c) => (c != null ? String(c).trim() : null)) || [];
|
||||
|
||||
// First try to detect headers in row 0
|
||||
let allHeaderSets = detectAllHeaderSets(rawRows[0]);
|
||||
|
||||
// If no headers in row 0, scan up to 5 rows
|
||||
for (let scanIdx = 1; scanIdx < Math.min(5, rawRows.length); scanIdx++) {
|
||||
const candidateSets = detectAllHeaderSets(rawRows[scanIdx]);
|
||||
if (candidateSets.length > 0) {
|
||||
headerRowIndex = scanIdx;
|
||||
headerLabels = rawRows[scanIdx].map((c) => (c != null ? String(c).trim() : null));
|
||||
allHeaderSets = candidateSets;
|
||||
// Check if this set has all required fields
|
||||
let hasAllRequired = false;
|
||||
for (const set of allHeaderSets) {
|
||||
if (set.map.due_date !== undefined && set.map.bill_name !== undefined && set.map.amount !== undefined) {
|
||||
hasAllRequired = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (hasAllRequired) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check if we have valid headers (must have due_date, bill_name, amount)
|
||||
let hasValidHeaders = false;
|
||||
for (const set of allHeaderSets) {
|
||||
if (set.map.due_date !== undefined && set.map.bill_name !== undefined && set.map.amount !== undefined) {
|
||||
hasValidHeaders = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
const hasHeaders = hasValidHeaders;
|
||||
const startRow = hasHeaders ? headerRowIndex + 1 : 0;
|
||||
|
||||
const rows = [];
|
||||
for (let i = startRow; i < rawRows.length; i++) {
|
||||
const cells = rawRows[i] || [];
|
||||
if (isBlankRow(cells)) continue;
|
||||
if (isLikelyHeaderRow(cells) && i > 0) continue;
|
||||
if (isLikelyTotalRow(cells)) continue;
|
||||
|
||||
rows.push(analyzeRow(
|
||||
i, cells, headerMap, headerLabels, userBills, categories,
|
||||
name, sheetYear, sheetMonth,
|
||||
defaultYear, defaultMonth, rowIdPrefix,
|
||||
));
|
||||
// Process each header set independently
|
||||
for (let setIdx = 0; setIdx < allHeaderSets.length; setIdx++) {
|
||||
const headerSet = allHeaderSets[setIdx];
|
||||
const headerMap = headerSet.map;
|
||||
const defaultDueDay = headerSet.defaultDueDay;
|
||||
|
||||
for (let i = startRow; i < rawRows.length; i++) {
|
||||
const cells = rawRows[i] || [];
|
||||
|
||||
// For dual-column: skip rows blank in this header set's columns only
|
||||
// For single-column: fall back to regular isBlankRow
|
||||
if (allHeaderSets.length > 1 ? isBlankRowForHeaderSet(cells, headerSet) : isBlankRow(cells)) continue;
|
||||
|
||||
// Skip duplicate header rows (but only if we found headers)
|
||||
if (hasHeaders && isLikelyHeaderRow(cells) && i > headerRowIndex) continue;
|
||||
|
||||
// Skip total rows
|
||||
if (isLikelyTotalRow(cells)) continue;
|
||||
|
||||
rows.push(analyzeRow(
|
||||
i, cells, headerMap, headerLabels, userBills, categories,
|
||||
name, sheetYear, sheetMonth,
|
||||
defaultYear, defaultMonth, rowIdPrefix,
|
||||
defaultDueDay, setIdx,
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
rows,
|
||||
headerRow: hasHeaders ? firstRow.map((c) => (c != null ? String(c) : null)) : null,
|
||||
headerRow: hasHeaders ? headerLabels : null,
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -1439,6 +1598,7 @@ function getImportHistory(userId) {
|
|||
}
|
||||
|
||||
module.exports = {
|
||||
detectAllHeaderSets,
|
||||
previewSpreadsheet,
|
||||
applyImportDecisions,
|
||||
getImportHistory,
|
||||
|
|
|
|||
Loading…
Reference in New Issue