|
|
|
|
@ -206,9 +206,9 @@ function parseXlsxBuffer(buffer) {
|
|
|
|
|
if (!cell) continue;
|
|
|
|
|
|
|
|
|
|
// Strict cell type validation
|
|
|
|
|
// Only allow n (number), t (text/string), b (boolean), d (date)
|
|
|
|
|
// Reject array (a), error (e), formula (f), shared formula (s)
|
|
|
|
|
if (cell.t && !['n', 't', 'b', 'd'].includes(cell.t)) {
|
|
|
|
|
// Only allow n (number), t (text/string), b (boolean), d (date), s (shared formula)
|
|
|
|
|
// Reject array (a), error (e), formula (f)
|
|
|
|
|
if (cell.t && !['n', 't', 'b', 'd', 's'].includes(cell.t)) {
|
|
|
|
|
const err = new Error(`Invalid cell type '${cell.t}' found in ${cellRef}. Only numbers and text are supported.`);
|
|
|
|
|
err.status = 400;
|
|
|
|
|
throw err;
|
|
|
|
|
@ -252,12 +252,114 @@ function detectHeaders(firstRow) {
|
|
|
|
|
return map;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// ─── Dual-Column Header Detection ──────────────────────────────────────────────
|
|
|
|
|
/**
|
|
|
|
|
* Detect all header sets in a row, handling dual-column layouts.
|
|
|
|
|
* When a single row contains TWO sets of bill headers (e.g., columns A-E and G-K),
|
|
|
|
|
* this function returns an array of header groups, each with its own column range.
|
|
|
|
|
*
|
|
|
|
|
* Each group has: startCol, endCol, map, defaultDueDay (1 or 15)
|
|
|
|
|
*/
|
|
|
|
|
function detectAllHeaderSets(firstRow) {
|
|
|
|
|
if (!Array.isArray(firstRow)) return [];
|
|
|
|
|
|
|
|
|
|
// First, detect header cells and their column indices
|
|
|
|
|
const headerCells = [];
|
|
|
|
|
firstRow.forEach((cell, idx) => {
|
|
|
|
|
if (cell == null) return;
|
|
|
|
|
const val = String(cell).trim();
|
|
|
|
|
for (const field of Object.keys(HEADER_PATTERNS)) {
|
|
|
|
|
if (HEADER_PATTERNS[field].test(val)) {
|
|
|
|
|
headerCells.push({ idx, field });
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
if (headerCells.length === 0) return [];
|
|
|
|
|
|
|
|
|
|
// Group consecutive header cells into sets
|
|
|
|
|
// A gap of more than 1 column (empty column) indicates a new header set
|
|
|
|
|
const headerSets = [];
|
|
|
|
|
let currentSet = { startCol: headerCells[0].idx, endCol: headerCells[0].idx, fields: [headerCells[0].field] };
|
|
|
|
|
|
|
|
|
|
for (let i = 1; i < headerCells.length; i++) {
|
|
|
|
|
const prevIdx = headerCells[i - 1].idx;
|
|
|
|
|
const currIdx = headerCells[i].idx;
|
|
|
|
|
|
|
|
|
|
// Check if there's an empty column between them (gap > 1)
|
|
|
|
|
let hasGap = false;
|
|
|
|
|
for (let gapIdx = prevIdx + 1; gapIdx < currIdx; gapIdx++) {
|
|
|
|
|
if (firstRow[gapIdx] == null || String(firstRow[gapIdx]).trim() === '') {
|
|
|
|
|
hasGap = true;
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (hasGap) {
|
|
|
|
|
// Save current set and start a new one
|
|
|
|
|
headerSets.push(currentSet);
|
|
|
|
|
currentSet = { startCol: currIdx, endCol: currIdx, fields: [headerCells[i].field] };
|
|
|
|
|
} else {
|
|
|
|
|
currentSet.endCol = currIdx;
|
|
|
|
|
currentSet.fields.push(headerCells[i].field);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
headerSets.push(currentSet);
|
|
|
|
|
|
|
|
|
|
// Convert to final format with maps and defaultDueDay
|
|
|
|
|
return headerSets.map(set => {
|
|
|
|
|
const map = {};
|
|
|
|
|
for (const field of set.fields) {
|
|
|
|
|
// Find the first occurrence of this field in the set
|
|
|
|
|
for (let i = set.startCol; i <= set.endCol; i++) {
|
|
|
|
|
if (firstRow[i] != null && HEADER_PATTERNS[field].test(String(firstRow[i]).trim())) {
|
|
|
|
|
map[field] = i;
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
// Default due_day based on column position: left half (cols < 5) = 1, right half (cols >= 6) = 15
|
|
|
|
|
const defaultDueDay = set.startCol < 5 ? 1 : 15;
|
|
|
|
|
|
|
|
|
|
return { startCol: set.startCol, endCol: set.endCol, map, defaultDueDay };
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// ─── Row Classification ───────────────────────────────────────────────────────
|
|
|
|
|
|
|
|
|
|
function isBlankRow(cells) {
|
|
|
|
|
return cells.every((c) => c == null || String(c).trim() === '');
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Check if a row is blank for a specific header set's columns.
|
|
|
|
|
* For dual-column layouts, a row may be blank on the left but have data on the right.
|
|
|
|
|
* Uses absolute column indices from the header set map.
|
|
|
|
|
*/
|
|
|
|
|
function isBlankRowForHeaderSet(cells, headerSet) {
|
|
|
|
|
const { map } = headerSet;
|
|
|
|
|
|
|
|
|
|
// Check the bill_name column and amount column for this header set
|
|
|
|
|
const billNameIdx = map.bill_name;
|
|
|
|
|
const amountIdx = map.amount;
|
|
|
|
|
|
|
|
|
|
// If we can't find bill_name or amount columns, fall back to full-row blank check
|
|
|
|
|
if (billNameIdx === undefined && amountIdx === undefined) {
|
|
|
|
|
return isBlankRow(cells);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const billNameCell = billNameIdx !== undefined ? cells[billNameIdx] : undefined;
|
|
|
|
|
const amountCell = amountIdx !== undefined ? cells[amountIdx] : undefined;
|
|
|
|
|
|
|
|
|
|
const billNameBlank = billNameCell == null || String(billNameCell).trim() === '';
|
|
|
|
|
const amountBlank = amountCell == null || String(amountCell).trim() === '' || parseAmount(amountCell) === null;
|
|
|
|
|
|
|
|
|
|
// If both bill name and amount are blank, this row is empty for this set
|
|
|
|
|
return billNameBlank && amountBlank;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function isLikelyHeaderRow(cells) {
|
|
|
|
|
const nonEmpty = cells.filter((c) => c != null && String(c).trim() !== '');
|
|
|
|
|
if (nonEmpty.length === 0) return false;
|
|
|
|
|
@ -507,6 +609,7 @@ function buildRecommendation({
|
|
|
|
|
warnings,
|
|
|
|
|
errors,
|
|
|
|
|
paymentDateIso,
|
|
|
|
|
defaultDueDay = null,
|
|
|
|
|
}) {
|
|
|
|
|
const recWarnings = [...warnings];
|
|
|
|
|
const topMatch = possibleMatches[0] || null;
|
|
|
|
|
@ -514,7 +617,11 @@ function buildRecommendation({
|
|
|
|
|
const mediumMatches = possibleMatches.filter((m) => m.match_confidence === 'medium');
|
|
|
|
|
|
|
|
|
|
const dateDay = parsedDate?.day;
|
|
|
|
|
const dueDay = Number.isInteger(dateDay) && dateDay >= 1 && dateDay <= 31 ? dateDay : null;
|
|
|
|
|
let dueDay = Number.isInteger(dateDay) && dateDay >= 1 && dateDay <= 31 ? dateDay : null;
|
|
|
|
|
// Use defaultDueDay from header set if date parsing didn't find a day
|
|
|
|
|
if (dueDay === null && defaultDueDay !== null) {
|
|
|
|
|
dueDay = defaultDueDay;
|
|
|
|
|
}
|
|
|
|
|
const paymentDate = isPaymentDateHeader(dateHeader);
|
|
|
|
|
if (dueDay && paymentDate && !isDueDateHeader(dateHeader)) {
|
|
|
|
|
recWarnings.push('Date appears to be a payment date, not a due date');
|
|
|
|
|
@ -666,7 +773,7 @@ function collectNotesCells(cells, headerMap, billName) {
|
|
|
|
|
|
|
|
|
|
// ─── Single-Row Analyzer ──────────────────────────────────────────────────────
|
|
|
|
|
|
|
|
|
|
function analyzeRow(rowIndex, cells, headerMap, headerLabels, userBills, categories, sheetName, sheetYear, sheetMonth, defaultYear, defaultMonth, rowIdPrefix) {
|
|
|
|
|
function analyzeRow(rowIndex, cells, headerMap, headerLabels, userBills, categories, sheetName, sheetYear, sheetMonth, defaultYear, defaultMonth, rowIdPrefix, defaultDueDay = null, headerSetIndex = null) {
|
|
|
|
|
const get = (field) => {
|
|
|
|
|
const idx = headerMap[field];
|
|
|
|
|
return idx !== undefined ? cells[idx] : undefined;
|
|
|
|
|
@ -721,6 +828,7 @@ function analyzeRow(rowIndex, cells, headerMap, headerLabels, userBills, categor
|
|
|
|
|
warnings,
|
|
|
|
|
errors,
|
|
|
|
|
paymentDateIso: detectedPaidDate,
|
|
|
|
|
defaultDueDay,
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
const proposedAction = recommendation.action === 'ambiguous' ? 'mark_ambiguous' : recommendation.action;
|
|
|
|
|
@ -751,6 +859,7 @@ function analyzeRow(rowIndex, cells, headerMap, headerLabels, userBills, categor
|
|
|
|
|
errors,
|
|
|
|
|
possible_bill_matches: possibleMatches,
|
|
|
|
|
requires_user_decision: requiresUserDecision,
|
|
|
|
|
due_day: recommendation.due_day,
|
|
|
|
|
recommendation,
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
@ -764,29 +873,79 @@ function analyzeRow(rowIndex, cells, headerMap, headerLabels, userBills, categor
|
|
|
|
|
function parseSheetRows({ name, rawRows, year: sheetYear, month: sheetMonth, rowIdPrefix }, userBills, categories, defaultYear, defaultMonth) {
|
|
|
|
|
if (!rawRows.length) return { rows: [], headerRow: null };
|
|
|
|
|
|
|
|
|
|
const firstRow = rawRows[0] || [];
|
|
|
|
|
const headerMap = detectHeaders(firstRow);
|
|
|
|
|
const headerLabels = firstRow.map((c) => (c != null ? String(c).trim() : null));
|
|
|
|
|
const hasHeaders = Object.keys(headerMap).length > 0;
|
|
|
|
|
const startRow = hasHeaders ? 1 : 0;
|
|
|
|
|
// Detect all header sets in each row to handle dual-column layouts
|
|
|
|
|
let headerRowIndex = 0;
|
|
|
|
|
let headerLabels = rawRows[0]?.map((c) => (c != null ? String(c).trim() : null)) || [];
|
|
|
|
|
|
|
|
|
|
// First try to detect headers in row 0
|
|
|
|
|
let allHeaderSets = detectAllHeaderSets(rawRows[0]);
|
|
|
|
|
|
|
|
|
|
// If no headers in row 0, scan up to 5 rows
|
|
|
|
|
for (let scanIdx = 1; scanIdx < Math.min(5, rawRows.length); scanIdx++) {
|
|
|
|
|
const candidateSets = detectAllHeaderSets(rawRows[scanIdx]);
|
|
|
|
|
if (candidateSets.length > 0) {
|
|
|
|
|
headerRowIndex = scanIdx;
|
|
|
|
|
headerLabels = rawRows[scanIdx].map((c) => (c != null ? String(c).trim() : null));
|
|
|
|
|
allHeaderSets = candidateSets;
|
|
|
|
|
// Check if this set has all required fields
|
|
|
|
|
let hasAllRequired = false;
|
|
|
|
|
for (const set of allHeaderSets) {
|
|
|
|
|
if (set.map.due_date !== undefined && set.map.bill_name !== undefined && set.map.amount !== undefined) {
|
|
|
|
|
hasAllRequired = true;
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if (hasAllRequired) {
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Check if we have valid headers (must have due_date, bill_name, amount)
|
|
|
|
|
let hasValidHeaders = false;
|
|
|
|
|
for (const set of allHeaderSets) {
|
|
|
|
|
if (set.map.due_date !== undefined && set.map.bill_name !== undefined && set.map.amount !== undefined) {
|
|
|
|
|
hasValidHeaders = true;
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const hasHeaders = hasValidHeaders;
|
|
|
|
|
const startRow = hasHeaders ? headerRowIndex + 1 : 0;
|
|
|
|
|
|
|
|
|
|
const rows = [];
|
|
|
|
|
for (let i = startRow; i < rawRows.length; i++) {
|
|
|
|
|
const cells = rawRows[i] || [];
|
|
|
|
|
if (isBlankRow(cells)) continue;
|
|
|
|
|
if (isLikelyHeaderRow(cells) && i > 0) continue;
|
|
|
|
|
if (isLikelyTotalRow(cells)) continue;
|
|
|
|
|
|
|
|
|
|
rows.push(analyzeRow(
|
|
|
|
|
i, cells, headerMap, headerLabels, userBills, categories,
|
|
|
|
|
name, sheetYear, sheetMonth,
|
|
|
|
|
defaultYear, defaultMonth, rowIdPrefix,
|
|
|
|
|
));
|
|
|
|
|
|
|
|
|
|
// Process each header set independently
|
|
|
|
|
for (let setIdx = 0; setIdx < allHeaderSets.length; setIdx++) {
|
|
|
|
|
const headerSet = allHeaderSets[setIdx];
|
|
|
|
|
const headerMap = headerSet.map;
|
|
|
|
|
const defaultDueDay = headerSet.defaultDueDay;
|
|
|
|
|
|
|
|
|
|
for (let i = startRow; i < rawRows.length; i++) {
|
|
|
|
|
const cells = rawRows[i] || [];
|
|
|
|
|
|
|
|
|
|
// For dual-column: skip rows blank in this header set's columns only
|
|
|
|
|
// For single-column: fall back to regular isBlankRow
|
|
|
|
|
if (allHeaderSets.length > 1 ? isBlankRowForHeaderSet(cells, headerSet) : isBlankRow(cells)) continue;
|
|
|
|
|
|
|
|
|
|
// Skip duplicate header rows (but only if we found headers)
|
|
|
|
|
if (hasHeaders && isLikelyHeaderRow(cells) && i > headerRowIndex) continue;
|
|
|
|
|
|
|
|
|
|
// Skip total rows
|
|
|
|
|
if (isLikelyTotalRow(cells)) continue;
|
|
|
|
|
|
|
|
|
|
rows.push(analyzeRow(
|
|
|
|
|
i, cells, headerMap, headerLabels, userBills, categories,
|
|
|
|
|
name, sheetYear, sheetMonth,
|
|
|
|
|
defaultYear, defaultMonth, rowIdPrefix,
|
|
|
|
|
defaultDueDay, setIdx,
|
|
|
|
|
));
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return {
|
|
|
|
|
rows,
|
|
|
|
|
headerRow: hasHeaders ? firstRow.map((c) => (c != null ? String(c) : null)) : null,
|
|
|
|
|
headerRow: hasHeaders ? headerLabels : null,
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@ -1439,6 +1598,7 @@ function getImportHistory(userId) {
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
module.exports = {
|
|
|
|
|
detectAllHeaderSets,
|
|
|
|
|
previewSpreadsheet,
|
|
|
|
|
applyImportDecisions,
|
|
|
|
|
getImportHistory,
|
|
|
|
|
|