From eaedeb398c4351241e8bf5e51212c44e05a576ac Mon Sep 17 00:00:00 2001 From: null Date: Tue, 14 Jul 2026 18:37:59 -0500 Subject: [PATCH] fix(seed): make build_db.py produce a Room-loadable asset database MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit build_db.py could not produce a shippable app.db and would have crashed the app on first launch. It created only 2 tables (Question, QuestionCategory) while AppDatabase declares 4 entities, and it never wrote room_master_table. Since createFromAsset runs with no migrations and no destructive fallback, Room validates strictly against identity hash 7e7d78fc... — so the output was guaranteed to fail. Hence the standing "never run build_db.py" rule; the tool had drifted from the schema it feeds. Schema is now taken from Room's own exported schema (app/schemas/app.closer.data.local.AppDatabase/1.json, via room.schemaLocation) rather than hand-written DDL: all 4 entity tables + indices + the identity row are created from it, so the script follows future schema changes automatically. Verified the output's PRAGMA table_info/index_list is identical to the shipped known-good db for all 5 tables, with a matching identity hash. Also fixes the answer_config contract. The app's only parser (QuestionMapper.parseAnswerConfig) reads camelCase keys, but the shipped db stores snake_case and the old script emitted a third shape again — so scale labels silently resolved to "" (the UI then shows bare numbers) and written max_length was ignored. The authoring JSON stays snake_case per QUESTION_SCHEMA.md; translating to the parser's shape is now this script's job. Content problems are now hard failures instead of silent corruption. Previously a JSON parse error or missing category made the script skip/mislabel a whole pack ("unknown"). Now rejected with a precise message: parse errors, missing category object (patch manifest under a production filename), zero questions, missing id, duplicate id/text, invalid type/access, category_id mismatch, non-integer depth, choice without options, this_or_that without exactly 2. All 12 guards verified against mutated fixtures; a valid pack still builds. Adds --check (validate only) and --out (build elsewhere). Builds are staged to a temp file, verified, and only then moved into place, backing up the existing asset db first. Co-Authored-By: Claude Opus 4.8 --- .gitignore | 4 + seed/build_db.py | 580 +++++++++++++++++++++++++++-------------------- 2 files changed, 332 insertions(+), 252 deletions(-) diff --git a/.gitignore b/.gitignore index 94758229..f4c179c0 100644 --- a/.gitignore +++ b/.gitignore @@ -103,3 +103,7 @@ revenucat_secret_api.md release_guide.md seed/questions/closer_question_guides_150_max.zip seed/questions/rebuilding_trust_BATCH1_NOTE.md + +# Python bytecode from the seed scripts +__pycache__/ +*.pyc diff --git a/seed/build_db.py b/seed/build_db.py index a5f191ae..a60644f0 100644 --- a/seed/build_db.py +++ b/seed/build_db.py @@ -1,276 +1,352 @@ #!/usr/bin/env python3 """ -Build SQLite database from question JSON files for Room asset loading. +Build the Room-loadable SQLite asset database from the question JSON packs. -This script reads JSON question files and creates a pre-seeded SQLite database -that can be bundled with the APK and loaded by Room via createFromAsset(). +The app loads this file via Room's `createFromAsset("database/app.db")` with **no +migrations and no destructive fallback**, so the output must satisfy Room exactly: + + * every entity table declared by `AppDatabase` must exist, with Room's exact SQL + * `room_master_table` must carry Room's identity hash for the schema version + +Both of those come from Room's own exported schema +(`app/schemas/app.closer.data.local.AppDatabase/.json`, produced by +`room.schemaLocation` in app/build.gradle.kts). That file is the single source of +truth here — when the Room schema changes, re-run the app build to re-export it and +this script follows automatically. Nothing about the schema is hardcoded. + +`answer_config` is written in the shape the app's parser actually reads +(`data/local/mapper/QuestionMapper.kt` -> `parseAnswerConfig`), which uses camelCase +keys. The authoring JSON uses snake_case (see seed/questions/QUESTION_SCHEMA.md); +translating between the two is this script's job. + +Usage: + python3 seed/build_db.py # build + replace the asset db (backs up first) + python3 seed/build_db.py --out /tmp/x.db # build somewhere else (no backup, no replace) + python3 seed/build_db.py --check # validate the JSON only, build nothing + +Any content problem is a hard failure. This script never writes a partial database. """ +import argparse import json +import shutil import sqlite3 -import os +import sys +import tempfile from pathlib import Path -from typing import Dict, List, Any +from typing import Any, Dict, List from validate_question_variety import load_json_records, validate_records +ROOT = Path(__file__).resolve().parents[1] +JSON_DIR = ROOT / "seed" / "questions" +SCHEMA_DIR = ROOT / "app" / "schemas" / "app.closer.data.local.AppDatabase" +ASSET_DB = ROOT / "app" / "src" / "main" / "assets" / "database" / "app.db" -def load_json_file(filepath: str) -> Dict[str, Any]: - """Load and parse a JSON file.""" - with open(filepath, 'r', encoding='utf-8') as f: - return json.load(f) +VALID_TYPES = {"written", "single_choice", "multi_choice", "scale", "this_or_that"} +VALID_ACCESS = {"free", "premium"} +CHOICE_TYPES = {"single_choice", "multi_choice"} + +# Room writes its identity row with this fixed id; see RoomOpenHelper. +ROOM_MASTER_TABLE_ID = 42 +ROOM_MASTER_DDL = ( + "CREATE TABLE IF NOT EXISTS room_master_table " + "(id INTEGER PRIMARY KEY, identity_hash TEXT)" +) -def get_category_id_from_filename(filename: str) -> str: - """Extract category id from filename.""" - # Handle both v1 and v2 filenames - # examples: questions_communication---...json, questions_communication_v2---...json - basename = os.path.basename(filename) - if '_v2---' in basename: - return basename.split('_v2---')[0].replace('questions_', '') - elif '---' in basename: - return basename.split('---')[0].replace('questions_', '') - return 'unknown' +class ContentError(Exception): + """A problem in the seed JSON. Always fatal — never import partial content.""" -def build_database(json_dir: str, output_path: str) -> None: - """Build SQLite database from JSON files.""" - variety_errors = validate_records(load_json_records(Path(json_dir)), "JSON") - if variety_errors: - raise ValueError( - "Question variety check failed:\n" + "\n".join(variety_errors) +# -------------------------------------------------------------------------- +# Room schema +# -------------------------------------------------------------------------- + +def load_room_schema() -> Dict[str, Any]: + """Load Room's exported schema (highest version present).""" + if not SCHEMA_DIR.is_dir(): + raise ContentError( + f"Room schema export not found at {SCHEMA_DIR}.\n" + "Build the app once (room.schemaLocation is configured) so Room exports it." ) - - # Ensure output directory exists - os.makedirs(os.path.dirname(output_path), exist_ok=True) - - # Remove existing database if present - if os.path.exists(output_path): - os.remove(output_path) - - # Connect to new database - conn = sqlite3.connect(output_path) - cursor = conn.cursor() - - # Create tables with Room-compatible schema - # Question table - cursor.execute(''' - CREATE TABLE Question ( - id TEXT NOT NULL, - text TEXT NOT NULL, - category_id TEXT NOT NULL, - depth_level INTEGER NOT NULL, - is_premium INTEGER NOT NULL, - type TEXT NOT NULL, - tags TEXT NOT NULL, - answer_config TEXT NOT NULL, - pack_id TEXT, - created_at INTEGER NOT NULL, - status TEXT NOT NULL, - sex TEXT, - PRIMARY KEY (id) - ) - ''') - - # QuestionCategory table - cursor.execute(''' - CREATE TABLE QuestionCategory ( - id TEXT NOT NULL, - display_name TEXT NOT NULL, - description TEXT NOT NULL, - access TEXT NOT NULL, - icon_name TEXT NOT NULL, - PRIMARY KEY (id) - ) - ''') - - # Create indexes - cursor.execute(''' - CREATE INDEX idx_question_category_id ON Question(category_id) - ''') - - # Process each JSON file (support both prefixed and clean filenames) - json_files = list(Path(json_dir).glob('*.json')) - - total_questions = 0 - categories_processed = set() - - for json_file in json_files: - print(f"\nProcessing: {json_file.name}") - + versions = [] + for path in SCHEMA_DIR.glob("*.json"): try: - data = load_json_file(str(json_file)) + versions.append((int(path.stem), path)) + except ValueError: + continue + if not versions: + raise ContentError(f"No exported schema json in {SCHEMA_DIR}") + _, path = max(versions) + return json.loads(path.read_text(encoding="utf-8"))["database"] + + +def create_schema(cursor: sqlite3.Cursor, db_schema: Dict[str, Any]) -> None: + """Create every Room entity table + the identity row, exactly as Room expects.""" + for entity in db_schema["entities"]: + table = entity["tableName"] + cursor.execute(entity["createSql"].replace("${TABLE_NAME}", table)) + for index in entity.get("indices", []): + cursor.execute(index["createSql"].replace("${TABLE_NAME}", table)) + + for view in db_schema.get("views", []): + cursor.execute(view["createSql"].replace("${VIEW_NAME}", view["viewName"])) + + cursor.execute(ROOM_MASTER_DDL) + cursor.execute( + "INSERT OR REPLACE INTO room_master_table (id, identity_hash) VALUES (?, ?)", + (ROOM_MASTER_TABLE_ID, db_schema["identityHash"]), + ) + + +# -------------------------------------------------------------------------- +# answer_config: authoring shape (snake_case) -> app parser shape (camelCase) +# -------------------------------------------------------------------------- + +def _options(question: Dict[str, Any]) -> List[Dict[str, str]]: + """Options live top-level per the schema guide, mirrored into answer_config.""" + opts = question.get("options") or (question.get("answer_config") or {}).get("options") + return opts or [] + + +def build_answer_config(question: Dict[str, Any], where: str) -> Dict[str, Any]: + """Produce the answer_config the app's QuestionMapper can actually read.""" + qtype = question["type"] + src = question.get("answer_config") or {} + + if qtype == "written": + return { + "type": "written", + "config": { + "minLength": src.get("min_length", 1), + "maxLength": src.get("max_length", 1000), + "placeholder": src.get("placeholder", "Write your answer..."), + }, + } + + if qtype in CHOICE_TYPES: + opts = _options(question) + if not opts: + raise ContentError(f"{where}: {qtype} has no options") + config: Dict[str, Any] = {"options": [{"id": o["id"], "text": o["text"]} for o in opts]} + if qtype == "multi_choice": + config["maxSelections"] = src.get("max_selections", 0) + return {"type": qtype, "config": config} + + if qtype == "scale": + return { + "type": "scale", + "config": { + "minScale": src.get("min", 1), + "maxScale": src.get("max", 5), + "minLabel": src.get("min_label", ""), + "maxLabel": src.get("max_label", ""), + "scaleStep": src.get("scale_step", 1), + }, + } + + if qtype == "this_or_that": + opts = _options(question) + if len(opts) != 2: + raise ContentError(f"{where}: this_or_that needs exactly 2 options, found {len(opts)}") + return { + "type": "this_or_that", + "config": { + "optionA": {"id": opts[0]["id"], "text": opts[0]["text"]}, + "optionB": {"id": opts[1]["id"], "text": opts[1]["text"]}, + }, + } + + raise ContentError(f"{where}: unsupported type {qtype!r}") + + +# -------------------------------------------------------------------------- +# Validation — every problem is fatal +# -------------------------------------------------------------------------- + +def load_packs() -> List[Dict[str, Any]]: + """Load and hard-validate every pack. Raises on the first structural problem.""" + files = sorted(JSON_DIR.glob("*.json")) + if not files: + raise ContentError(f"No question JSON found in {JSON_DIR}") + + packs, seen_ids = [], {} + for path in files: + name = path.name + try: + data = json.loads(path.read_text(encoding="utf-8")) except json.JSONDecodeError as e: - print(f" ❌ Invalid JSON: {e}") - continue - except Exception as e: - print(f" ❌ Error reading file: {e}") - continue - - # Extract category info - category_data = data.get('category', {}) - category_id = category_data.get('id', get_category_id_from_filename(str(json_file))) - category_display_name = category_data.get('display_name', category_id) - category_description = category_data.get('description', '') - category_access = category_data.get('access', 'free') - category_icon = category_data.get('icon_name', 'question') - - # Insert category (ignore duplicates) - cursor.execute(''' - INSERT OR IGNORE INTO QuestionCategory - (id, display_name, description, access, icon_name) - VALUES (?, ?, ?, ?, ?) - ''', (category_id, category_display_name, category_description, category_access, category_icon)) - - if category_id not in categories_processed: - categories_processed.add(category_id) - print(f" Category: {category_display_name} (id: {category_id})") - - # Insert questions - questions = data.get('questions', []) - questions_inserted = 0 - - for q in questions: - question_id = q.get('id') - text = q.get('text', '') - category_id_q = q.get('category_id', category_id) - depth_level = q.get('depth', q.get('depth_level', 1)) - is_premium = q.get('access', 'free') == 'premium' - question_type = q.get('type', 'written') - tags = q.get('tags', []) - - # Handle answer_config - check both patterns - answer_config = q.get('answer_config', {}) - options = q.get('options', []) # Alternative location for choice types - - # Build answer_config JSON - ac_json = {} - - if question_type == 'written': - ac_json = { - 'type': 'written', - 'config': answer_config if answer_config else { - 'minLength': 1, - 'maxLength': 1000, - 'placeholder': 'Write your answer...' - } - } - elif question_type == 'single_choice': - if options: - ac_json = { - 'type': 'single_choice', - 'config': {'options': options} - } - elif answer_config: - ac_json = { - 'type': 'single_choice', - 'config': answer_config - } - elif question_type == 'multi_choice': - if options: - ac_json = { - 'type': 'multi_choice', - 'config': {'options': options} - } - elif answer_config: - ac_json = { - 'type': 'multi_choice', - 'config': answer_config - } - elif question_type == 'scale': - ac_json = { - 'type': 'scale', - 'config': answer_config if answer_config else { - 'minScale': 1, - 'maxScale': 5, - 'minLabel': 'Disagree', - 'maxLabel': 'Agree' - } - } - elif question_type == 'this_or_that': - if options: - ac_json = { - 'type': 'this_or_that', - 'config': { - 'optionA': options[0] if len(options) > 0 else {'id': 'a', 'text': ''}, - 'optionB': options[1] if len(options) > 1 else {'id': 'b', 'text': ''} - } - } - elif answer_config: - ac_json = { - 'type': 'this_or_that', - 'config': answer_config - } - - # Convert tags and answer_config to JSON strings - tags_json = json.dumps(tags, separators=(',', ':')) - ac_json_str = json.dumps(ac_json, separators=(',', ':')) - - # Extract optional sex field (used for Desire Sync filtering) - sex = q.get('sex') + raise ContentError(f"{name}: invalid JSON: {e}") from e - # Insert question - cursor.execute(''' - INSERT OR REPLACE INTO Question - (id, text, category_id, depth_level, is_premium, type, tags, answer_config, - created_at, status, sex) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - ''', ( - question_id, - text, - category_id_q, - depth_level, - 1 if is_premium else 0, - question_type, - tags_json, - ac_json_str, - int(json_file.stat().st_mtime), # Use file mtime as created_at - 'active', - sex - )) - - questions_inserted += 1 - total_questions += 1 - - free_count = sum(1 for q in questions if q.get('access', 'free') == 'free') - premium_count = sum(1 for q in questions if q.get('access', 'free') == 'premium') - print(f" Questions: {questions_inserted} ({free_count} free, {premium_count} premium)") - - # Commit and close - conn.commit() - conn.close() - - print(f"\n{'='*60}") - print(f"Database built: {output_path}") - print(f"Categories: {len(categories_processed)}") - print(f"Total questions: {total_questions}") - print(f"{'='*60}") + category = data.get("category") + if not isinstance(category, dict): + raise ContentError( + f"{name}: missing top-level 'category' object. " + "A patch manifest or partial batch must never sit under a production filename." + ) + for field in ("id", "display_name", "description", "access", "icon_name"): + if not category.get(field): + raise ContentError(f"{name}: category.{field} is missing or empty") + + questions = data.get("questions") + if not isinstance(questions, list) or not questions: + raise ContentError(f"{name}: 'questions' is missing or empty") + + cid = category["id"] + for i, q in enumerate(questions): + qid = q.get("id") + where = f"{name}:{qid or f'#{i}'}" + if not qid: + raise ContentError(f"{where}: question has no id") + if qid in seen_ids: + raise ContentError(f"{where}: duplicate id (also in {seen_ids[qid]})") + seen_ids[qid] = name + if not q.get("text"): + raise ContentError(f"{where}: empty text") + if q.get("type") not in VALID_TYPES: + raise ContentError(f"{where}: invalid type {q.get('type')!r}") + if q.get("access") not in VALID_ACCESS: + raise ContentError(f"{where}: invalid access {q.get('access')!r}") + if q.get("category_id") != cid: + raise ContentError( + f"{where}: category_id {q.get('category_id')!r} != category.id {cid!r}" + ) + depth = q.get("depth", q.get("depth_level")) + if not isinstance(depth, int) or isinstance(depth, bool): + raise ContentError( + f"{where}: depth {depth!r} is not an integer. " + "String depth is a future migration and is not production-ready " + "(see QUESTION_SCHEMA.md); Room reads depth_level as an integer." + ) + if not isinstance(q.get("tags"), list): + raise ContentError(f"{where}: tags must be an array") + # Surfaces option problems now rather than as an unusable question later. + build_answer_config(q, where) + + packs.append({"name": name, "category": category, "questions": questions, + "mtime": int(path.stat().st_mtime)}) + + errors = validate_records(load_json_records(JSON_DIR), "JSON") + if errors: + raise ContentError("Catalog variety gate failed:\n" + "\n".join(errors)) + + return packs -def main(): - """Main entry point.""" - # Paths - script_dir = Path(__file__).parent - json_dir = script_dir / 'questions' - output_dir = script_dir.parent / 'app' / 'src' / 'main' / 'assets' / 'database' - output_path = output_dir / 'app.db' - - print("Building SQLite database from question JSON files...") - print(f"JSON directory: {json_dir}") - print(f"Output path: {output_path}") - - # Verify JSON directory exists - if not json_dir.exists(): - print(f"❌ JSON directory not found: {json_dir}") - return - - if not list(json_dir.glob('*.json')): - print(f"❌ No question JSON files found in: {json_dir}") - return - - # Build database - build_database(str(json_dir), str(output_path)) +# -------------------------------------------------------------------------- +# Build +# -------------------------------------------------------------------------- + +def build(packs: List[Dict[str, Any]], db_schema: Dict[str, Any], out: Path) -> Dict[str, int]: + out.parent.mkdir(parents=True, exist_ok=True) + if out.exists(): + out.unlink() + + conn = sqlite3.connect(out) + try: + cur = conn.cursor() + create_schema(cur, db_schema) + + total = 0 + for pack in packs: + c = pack["category"] + cur.execute( + "INSERT OR REPLACE INTO question_category " + "(id, display_name, description, access, icon_name) VALUES (?, ?, ?, ?, ?)", + (c["id"], c["display_name"], c["description"], c["access"], c["icon_name"]), + ) + for q in pack["questions"]: + where = f"{pack['name']}:{q['id']}" + cur.execute( + "INSERT OR REPLACE INTO question " + "(id, text, category_id, depth_level, is_premium, type, tags, answer_config," + " pack_id, created_at, status, sex) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ( + q["id"], + q["text"], + q["category_id"], + q.get("depth", q.get("depth_level")), + 1 if q["access"] == "premium" else 0, + q["type"], + json.dumps(q.get("tags", []), separators=(",", ":")), + json.dumps(build_answer_config(q, where), separators=(",", ":")), + None, + pack["mtime"], + "active", + q.get("sex"), + ), + ) + total += 1 + conn.commit() + finally: + conn.close() + return {"categories": len(packs), "questions": total} -if __name__ == '__main__': - main() +def verify(out: Path, db_schema: Dict[str, Any]) -> None: + """Re-open the built file and prove it satisfies Room before it is shipped.""" + conn = sqlite3.connect(out) + try: + tables = {r[0] for r in conn.execute( + "SELECT name FROM sqlite_master WHERE type='table'")} + expected = {e["tableName"] for e in db_schema["entities"]} | {"room_master_table"} + missing = expected - tables + if missing: + raise ContentError(f"built db is missing Room tables: {sorted(missing)}") + + row = conn.execute( + "SELECT id, identity_hash FROM room_master_table").fetchone() + if not row or row[0] != ROOM_MASTER_TABLE_ID or row[1] != db_schema["identityHash"]: + raise ContentError(f"identity hash mismatch: {row} != {db_schema['identityHash']}") + + empty = [t for t in ("question", "question_category") + if conn.execute(f"SELECT COUNT(*) FROM {t}").fetchone()[0] == 0] + if empty: + raise ContentError(f"built db has empty tables: {empty}") + finally: + conn.close() + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--out", type=Path, default=ASSET_DB, + help=f"output path (default: {ASSET_DB})") + ap.add_argument("--check", action="store_true", + help="validate the JSON and exit without building") + args = ap.parse_args() + + try: + db_schema = load_room_schema() + packs = load_packs() + print(f"Validated {len(packs)} packs, " + f"{sum(len(p['questions']) for p in packs)} questions — catalog gate passed.") + if args.check: + return 0 + + replacing_asset = args.out.resolve() == ASSET_DB.resolve() + with tempfile.TemporaryDirectory() as tmp: + staged = Path(tmp) / "app.db" + stats = build(packs, db_schema, staged) + verify(staged, db_schema) + + if replacing_asset and args.out.exists(): + backup = args.out.with_suffix(".db.bak") + shutil.copy2(args.out, backup) + print(f"Backed up existing asset db -> {backup.name}") + args.out.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(staged, args.out) + except ContentError as e: + print(f"\nBUILD FAILED\n{e}", file=sys.stderr) + return 1 + + print(f"Built {args.out}") + print(f" schema version {db_schema['version']} · identity {db_schema['identityHash']}") + print(f" {stats['categories']} categories · {stats['questions']} questions") + return 0 + + +if __name__ == "__main__": + sys.exit(main())