fix(seed): make build_db.py produce a Room-loadable asset database

build_db.py could not produce a shippable app.db and would have crashed the
app on first launch. It created only 2 tables (Question, QuestionCategory)
while AppDatabase declares 4 entities, and it never wrote room_master_table.
Since createFromAsset runs with no migrations and no destructive fallback,
Room validates strictly against identity hash 7e7d78fc... — so the output was
guaranteed to fail. Hence the standing "never run build_db.py" rule; the tool
had drifted from the schema it feeds.

Schema is now taken from Room's own exported schema
(app/schemas/app.closer.data.local.AppDatabase/1.json, via room.schemaLocation)
rather than hand-written DDL: all 4 entity tables + indices + the identity row
are created from it, so the script follows future schema changes automatically.
Verified the output's PRAGMA table_info/index_list is identical to the shipped
known-good db for all 5 tables, with a matching identity hash.

Also fixes the answer_config contract. The app's only parser
(QuestionMapper.parseAnswerConfig) reads camelCase keys, but the shipped db
stores snake_case and the old script emitted a third shape again — so scale
labels silently resolved to "" (the UI then shows bare numbers) and written
max_length was ignored. The authoring JSON stays snake_case per
QUESTION_SCHEMA.md; translating to the parser's shape is now this script's job.

Content problems are now hard failures instead of silent corruption. Previously
a JSON parse error or missing category made the script skip/mislabel a whole
pack ("unknown"). Now rejected with a precise message: parse errors, missing
category object (patch manifest under a production filename), zero questions,
missing id, duplicate id/text, invalid type/access, category_id mismatch,
non-integer depth, choice without options, this_or_that without exactly 2.
All 12 guards verified against mutated fixtures; a valid pack still builds.

Adds --check (validate only) and --out (build elsewhere). Builds are staged to
a temp file, verified, and only then moved into place, backing up the existing
asset db first.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

.gitignore

@@ -103,3 +103,7 @@ revenucat_secret_api.md
 release_guide.md
 seed/questions/closer_question_guides_150_max.zip
 seed/questions/rebuilding_trust_BATCH1_NOTE.md
+
+# Python bytecode from the seed scripts
+__pycache__/
+*.pyc

seed/build_db.py

@@ -1,276 +1,352 @@
 #!/usr/bin/env python3
 """
-Build SQLite database from question JSON files for Room asset loading.
+Build the Room-loadable SQLite asset database from the question JSON packs.
 
-This script reads JSON question files and creates a pre-seeded SQLite database
+The app loads this file via Room's `createFromAsset("database/app.db")` with **no
-that can be bundled with the APK and loaded by Room via createFromAsset().
+migrations and no destructive fallback**, so the output must satisfy Room exactly:
+
+  * every entity table declared by `AppDatabase` must exist, with Room's exact SQL
+  * `room_master_table` must carry Room's identity hash for the schema version
+
+Both of those come from Room's own exported schema
+(`app/schemas/app.closer.data.local.AppDatabase/<version>.json`, produced by
+`room.schemaLocation` in app/build.gradle.kts). That file is the single source of
+truth here — when the Room schema changes, re-run the app build to re-export it and
+this script follows automatically. Nothing about the schema is hardcoded.
+
+`answer_config` is written in the shape the app's parser actually reads
+(`data/local/mapper/QuestionMapper.kt` -> `parseAnswerConfig`), which uses camelCase
+keys. The authoring JSON uses snake_case (see seed/questions/QUESTION_SCHEMA.md);
+translating between the two is this script's job.
+
+Usage:
+    python3 seed/build_db.py                  # build + replace the asset db (backs up first)
+    python3 seed/build_db.py --out /tmp/x.db  # build somewhere else (no backup, no replace)
+    python3 seed/build_db.py --check          # validate the JSON only, build nothing
+
+Any content problem is a hard failure. This script never writes a partial database.
 """
 
+import argparse
 import json
+import shutil
 import sqlite3
-import os
+import sys
+import tempfile
 from pathlib import Path
-from typing import Dict, List, Any
+from typing import Any, Dict, List
 
 from validate_question_variety import load_json_records, validate_records
 
+ROOT = Path(__file__).resolve().parents[1]
+JSON_DIR = ROOT / "seed" / "questions"
+SCHEMA_DIR = ROOT / "app" / "schemas" / "app.closer.data.local.AppDatabase"
+ASSET_DB = ROOT / "app" / "src" / "main" / "assets" / "database" / "app.db"
 
-def load_json_file(filepath: str) -> Dict[str, Any]:
+VALID_TYPES = {"written", "single_choice", "multi_choice", "scale", "this_or_that"}
-    """Load and parse a JSON file."""
+VALID_ACCESS = {"free", "premium"}
-    with open(filepath, 'r', encoding='utf-8') as f:
+CHOICE_TYPES = {"single_choice", "multi_choice"}
-        return json.load(f)
 
-
+# Room writes its identity row with this fixed id; see RoomOpenHelper.
-def get_category_id_from_filename(filename: str) -> str:
+ROOM_MASTER_TABLE_ID = 42
-    """Extract category id from filename."""
+ROOM_MASTER_DDL = (
-    # Handle both v1 and v2 filenames
+    "CREATE TABLE IF NOT EXISTS room_master_table "
-    # examples: questions_communication---...json, questions_communication_v2---...json
+    "(id INTEGER PRIMARY KEY, identity_hash TEXT)"
-    basename = os.path.basename(filename)
-    if '_v2---' in basename:
-        return basename.split('_v2---')[0].replace('questions_', '')
-    elif '---' in basename:
-        return basename.split('---')[0].replace('questions_', '')
-    return 'unknown'
-
-
-def build_database(json_dir: str, output_path: str) -> None:
-    """Build SQLite database from JSON files."""
-    variety_errors = validate_records(load_json_records(Path(json_dir)), "JSON")
-    if variety_errors:
-        raise ValueError(
-            "Question variety check failed:\n" + "\n".join(variety_errors)
 )
 
-    # Ensure output directory exists
-    os.makedirs(os.path.dirname(output_path), exist_ok=True)
 
-    # Remove existing database if present
+class ContentError(Exception):
-    if os.path.exists(output_path):
+    """A problem in the seed JSON. Always fatal — never import partial content."""
-        os.remove(output_path)
 
-    # Connect to new database
-    conn = sqlite3.connect(output_path)
-    cursor = conn.cursor()
 
-    # Create tables with Room-compatible schema
+# --------------------------------------------------------------------------
-    # Question table
+# Room schema
-    cursor.execute('''
+# --------------------------------------------------------------------------
-        CREATE TABLE Question (
+
-            id TEXT NOT NULL,
+def load_room_schema() -> Dict[str, Any]:
-            text TEXT NOT NULL,
+    """Load Room's exported schema (highest version present)."""
-            category_id TEXT NOT NULL,
+    if not SCHEMA_DIR.is_dir():
-            depth_level INTEGER NOT NULL,
+        raise ContentError(
-            is_premium INTEGER NOT NULL,
+            f"Room schema export not found at {SCHEMA_DIR}.\n"
-            type TEXT NOT NULL,
+            "Build the app once (room.schemaLocation is configured) so Room exports it."
-            tags TEXT NOT NULL,
-            answer_config TEXT NOT NULL,
-            pack_id TEXT,
-            created_at INTEGER NOT NULL,
-            status TEXT NOT NULL,
-            sex TEXT,
-            PRIMARY KEY (id)
         )
-    ''')
+    versions = []
-    
+    for path in SCHEMA_DIR.glob("*.json"):
-    # QuestionCategory table
-    cursor.execute('''
-        CREATE TABLE QuestionCategory (
-            id TEXT NOT NULL,
-            display_name TEXT NOT NULL,
-            description TEXT NOT NULL,
-            access TEXT NOT NULL,
-            icon_name TEXT NOT NULL,
-            PRIMARY KEY (id)
-        )
-    ''')
-    
-    # Create indexes
-    cursor.execute('''
-        CREATE INDEX idx_question_category_id ON Question(category_id)
-    ''')
-    
-    # Process each JSON file (support both prefixed and clean filenames)
-    json_files = list(Path(json_dir).glob('*.json'))
-    
-    total_questions = 0
-    categories_processed = set()
-    
-    for json_file in json_files:
-        print(f"\nProcessing: {json_file.name}")
-        
         try:
-            data = load_json_file(str(json_file))
+            versions.append((int(path.stem), path))
+        except ValueError:
+            continue
+    if not versions:
+        raise ContentError(f"No exported schema json in {SCHEMA_DIR}")
+    _, path = max(versions)
+    return json.loads(path.read_text(encoding="utf-8"))["database"]
+
+
+def create_schema(cursor: sqlite3.Cursor, db_schema: Dict[str, Any]) -> None:
+    """Create every Room entity table + the identity row, exactly as Room expects."""
+    for entity in db_schema["entities"]:
+        table = entity["tableName"]
+        cursor.execute(entity["createSql"].replace("${TABLE_NAME}", table))
+        for index in entity.get("indices", []):
+            cursor.execute(index["createSql"].replace("${TABLE_NAME}", table))
+
+    for view in db_schema.get("views", []):
+        cursor.execute(view["createSql"].replace("${VIEW_NAME}", view["viewName"]))
+
+    cursor.execute(ROOM_MASTER_DDL)
+    cursor.execute(
+        "INSERT OR REPLACE INTO room_master_table (id, identity_hash) VALUES (?, ?)",
+        (ROOM_MASTER_TABLE_ID, db_schema["identityHash"]),
+    )
+
+
+# --------------------------------------------------------------------------
+# answer_config: authoring shape (snake_case) -> app parser shape (camelCase)
+# --------------------------------------------------------------------------
+
+def _options(question: Dict[str, Any]) -> List[Dict[str, str]]:
+    """Options live top-level per the schema guide, mirrored into answer_config."""
+    opts = question.get("options") or (question.get("answer_config") or {}).get("options")
+    return opts or []
+
+
+def build_answer_config(question: Dict[str, Any], where: str) -> Dict[str, Any]:
+    """Produce the answer_config the app's QuestionMapper can actually read."""
+    qtype = question["type"]
+    src = question.get("answer_config") or {}
+
+    if qtype == "written":
+        return {
+            "type": "written",
+            "config": {
+                "minLength": src.get("min_length", 1),
+                "maxLength": src.get("max_length", 1000),
+                "placeholder": src.get("placeholder", "Write your answer..."),
+            },
+        }
+
+    if qtype in CHOICE_TYPES:
+        opts = _options(question)
+        if not opts:
+            raise ContentError(f"{where}: {qtype} has no options")
+        config: Dict[str, Any] = {"options": [{"id": o["id"], "text": o["text"]} for o in opts]}
+        if qtype == "multi_choice":
+            config["maxSelections"] = src.get("max_selections", 0)
+        return {"type": qtype, "config": config}
+
+    if qtype == "scale":
+        return {
+            "type": "scale",
+            "config": {
+                "minScale": src.get("min", 1),
+                "maxScale": src.get("max", 5),
+                "minLabel": src.get("min_label", ""),
+                "maxLabel": src.get("max_label", ""),
+                "scaleStep": src.get("scale_step", 1),
+            },
+        }
+
+    if qtype == "this_or_that":
+        opts = _options(question)
+        if len(opts) != 2:
+            raise ContentError(f"{where}: this_or_that needs exactly 2 options, found {len(opts)}")
+        return {
+            "type": "this_or_that",
+            "config": {
+                "optionA": {"id": opts[0]["id"], "text": opts[0]["text"]},
+                "optionB": {"id": opts[1]["id"], "text": opts[1]["text"]},
+            },
+        }
+
+    raise ContentError(f"{where}: unsupported type {qtype!r}")
+
+
+# --------------------------------------------------------------------------
+# Validation — every problem is fatal
+# --------------------------------------------------------------------------
+
+def load_packs() -> List[Dict[str, Any]]:
+    """Load and hard-validate every pack. Raises on the first structural problem."""
+    files = sorted(JSON_DIR.glob("*.json"))
+    if not files:
+        raise ContentError(f"No question JSON found in {JSON_DIR}")
+
+    packs, seen_ids = [], {}
+    for path in files:
+        name = path.name
+        try:
+            data = json.loads(path.read_text(encoding="utf-8"))
         except json.JSONDecodeError as e:
-            print(f"  ❌ Invalid JSON: {e}")
+            raise ContentError(f"{name}: invalid JSON: {e}") from e
-            continue
-        except Exception as e:
-            print(f"  ❌ Error reading file: {e}")
-            continue
 
-        # Extract category info
+        category = data.get("category")
-        category_data = data.get('category', {})
+        if not isinstance(category, dict):
-        category_id = category_data.get('id', get_category_id_from_filename(str(json_file)))
+            raise ContentError(
-        category_display_name = category_data.get('display_name', category_id)
+                f"{name}: missing top-level 'category' object. "
-        category_description = category_data.get('description', '')
+                "A patch manifest or partial batch must never sit under a production filename."
-        category_access = category_data.get('access', 'free')
+            )
-        category_icon = category_data.get('icon_name', 'question')
+        for field in ("id", "display_name", "description", "access", "icon_name"):
+            if not category.get(field):
+                raise ContentError(f"{name}: category.{field} is missing or empty")
 
-        # Insert category (ignore duplicates)
+        questions = data.get("questions")
-        cursor.execute('''
+        if not isinstance(questions, list) or not questions:
-            INSERT OR IGNORE INTO QuestionCategory 
+            raise ContentError(f"{name}: 'questions' is missing or empty")
-            (id, display_name, description, access, icon_name)
-            VALUES (?, ?, ?, ?, ?)
-        ''', (category_id, category_display_name, category_description, category_access, category_icon))
 
-        if category_id not in categories_processed:
+        cid = category["id"]
-            categories_processed.add(category_id)
+        for i, q in enumerate(questions):
-            print(f"  Category: {category_display_name} (id: {category_id})")
+            qid = q.get("id")
+            where = f"{name}:{qid or f'#{i}'}"
+            if not qid:
+                raise ContentError(f"{where}: question has no id")
+            if qid in seen_ids:
+                raise ContentError(f"{where}: duplicate id (also in {seen_ids[qid]})")
+            seen_ids[qid] = name
+            if not q.get("text"):
+                raise ContentError(f"{where}: empty text")
+            if q.get("type") not in VALID_TYPES:
+                raise ContentError(f"{where}: invalid type {q.get('type')!r}")
+            if q.get("access") not in VALID_ACCESS:
+                raise ContentError(f"{where}: invalid access {q.get('access')!r}")
+            if q.get("category_id") != cid:
+                raise ContentError(
+                    f"{where}: category_id {q.get('category_id')!r} != category.id {cid!r}"
+                )
+            depth = q.get("depth", q.get("depth_level"))
+            if not isinstance(depth, int) or isinstance(depth, bool):
+                raise ContentError(
+                    f"{where}: depth {depth!r} is not an integer. "
+                    "String depth is a future migration and is not production-ready "
+                    "(see QUESTION_SCHEMA.md); Room reads depth_level as an integer."
+                )
+            if not isinstance(q.get("tags"), list):
+                raise ContentError(f"{where}: tags must be an array")
+            # Surfaces option problems now rather than as an unusable question later.
+            build_answer_config(q, where)
 
-        # Insert questions
+        packs.append({"name": name, "category": category, "questions": questions,
-        questions = data.get('questions', [])
+                      "mtime": int(path.stat().st_mtime)})
-        questions_inserted = 0
 
-        for q in questions:
+    errors = validate_records(load_json_records(JSON_DIR), "JSON")
-            question_id = q.get('id')
+    if errors:
-            text = q.get('text', '')
+        raise ContentError("Catalog variety gate failed:\n" + "\n".join(errors))
-            category_id_q = q.get('category_id', category_id)
-            depth_level = q.get('depth', q.get('depth_level', 1))
-            is_premium = q.get('access', 'free') == 'premium'
-            question_type = q.get('type', 'written')
-            tags = q.get('tags', [])
 
-            # Handle answer_config - check both patterns
+    return packs
-            answer_config = q.get('answer_config', {})
-            options = q.get('options', [])  # Alternative location for choice types
 
-            # Build answer_config JSON
-            ac_json = {}
 
-            if question_type == 'written':
+# --------------------------------------------------------------------------
-                ac_json = {
+# Build
-                    'type': 'written',
+# --------------------------------------------------------------------------
-                    'config': answer_config if answer_config else {
-                        'minLength': 1,
-                        'maxLength': 1000,
-                        'placeholder': 'Write your answer...'
-                    }
-                }
-            elif question_type == 'single_choice':
-                if options:
-                    ac_json = {
-                        'type': 'single_choice',
-                        'config': {'options': options}
-                    }
-                elif answer_config:
-                    ac_json = {
-                        'type': 'single_choice',
-                        'config': answer_config
-                    }
-            elif question_type == 'multi_choice':
-                if options:
-                    ac_json = {
-                        'type': 'multi_choice',
-                        'config': {'options': options}
-                    }
-                elif answer_config:
-                    ac_json = {
-                        'type': 'multi_choice',
-                        'config': answer_config
-                    }
-            elif question_type == 'scale':
-                ac_json = {
-                    'type': 'scale',
-                    'config': answer_config if answer_config else {
-                        'minScale': 1,
-                        'maxScale': 5,
-                        'minLabel': 'Disagree',
-                        'maxLabel': 'Agree'
-                    }
-                }
-            elif question_type == 'this_or_that':
-                if options:
-                    ac_json = {
-                        'type': 'this_or_that',
-                        'config': {
-                            'optionA': options[0] if len(options) > 0 else {'id': 'a', 'text': ''},
-                            'optionB': options[1] if len(options) > 1 else {'id': 'b', 'text': ''}
-                        }
-                    }
-                elif answer_config:
-                    ac_json = {
-                        'type': 'this_or_that',
-                        'config': answer_config
-                    }
 
-            # Convert tags and answer_config to JSON strings
+def build(packs: List[Dict[str, Any]], db_schema: Dict[str, Any], out: Path) -> Dict[str, int]:
-            tags_json = json.dumps(tags, separators=(',', ':'))
+    out.parent.mkdir(parents=True, exist_ok=True)
-            ac_json_str = json.dumps(ac_json, separators=(',', ':'))
+    if out.exists():
+        out.unlink()
 
-            # Extract optional sex field (used for Desire Sync filtering)
+    conn = sqlite3.connect(out)
-            sex = q.get('sex')
+    try:
+        cur = conn.cursor()
+        create_schema(cur, db_schema)
 
-            # Insert question
+        total = 0
-            cursor.execute('''
+        for pack in packs:
-                INSERT OR REPLACE INTO Question 
+            c = pack["category"]
-                (id, text, category_id, depth_level, is_premium, type, tags, answer_config, 
+            cur.execute(
-                 created_at, status, sex)
+                "INSERT OR REPLACE INTO question_category "
-                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                "(id, display_name, description, access, icon_name) VALUES (?, ?, ?, ?, ?)",
-            ''', (
+                (c["id"], c["display_name"], c["description"], c["access"], c["icon_name"]),
-                question_id,
+            )
-                text,
+            for q in pack["questions"]:
-                category_id_q,
+                where = f"{pack['name']}:{q['id']}"
-                depth_level,
+                cur.execute(
-                1 if is_premium else 0,
+                    "INSERT OR REPLACE INTO question "
-                question_type,
+                    "(id, text, category_id, depth_level, is_premium, type, tags, answer_config,"
-                tags_json,
+                    " pack_id, created_at, status, sex) "
-                ac_json_str,
+                    "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
-                int(json_file.stat().st_mtime),  # Use file mtime as created_at
+                    (
-                'active',
+                        q["id"],
-                sex
+                        q["text"],
-            ))
+                        q["category_id"],
-            
+                        q.get("depth", q.get("depth_level")),
-            questions_inserted += 1
+                        1 if q["access"] == "premium" else 0,
-            total_questions += 1
+                        q["type"],
-        
+                        json.dumps(q.get("tags", []), separators=(",", ":")),
-        free_count = sum(1 for q in questions if q.get('access', 'free') == 'free')
+                        json.dumps(build_answer_config(q, where), separators=(",", ":")),
-        premium_count = sum(1 for q in questions if q.get('access', 'free') == 'premium')
+                        None,
-        print(f"  Questions: {questions_inserted} ({free_count} free, {premium_count} premium)")
+                        pack["mtime"],
-    
+                        "active",
-    # Commit and close
+                        q.get("sex"),
+                    ),
+                )
+                total += 1
         conn.commit()
+    finally:
+        conn.close()
+    return {"categories": len(packs), "questions": total}
+
+
+def verify(out: Path, db_schema: Dict[str, Any]) -> None:
+    """Re-open the built file and prove it satisfies Room before it is shipped."""
+    conn = sqlite3.connect(out)
+    try:
+        tables = {r[0] for r in conn.execute(
+            "SELECT name FROM sqlite_master WHERE type='table'")}
+        expected = {e["tableName"] for e in db_schema["entities"]} | {"room_master_table"}
+        missing = expected - tables
+        if missing:
+            raise ContentError(f"built db is missing Room tables: {sorted(missing)}")
+
+        row = conn.execute(
+            "SELECT id, identity_hash FROM room_master_table").fetchone()
+        if not row or row[0] != ROOM_MASTER_TABLE_ID or row[1] != db_schema["identityHash"]:
+            raise ContentError(f"identity hash mismatch: {row} != {db_schema['identityHash']}")
+
+        empty = [t for t in ("question", "question_category")
+                 if conn.execute(f"SELECT COUNT(*) FROM {t}").fetchone()[0] == 0]
+        if empty:
+            raise ContentError(f"built db has empty tables: {empty}")
+    finally:
         conn.close()
 
-    print(f"\n{'='*60}")
+
-    print(f"Database built: {output_path}")
+def main() -> int:
-    print(f"Categories: {len(categories_processed)}")
+    ap = argparse.ArgumentParser(description=__doc__,
-    print(f"Total questions: {total_questions}")
+                                 formatter_class=argparse.RawDescriptionHelpFormatter)
-    print(f"{'='*60}")
+    ap.add_argument("--out", type=Path, default=ASSET_DB,
+                    help=f"output path (default: {ASSET_DB})")
+    ap.add_argument("--check", action="store_true",
+                    help="validate the JSON and exit without building")
+    args = ap.parse_args()
+
+    try:
+        db_schema = load_room_schema()
+        packs = load_packs()
+        print(f"Validated {len(packs)} packs, "
+              f"{sum(len(p['questions']) for p in packs)} questions — catalog gate passed.")
+        if args.check:
+            return 0
+
+        replacing_asset = args.out.resolve() == ASSET_DB.resolve()
+        with tempfile.TemporaryDirectory() as tmp:
+            staged = Path(tmp) / "app.db"
+            stats = build(packs, db_schema, staged)
+            verify(staged, db_schema)
+
+            if replacing_asset and args.out.exists():
+                backup = args.out.with_suffix(".db.bak")
+                shutil.copy2(args.out, backup)
+                print(f"Backed up existing asset db -> {backup.name}")
+            args.out.parent.mkdir(parents=True, exist_ok=True)
+            shutil.copy2(staged, args.out)
+    except ContentError as e:
+        print(f"\nBUILD FAILED\n{e}", file=sys.stderr)
+        return 1
+
+    print(f"Built {args.out}")
+    print(f"  schema version {db_schema['version']} · identity {db_schema['identityHash']}")
+    print(f"  {stats['categories']} categories · {stats['questions']} questions")
+    return 0
 
 
-def main():
+if __name__ == "__main__":
-    """Main entry point."""
+    sys.exit(main())
-    # Paths
-    script_dir = Path(__file__).parent
-    json_dir = script_dir / 'questions'
-    output_dir = script_dir.parent / 'app' / 'src' / 'main' / 'assets' / 'database'
-    output_path = output_dir / 'app.db'
-    
-    print("Building SQLite database from question JSON files...")
-    print(f"JSON directory: {json_dir}")
-    print(f"Output path: {output_path}")
-    
-    # Verify JSON directory exists
-    if not json_dir.exists():
-        print(f"❌ JSON directory not found: {json_dir}")
-        return
-    
-    if not list(json_dir.glob('*.json')):
-        print(f"❌ No question JSON files found in: {json_dir}")
-        return
-    
-    # Build database
-    build_database(str(json_dir), str(output_path))
-
-
-if __name__ == '__main__':
-    main()