"""One-off migration: import participants from a legacy MongoDB export into the new PostgreSQL Participant table. Usage: uv run python scripts/import_participants.py The source file is a raw MongoDB shell/Compass dump, not valid JSON: it contains `/* N */` comment markers between records and `ObjectId("...")` function calls instead of plain strings. This script cleans that up before parsing. Safe to re-run: participants are upserted on (route_id, lowercased email). """ import json import re import sys from collections import defaultdict from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) from sqlmodel import Session, select from app.db import engine from app.models.organization import Organization from app.models.participant import Participant from app.models.route import Route ORGANIZATION_NAME = "Fælles Vinindkøb" ROUTE_NAME = "Fælles Vinindkøb" OBJECT_ID_RE = re.compile(r'ObjectId\("([0-9a-fA-F]+)"\)') COMMENT_RE = re.compile(r"/\*.*?\*/", re.DOTALL) def load_mongo_dump(path: Path) -> list[dict]: """Parse a raw mongo shell/Compass export (not valid JSON) into a list of plain dicts.""" text = path.read_text(encoding="utf-8") text = COMMENT_RE.sub("", text) text = OBJECT_ID_RE.sub(r'"\1"', text) decoder = json.JSONDecoder() records = [] idx = 0 length = len(text) while idx < length: while idx < length and text[idx].isspace(): idx += 1 if idx >= length: break obj, end = decoder.raw_decode(text, idx) records.append(obj) idx = end return records def is_test_row(name: str, email: str) -> bool: if email.endswith("@mail-tester.com"): return True if name.strip().lower().startswith("carsten test"): return True return False def map_participant(raw: dict) -> dict | None: name = (raw.get("navn") or "").strip() email = (raw.get("email") or "").strip().lower() if not email: return None if is_test_row(name, email): return None phone = (raw.get("telefon") or "").strip() or None afmeldt = bool(raw.get("afmeldt") or raw.get("afmldt") or False) return { "name": name, "email": email, "phone": phone, "is_active": not afmeldt, } def get_or_create_organization(session: Session, name: str) -> Organization: org = session.exec(select(Organization).where(Organization.name == name)).first() if org is None: org = Organization(name=name) session.add(org) session.flush() return org def get_or_create_route(session: Session, name: str, organization_id: int) -> Route: route = session.exec(select(Route).where(Route.name == name)).first() if route is None: route = Route(name=name, organization_id=organization_id) session.add(route) session.flush() return route def upsert_participant(session: Session, route_id: int, mapped: dict) -> str: existing = session.exec( select(Participant).where( Participant.route_id == route_id, Participant.email == mapped["email"], ) ).first() if existing is None: session.add(Participant(route_id=route_id, **mapped)) return "created" existing.name = mapped["name"] existing.phone = mapped["phone"] existing.is_active = mapped["is_active"] session.add(existing) return "updated" def main() -> None: if len(sys.argv) != 2: print("Usage: uv run python scripts/import_participants.py ") sys.exit(1) source_path = Path(sys.argv[1]) raw_records = load_mongo_dump(source_path) counts = defaultdict(int) names_to_emails: dict[str, set[str]] = defaultdict(set) with Session(engine) as session: organization = get_or_create_organization(session, ORGANIZATION_NAME) route = get_or_create_route(session, ROUTE_NAME, organization.id) session.flush() organization_id = organization.id route_id = route.id for raw in raw_records: mapped = map_participant(raw) if mapped is None: counts["skipped_test_or_no_email"] += 1 continue outcome = upsert_participant(session, route_id, mapped) counts[outcome] += 1 names_to_emails[mapped["name"].strip().lower()].add(mapped["email"]) session.commit() print(f"Organization: {ORGANIZATION_NAME!r} (id={organization_id})") print(f"Route: {ROUTE_NAME!r} (id={route_id})") print(f"Source records: {len(raw_records)}") print(f"Created: {counts['created']}") print(f"Updated: {counts['updated']}") print(f"Skipped (test/no email): {counts['skipped_test_or_no_email']}") duplicates = {name: emails for name, emails in names_to_emails.items() if len(emails) > 1} if duplicates: print(f"\nPossible duplicates (same name, different emails) — {len(duplicates)} name(s):") for name, emails in sorted(duplicates.items()): print(f" - {name}: {', '.join(sorted(emails))}") else: print("\nNo same-name/different-email duplicates detected.") if __name__ == "__main__": main()