From 5a309c8acad7647858d490bfcf71b2e4938a728c Mon Sep 17 00:00:00 2001 From: carsten Date: Sun, 27 Sep 2026 19:16:26 +0200 Subject: [PATCH] Add one-off participant import script for legacy MongoDB data Parses the raw MongoDB shell/Compass export (comments + ObjectId(...) calls, not valid JSON), maps navn/email/telefon/afmeldt to the new Participant model, filters out known test rows, and upserts on (route_id, lowercased email) so re-running is safe. Excludes the raw export (old_participants.json) from version control since it contains real participants' personal data. Co-Authored-By: Claude Sonnet 5 --- .gitignore | 3 + scripts/import_participants.py | 169 +++++++++++++++++++++++++++++++++ 2 files changed, 172 insertions(+) create mode 100644 scripts/import_participants.py diff --git a/.gitignore b/.gitignore index f60a399..e5139f0 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,6 @@ dist/ # Test cache .pytest_cache/ + +# Raw personal-data exports used for one-off migrations — not for version control +old_participants.json diff --git a/scripts/import_participants.py b/scripts/import_participants.py new file mode 100644 index 0000000..4ce9470 --- /dev/null +++ b/scripts/import_participants.py @@ -0,0 +1,169 @@ +"""One-off migration: import participants from a legacy MongoDB export +into the new PostgreSQL Participant table. + +Usage: + uv run python scripts/import_participants.py + +The source file is a raw MongoDB shell/Compass dump, not valid JSON: +it contains `/* N */` comment markers between records and `ObjectId("...")` +function calls instead of plain strings. This script cleans that up +before parsing. + +Safe to re-run: participants are upserted on (route_id, lowercased email). +""" + +import json +import re +import sys +from collections import defaultdict +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from sqlmodel import Session, select + +from app.db import engine +from app.models.organization import Organization +from app.models.participant import Participant +from app.models.route import Route + +ORGANIZATION_NAME = "Fælles Vinindkøb" +ROUTE_NAME = "Fælles Vinindkøb" + +OBJECT_ID_RE = re.compile(r'ObjectId\("([0-9a-fA-F]+)"\)') +COMMENT_RE = re.compile(r"/\*.*?\*/", re.DOTALL) + + +def load_mongo_dump(path: Path) -> list[dict]: + """Parse a raw mongo shell/Compass export (not valid JSON) into a + list of plain dicts.""" + text = path.read_text(encoding="utf-8") + text = COMMENT_RE.sub("", text) + text = OBJECT_ID_RE.sub(r'"\1"', text) + + decoder = json.JSONDecoder() + records = [] + idx = 0 + length = len(text) + while idx < length: + while idx < length and text[idx].isspace(): + idx += 1 + if idx >= length: + break + obj, end = decoder.raw_decode(text, idx) + records.append(obj) + idx = end + return records + + +def is_test_row(name: str, email: str) -> bool: + if email.endswith("@mail-tester.com"): + return True + if name.strip().lower().startswith("carsten test"): + return True + return False + + +def map_participant(raw: dict) -> dict | None: + name = (raw.get("navn") or "").strip() + email = (raw.get("email") or "").strip().lower() + if not email: + return None + if is_test_row(name, email): + return None + + phone = (raw.get("telefon") or "").strip() or None + afmeldt = bool(raw.get("afmeldt") or raw.get("afmldt") or False) + + return { + "name": name, + "email": email, + "phone": phone, + "is_active": not afmeldt, + } + + +def get_or_create_organization(session: Session, name: str) -> Organization: + org = session.exec(select(Organization).where(Organization.name == name)).first() + if org is None: + org = Organization(name=name) + session.add(org) + session.flush() + return org + + +def get_or_create_route(session: Session, name: str, organization_id: int) -> Route: + route = session.exec(select(Route).where(Route.name == name)).first() + if route is None: + route = Route(name=name, organization_id=organization_id) + session.add(route) + session.flush() + return route + + +def upsert_participant(session: Session, route_id: int, mapped: dict) -> str: + existing = session.exec( + select(Participant).where( + Participant.route_id == route_id, + Participant.email == mapped["email"], + ) + ).first() + if existing is None: + session.add(Participant(route_id=route_id, **mapped)) + return "created" + + existing.name = mapped["name"] + existing.phone = mapped["phone"] + existing.is_active = mapped["is_active"] + session.add(existing) + return "updated" + + +def main() -> None: + if len(sys.argv) != 2: + print("Usage: uv run python scripts/import_participants.py ") + sys.exit(1) + + source_path = Path(sys.argv[1]) + raw_records = load_mongo_dump(source_path) + + counts = defaultdict(int) + names_to_emails: dict[str, set[str]] = defaultdict(set) + + with Session(engine) as session: + organization = get_or_create_organization(session, ORGANIZATION_NAME) + route = get_or_create_route(session, ROUTE_NAME, organization.id) + session.flush() + organization_id = organization.id + route_id = route.id + + for raw in raw_records: + mapped = map_participant(raw) + if mapped is None: + counts["skipped_test_or_no_email"] += 1 + continue + + outcome = upsert_participant(session, route_id, mapped) + counts[outcome] += 1 + names_to_emails[mapped["name"].strip().lower()].add(mapped["email"]) + + session.commit() + + print(f"Organization: {ORGANIZATION_NAME!r} (id={organization_id})") + print(f"Route: {ROUTE_NAME!r} (id={route_id})") + print(f"Source records: {len(raw_records)}") + print(f"Created: {counts['created']}") + print(f"Updated: {counts['updated']}") + print(f"Skipped (test/no email): {counts['skipped_test_or_no_email']}") + + duplicates = {name: emails for name, emails in names_to_emails.items() if len(emails) > 1} + if duplicates: + print(f"\nPossible duplicates (same name, different emails) — {len(duplicates)} name(s):") + for name, emails in sorted(duplicates.items()): + print(f" - {name}: {', '.join(sorted(emails))}") + else: + print("\nNo same-name/different-email duplicates detected.") + + +if __name__ == "__main__": + main()