Add one-off participant import script for legacy MongoDB data

Parses the raw MongoDB shell/Compass export (comments + ObjectId(...)
calls, not valid JSON), maps navn/email/telefon/afmeldt to the new
Participant model, filters out known test rows, and upserts on
(route_id, lowercased email) so re-running is safe. Excludes the raw
export (old_participants.json) from version control since it contains
real participants' personal data.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Carsten Gram 2026-09-27 19:16:26 +02:00
parent d2abe70333
commit 5a309c8aca
2 changed files with 172 additions and 0 deletions

3
.gitignore vendored
View file

@ -22,3 +22,6 @@ dist/
# Test cache # Test cache
.pytest_cache/ .pytest_cache/
# Raw personal-data exports used for one-off migrations — not for version control
old_participants.json

View file

@ -0,0 +1,169 @@
"""One-off migration: import participants from a legacy MongoDB export
into the new PostgreSQL Participant table.
Usage:
uv run python scripts/import_participants.py <path-to-export.json>
The source file is a raw MongoDB shell/Compass dump, not valid JSON:
it contains `/* N */` comment markers between records and `ObjectId("...")`
function calls instead of plain strings. This script cleans that up
before parsing.
Safe to re-run: participants are upserted on (route_id, lowercased email).
"""
import json
import re
import sys
from collections import defaultdict
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from sqlmodel import Session, select
from app.db import engine
from app.models.organization import Organization
from app.models.participant import Participant
from app.models.route import Route
ORGANIZATION_NAME = "Fælles Vinindkøb"
ROUTE_NAME = "Fælles Vinindkøb"
OBJECT_ID_RE = re.compile(r'ObjectId\("([0-9a-fA-F]+)"\)')
COMMENT_RE = re.compile(r"/\*.*?\*/", re.DOTALL)
def load_mongo_dump(path: Path) -> list[dict]:
"""Parse a raw mongo shell/Compass export (not valid JSON) into a
list of plain dicts."""
text = path.read_text(encoding="utf-8")
text = COMMENT_RE.sub("", text)
text = OBJECT_ID_RE.sub(r'"\1"', text)
decoder = json.JSONDecoder()
records = []
idx = 0
length = len(text)
while idx < length:
while idx < length and text[idx].isspace():
idx += 1
if idx >= length:
break
obj, end = decoder.raw_decode(text, idx)
records.append(obj)
idx = end
return records
def is_test_row(name: str, email: str) -> bool:
if email.endswith("@mail-tester.com"):
return True
if name.strip().lower().startswith("carsten test"):
return True
return False
def map_participant(raw: dict) -> dict | None:
name = (raw.get("navn") or "").strip()
email = (raw.get("email") or "").strip().lower()
if not email:
return None
if is_test_row(name, email):
return None
phone = (raw.get("telefon") or "").strip() or None
afmeldt = bool(raw.get("afmeldt") or raw.get("afmldt") or False)
return {
"name": name,
"email": email,
"phone": phone,
"is_active": not afmeldt,
}
def get_or_create_organization(session: Session, name: str) -> Organization:
org = session.exec(select(Organization).where(Organization.name == name)).first()
if org is None:
org = Organization(name=name)
session.add(org)
session.flush()
return org
def get_or_create_route(session: Session, name: str, organization_id: int) -> Route:
route = session.exec(select(Route).where(Route.name == name)).first()
if route is None:
route = Route(name=name, organization_id=organization_id)
session.add(route)
session.flush()
return route
def upsert_participant(session: Session, route_id: int, mapped: dict) -> str:
existing = session.exec(
select(Participant).where(
Participant.route_id == route_id,
Participant.email == mapped["email"],
)
).first()
if existing is None:
session.add(Participant(route_id=route_id, **mapped))
return "created"
existing.name = mapped["name"]
existing.phone = mapped["phone"]
existing.is_active = mapped["is_active"]
session.add(existing)
return "updated"
def main() -> None:
if len(sys.argv) != 2:
print("Usage: uv run python scripts/import_participants.py <path-to-export.json>")
sys.exit(1)
source_path = Path(sys.argv[1])
raw_records = load_mongo_dump(source_path)
counts = defaultdict(int)
names_to_emails: dict[str, set[str]] = defaultdict(set)
with Session(engine) as session:
organization = get_or_create_organization(session, ORGANIZATION_NAME)
route = get_or_create_route(session, ROUTE_NAME, organization.id)
session.flush()
organization_id = organization.id
route_id = route.id
for raw in raw_records:
mapped = map_participant(raw)
if mapped is None:
counts["skipped_test_or_no_email"] += 1
continue
outcome = upsert_participant(session, route_id, mapped)
counts[outcome] += 1
names_to_emails[mapped["name"].strip().lower()].add(mapped["email"])
session.commit()
print(f"Organization: {ORGANIZATION_NAME!r} (id={organization_id})")
print(f"Route: {ROUTE_NAME!r} (id={route_id})")
print(f"Source records: {len(raw_records)}")
print(f"Created: {counts['created']}")
print(f"Updated: {counts['updated']}")
print(f"Skipped (test/no email): {counts['skipped_test_or_no_email']}")
duplicates = {name: emails for name, emails in names_to_emails.items() if len(emails) > 1}
if duplicates:
print(f"\nPossible duplicates (same name, different emails) — {len(duplicates)} name(s):")
for name, emails in sorted(duplicates.items()):
print(f" - {name}: {', '.join(sorted(emails))}")
else:
print("\nNo same-name/different-email duplicates detected.")
if __name__ == "__main__":
main()