Add one-off participant import script for legacy MongoDB data
Parses the raw MongoDB shell/Compass export (comments + ObjectId(...) calls, not valid JSON), maps navn/email/telefon/afmeldt to the new Participant model, filters out known test rows, and upserts on (route_id, lowercased email) so re-running is safe. Excludes the raw export (old_participants.json) from version control since it contains real participants' personal data. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
parent
d2abe70333
commit
5a309c8aca
2 changed files with 172 additions and 0 deletions
3
.gitignore
vendored
3
.gitignore
vendored
|
|
@ -22,3 +22,6 @@ dist/
|
|||
|
||||
# Test cache
|
||||
.pytest_cache/
|
||||
|
||||
# Raw personal-data exports used for one-off migrations — not for version control
|
||||
old_participants.json
|
||||
|
|
|
|||
169
scripts/import_participants.py
Normal file
169
scripts/import_participants.py
Normal file
|
|
@ -0,0 +1,169 @@
|
|||
"""One-off migration: import participants from a legacy MongoDB export
|
||||
into the new PostgreSQL Participant table.
|
||||
|
||||
Usage:
|
||||
uv run python scripts/import_participants.py <path-to-export.json>
|
||||
|
||||
The source file is a raw MongoDB shell/Compass dump, not valid JSON:
|
||||
it contains `/* N */` comment markers between records and `ObjectId("...")`
|
||||
function calls instead of plain strings. This script cleans that up
|
||||
before parsing.
|
||||
|
||||
Safe to re-run: participants are upserted on (route_id, lowercased email).
|
||||
"""
|
||||
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
from sqlmodel import Session, select
|
||||
|
||||
from app.db import engine
|
||||
from app.models.organization import Organization
|
||||
from app.models.participant import Participant
|
||||
from app.models.route import Route
|
||||
|
||||
ORGANIZATION_NAME = "Fælles Vinindkøb"
|
||||
ROUTE_NAME = "Fælles Vinindkøb"
|
||||
|
||||
OBJECT_ID_RE = re.compile(r'ObjectId\("([0-9a-fA-F]+)"\)')
|
||||
COMMENT_RE = re.compile(r"/\*.*?\*/", re.DOTALL)
|
||||
|
||||
|
||||
def load_mongo_dump(path: Path) -> list[dict]:
|
||||
"""Parse a raw mongo shell/Compass export (not valid JSON) into a
|
||||
list of plain dicts."""
|
||||
text = path.read_text(encoding="utf-8")
|
||||
text = COMMENT_RE.sub("", text)
|
||||
text = OBJECT_ID_RE.sub(r'"\1"', text)
|
||||
|
||||
decoder = json.JSONDecoder()
|
||||
records = []
|
||||
idx = 0
|
||||
length = len(text)
|
||||
while idx < length:
|
||||
while idx < length and text[idx].isspace():
|
||||
idx += 1
|
||||
if idx >= length:
|
||||
break
|
||||
obj, end = decoder.raw_decode(text, idx)
|
||||
records.append(obj)
|
||||
idx = end
|
||||
return records
|
||||
|
||||
|
||||
def is_test_row(name: str, email: str) -> bool:
|
||||
if email.endswith("@mail-tester.com"):
|
||||
return True
|
||||
if name.strip().lower().startswith("carsten test"):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def map_participant(raw: dict) -> dict | None:
|
||||
name = (raw.get("navn") or "").strip()
|
||||
email = (raw.get("email") or "").strip().lower()
|
||||
if not email:
|
||||
return None
|
||||
if is_test_row(name, email):
|
||||
return None
|
||||
|
||||
phone = (raw.get("telefon") or "").strip() or None
|
||||
afmeldt = bool(raw.get("afmeldt") or raw.get("afmldt") or False)
|
||||
|
||||
return {
|
||||
"name": name,
|
||||
"email": email,
|
||||
"phone": phone,
|
||||
"is_active": not afmeldt,
|
||||
}
|
||||
|
||||
|
||||
def get_or_create_organization(session: Session, name: str) -> Organization:
|
||||
org = session.exec(select(Organization).where(Organization.name == name)).first()
|
||||
if org is None:
|
||||
org = Organization(name=name)
|
||||
session.add(org)
|
||||
session.flush()
|
||||
return org
|
||||
|
||||
|
||||
def get_or_create_route(session: Session, name: str, organization_id: int) -> Route:
|
||||
route = session.exec(select(Route).where(Route.name == name)).first()
|
||||
if route is None:
|
||||
route = Route(name=name, organization_id=organization_id)
|
||||
session.add(route)
|
||||
session.flush()
|
||||
return route
|
||||
|
||||
|
||||
def upsert_participant(session: Session, route_id: int, mapped: dict) -> str:
|
||||
existing = session.exec(
|
||||
select(Participant).where(
|
||||
Participant.route_id == route_id,
|
||||
Participant.email == mapped["email"],
|
||||
)
|
||||
).first()
|
||||
if existing is None:
|
||||
session.add(Participant(route_id=route_id, **mapped))
|
||||
return "created"
|
||||
|
||||
existing.name = mapped["name"]
|
||||
existing.phone = mapped["phone"]
|
||||
existing.is_active = mapped["is_active"]
|
||||
session.add(existing)
|
||||
return "updated"
|
||||
|
||||
|
||||
def main() -> None:
|
||||
if len(sys.argv) != 2:
|
||||
print("Usage: uv run python scripts/import_participants.py <path-to-export.json>")
|
||||
sys.exit(1)
|
||||
|
||||
source_path = Path(sys.argv[1])
|
||||
raw_records = load_mongo_dump(source_path)
|
||||
|
||||
counts = defaultdict(int)
|
||||
names_to_emails: dict[str, set[str]] = defaultdict(set)
|
||||
|
||||
with Session(engine) as session:
|
||||
organization = get_or_create_organization(session, ORGANIZATION_NAME)
|
||||
route = get_or_create_route(session, ROUTE_NAME, organization.id)
|
||||
session.flush()
|
||||
organization_id = organization.id
|
||||
route_id = route.id
|
||||
|
||||
for raw in raw_records:
|
||||
mapped = map_participant(raw)
|
||||
if mapped is None:
|
||||
counts["skipped_test_or_no_email"] += 1
|
||||
continue
|
||||
|
||||
outcome = upsert_participant(session, route_id, mapped)
|
||||
counts[outcome] += 1
|
||||
names_to_emails[mapped["name"].strip().lower()].add(mapped["email"])
|
||||
|
||||
session.commit()
|
||||
|
||||
print(f"Organization: {ORGANIZATION_NAME!r} (id={organization_id})")
|
||||
print(f"Route: {ROUTE_NAME!r} (id={route_id})")
|
||||
print(f"Source records: {len(raw_records)}")
|
||||
print(f"Created: {counts['created']}")
|
||||
print(f"Updated: {counts['updated']}")
|
||||
print(f"Skipped (test/no email): {counts['skipped_test_or_no_email']}")
|
||||
|
||||
duplicates = {name: emails for name, emails in names_to_emails.items() if len(emails) > 1}
|
||||
if duplicates:
|
||||
print(f"\nPossible duplicates (same name, different emails) — {len(duplicates)} name(s):")
|
||||
for name, emails in sorted(duplicates.items()):
|
||||
print(f" - {name}: {', '.join(sorted(emails))}")
|
||||
else:
|
||||
print("\nNo same-name/different-email duplicates detected.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
Add table
Reference in a new issue