Files
Trey t 63acf7accb feat: add Django web app, CloudKit sync, dashboard, and game_datetime_utc export
Adds the full Django application layer on top of sportstime_parser:
- core: Sport, Team, Stadium, Game models with aliases and league structure
- scraper: orchestration engine, adapter, job management, Celery tasks
- cloudkit: CloudKit sync client, sync state tracking, sync jobs
- dashboard: staff dashboard for monitoring scrapers, sync, review queue
- notifications: email reports for scrape/sync results
- Docker setup for deployment (Dockerfile, docker-compose, entrypoint)

Game exports now use game_datetime_utc (ISO 8601 UTC) instead of
venue-local date+time strings, matching the canonical format used
by the iOS app.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-19 14:04:27 -06:00

744 lines
26 KiB
Python

"""Record differ for CloudKit uploads.
This module compares local records with CloudKit records to determine
what needs to be created, updated, or deleted.
Field names must match CKModels.swift exactly:
- Stadium: stadiumId, canonicalId, name, city, state, location (CLLocation),
capacity, yearOpened, imageURL, sport
- Team: teamId, canonicalId, name, abbreviation, sport, city, stadiumCanonicalId,
logoURL, primaryColor, secondaryColor
- Game: gameId, canonicalId, homeTeamCanonicalId, awayTeamCanonicalId,
stadiumCanonicalId, dateTime, sport, season, isPlayoff, broadcastInfo
- TeamAlias: aliasId, teamCanonicalId, aliasType, aliasValue, validFrom, validUntil
- StadiumAlias: aliasName, stadiumCanonicalId, validFrom, validUntil
- Sport: sportId, abbreviation, displayName, iconName, colorHex,
seasonStartMonth, seasonEndMonth, isActive
- LeagueStructure: structureId, sport, type, name, abbreviation, parentId, displayOrder
"""
from dataclasses import dataclass, field
from datetime import datetime, date
from enum import Enum
from typing import Any, Optional
from ..models.game import Game
from ..models.team import Team
from ..models.stadium import Stadium
from ..models.aliases import TeamAlias, StadiumAlias, AliasType
from ..models.sport import Sport, LeagueStructure
from .cloudkit import CloudKitRecord, RecordType
def _date_to_datetime(d: Optional[date]) -> Optional[datetime]:
"""Convert a date to a datetime at midnight UTC.
CloudKit TIMESTAMP fields require datetime, not date.
"""
if d is None:
return None
return datetime(d.year, d.month, d.day, 0, 0, 0)
class DiffAction(str, Enum):
"""Action to take for a record."""
CREATE = "create"
UPDATE = "update"
DELETE = "delete"
UNCHANGED = "unchanged"
@dataclass
class RecordDiff:
"""Represents the difference between local and remote records.
Attributes:
record_name: Canonical record ID
record_type: CloudKit record type
action: Action to take (create, update, delete, unchanged)
local_record: Local CloudKitRecord (None if delete)
remote_record: Remote record dict (None if create)
changed_fields: List of field names that changed (for update)
record_change_tag: Remote record's change tag (for update)
"""
record_name: str
record_type: RecordType
action: DiffAction
local_record: Optional[CloudKitRecord] = None
remote_record: Optional[dict] = None
changed_fields: list[str] = field(default_factory=list)
record_change_tag: Optional[str] = None
@dataclass
class DiffResult:
"""Result of diffing local and remote records.
Attributes:
creates: Records to create
updates: Records to update
deletes: Records to delete (record names)
unchanged: Records with no changes
"""
creates: list[RecordDiff] = field(default_factory=list)
updates: list[RecordDiff] = field(default_factory=list)
deletes: list[RecordDiff] = field(default_factory=list)
unchanged: list[RecordDiff] = field(default_factory=list)
@property
def create_count(self) -> int:
return len(self.creates)
@property
def update_count(self) -> int:
return len(self.updates)
@property
def delete_count(self) -> int:
return len(self.deletes)
@property
def unchanged_count(self) -> int:
return len(self.unchanged)
@property
def total_changes(self) -> int:
return self.create_count + self.update_count + self.delete_count
def get_records_to_upload(self) -> list[CloudKitRecord]:
"""Get all records that need to be uploaded (creates + updates)."""
records = []
for diff in self.creates:
if diff.local_record:
records.append(diff.local_record)
for diff in self.updates:
if diff.local_record:
# Add change tag for update
diff.local_record.record_change_tag = diff.record_change_tag
records.append(diff.local_record)
return records
class RecordDiffer:
"""Compares local records with CloudKit records.
Field names must match CKModels.swift field keys exactly (camelCase).
"""
# Fields to compare for each record type (matching CKModels.swift keys)
GAME_FIELDS = [
"gameId", "canonicalId", "sport", "season", "dateTime",
"homeTeamCanonicalId", "awayTeamCanonicalId", "stadiumCanonicalId",
"isPlayoff", "broadcastInfo",
]
TEAM_FIELDS = [
"teamId", "canonicalId", "sport", "city", "name", "abbreviation",
"stadiumCanonicalId", "logoURL", "primaryColor", "secondaryColor",
]
STADIUM_FIELDS = [
"stadiumId", "canonicalId", "sport", "name", "city", "state",
"location", "capacity", "yearOpened", "imageURL",
]
TEAM_ALIAS_FIELDS = [
"aliasId", "teamCanonicalId", "aliasType", "aliasValue",
"validFrom", "validUntil",
]
STADIUM_ALIAS_FIELDS = [
"aliasName", "stadiumCanonicalId", "validFrom", "validUntil",
]
SPORT_FIELDS = [
"sportId", "abbreviation", "displayName", "iconName",
"colorHex", "seasonStartMonth", "seasonEndMonth", "isActive",
]
LEAGUE_STRUCTURE_FIELDS = [
"structureId", "sport", "type", "name", "abbreviation",
"parentId", "displayOrder",
]
def diff_games(
self,
local_games: list[Game],
remote_records: list[dict],
) -> DiffResult:
"""Diff local games against remote CloudKit records.
Args:
local_games: List of local Game objects
remote_records: List of remote record dictionaries
Returns:
DiffResult with creates, updates, deletes
"""
local_records = [self._game_to_record(g) for g in local_games]
return self._diff_records(
local_records,
remote_records,
RecordType.GAME,
self.GAME_FIELDS,
)
def diff_teams(
self,
local_teams: list[Team],
remote_records: list[dict],
) -> DiffResult:
"""Diff local teams against remote CloudKit records.
Args:
local_teams: List of local Team objects
remote_records: List of remote record dictionaries
Returns:
DiffResult with creates, updates, deletes
"""
local_records = [self._team_to_record(t) for t in local_teams]
return self._diff_records(
local_records,
remote_records,
RecordType.TEAM,
self.TEAM_FIELDS,
)
def diff_stadiums(
self,
local_stadiums: list[Stadium],
remote_records: list[dict],
) -> DiffResult:
"""Diff local stadiums against remote CloudKit records.
Args:
local_stadiums: List of local Stadium objects
remote_records: List of remote record dictionaries
Returns:
DiffResult with creates, updates, deletes
"""
local_records = [self._stadium_to_record(s) for s in local_stadiums]
return self._diff_records(
local_records,
remote_records,
RecordType.STADIUM,
self.STADIUM_FIELDS,
)
def diff_team_aliases(
self,
local_aliases: list[TeamAlias],
remote_records: list[dict],
) -> DiffResult:
"""Diff local team aliases against remote CloudKit records.
Args:
local_aliases: List of local TeamAlias objects
remote_records: List of remote record dictionaries
Returns:
DiffResult with creates, updates, deletes
"""
local_records = [self._team_alias_to_record(a) for a in local_aliases]
return self._diff_records(
local_records,
remote_records,
RecordType.TEAM_ALIAS,
self.TEAM_ALIAS_FIELDS,
)
def diff_stadium_aliases(
self,
local_aliases: list[StadiumAlias],
remote_records: list[dict],
) -> DiffResult:
"""Diff local stadium aliases against remote CloudKit records.
Args:
local_aliases: List of local StadiumAlias objects
remote_records: List of remote record dictionaries
Returns:
DiffResult with creates, updates, deletes
"""
local_records = [self._stadium_alias_to_record(a) for a in local_aliases]
return self._diff_records(
local_records,
remote_records,
RecordType.STADIUM_ALIAS,
self.STADIUM_ALIAS_FIELDS,
)
def diff_sports(
self,
local_sports: list[Sport],
remote_records: list[dict],
) -> DiffResult:
"""Diff local sports against remote CloudKit records.
Args:
local_sports: List of local Sport objects
remote_records: List of remote record dictionaries
Returns:
DiffResult with creates, updates, deletes
"""
local_records = [self._sport_to_record(s) for s in local_sports]
return self._diff_records(
local_records,
remote_records,
RecordType.SPORT,
self.SPORT_FIELDS,
)
def diff_league_structures(
self,
local_structures: list[LeagueStructure],
remote_records: list[dict],
) -> DiffResult:
"""Diff local league structures against remote CloudKit records.
Args:
local_structures: List of local LeagueStructure objects
remote_records: List of remote record dictionaries
Returns:
DiffResult with creates, updates, deletes
"""
local_records = [self._league_structure_to_record(s) for s in local_structures]
return self._diff_records(
local_records,
remote_records,
RecordType.LEAGUE_STRUCTURE,
self.LEAGUE_STRUCTURE_FIELDS,
)
def _diff_records(
self,
local_records: list[CloudKitRecord],
remote_records: list[dict],
record_type: RecordType,
compare_fields: list[str],
) -> DiffResult:
"""Compare local and remote records.
Args:
local_records: List of local CloudKitRecord objects
remote_records: List of remote record dictionaries
record_type: Type of records being compared
compare_fields: List of field names to compare
Returns:
DiffResult with categorized differences
"""
result = DiffResult()
# Index remote records by name
remote_by_name: dict[str, dict] = {}
for record in remote_records:
name = record.get("recordName")
if name:
remote_by_name[name] = record
# Index local records by name
local_by_name: dict[str, CloudKitRecord] = {}
for record in local_records:
local_by_name[record.record_name] = record
# Find creates and updates
for local_record in local_records:
remote = remote_by_name.get(local_record.record_name)
if remote is None:
# New record
result.creates.append(RecordDiff(
record_name=local_record.record_name,
record_type=record_type,
action=DiffAction.CREATE,
local_record=local_record,
))
else:
# Check for changes
changed_fields = self._compare_fields(
local_record.fields,
remote.get("fields", {}),
compare_fields,
)
if changed_fields:
result.updates.append(RecordDiff(
record_name=local_record.record_name,
record_type=record_type,
action=DiffAction.UPDATE,
local_record=local_record,
remote_record=remote,
changed_fields=changed_fields,
record_change_tag=remote.get("recordChangeTag"),
))
else:
result.unchanged.append(RecordDiff(
record_name=local_record.record_name,
record_type=record_type,
action=DiffAction.UNCHANGED,
local_record=local_record,
remote_record=remote,
record_change_tag=remote.get("recordChangeTag"),
))
# Find deletes (remote records not in local)
local_names = set(local_by_name.keys())
for remote_name, remote in remote_by_name.items():
if remote_name not in local_names:
result.deletes.append(RecordDiff(
record_name=remote_name,
record_type=record_type,
action=DiffAction.DELETE,
remote_record=remote,
record_change_tag=remote.get("recordChangeTag"),
))
return result
def _compare_fields(
self,
local_fields: dict[str, Any],
remote_fields: dict[str, dict],
compare_fields: list[str],
) -> list[str]:
"""Compare field values between local and remote.
Args:
local_fields: Local field values
remote_fields: Remote field values (CloudKit format)
compare_fields: Fields to compare
Returns:
List of field names that differ
"""
changed = []
for field_name in compare_fields:
local_value = local_fields.get(field_name)
remote_field = remote_fields.get(field_name, {})
remote_value = remote_field.get("value") if remote_field else None
# Normalize values for comparison
local_normalized = self._normalize_value(local_value)
remote_normalized = self._normalize_remote_value(remote_value, remote_field)
if local_normalized != remote_normalized:
changed.append(field_name)
return changed
def _normalize_value(self, value: Any) -> Any:
"""Normalize a local value for comparison."""
if value is None:
return None
if isinstance(value, datetime):
# Convert to milliseconds since epoch
return int(value.timestamp() * 1000)
if isinstance(value, float):
# Round to 6 decimal places for coordinate comparison
return round(value, 6)
return value
def _normalize_remote_value(self, value: Any, field_data: dict) -> Any:
"""Normalize a remote CloudKit value for comparison."""
if value is None:
return None
field_type = field_data.get("type", "")
if field_type == "TIMESTAMP":
# Already in milliseconds
return value
if field_type == "DOUBLE":
return round(value, 6)
if field_type == "LOCATION":
# Return as tuple for comparison
if isinstance(value, dict):
return (
round(value.get("latitude", 0), 6),
round(value.get("longitude", 0), 6),
)
return value
def _game_to_record(self, game: Game) -> CloudKitRecord:
"""Convert a Game to a CloudKitRecord.
Field names match CKGame keys in CKModels.swift:
- gameId, canonicalId: Unique identifiers
- homeTeamCanonicalId, awayTeamCanonicalId, stadiumCanonicalId: References as strings
- dateTime: Game time as datetime (will be converted to TIMESTAMP)
- sport: Sport code uppercase (e.g., "MLB")
- season: Season string (e.g., "2025-26" or "2026")
- isPlayoff: Boolean as int (1 or 0)
- broadcastInfo: Optional broadcast network string
"""
# Format season as string
sport_lower = game.sport.lower()
if sport_lower in ("nba", "nhl"):
season_str = f"{game.season}-{str(game.season + 1)[-2:]}"
else:
season_str = str(game.season)
return CloudKitRecord(
record_name=game.id,
record_type=RecordType.GAME,
fields={
"gameId": game.id,
"canonicalId": game.id,
"sport": game.sport.upper(),
"season": season_str,
"dateTime": game.game_date,
"homeTeamCanonicalId": game.home_team_id,
"awayTeamCanonicalId": game.away_team_id,
"stadiumCanonicalId": game.stadium_id,
"isPlayoff": False, # Default, can be overridden
"broadcastInfo": None, # Default, can be overridden
},
)
def _team_to_record(self, team: Team) -> CloudKitRecord:
"""Convert a Team to a CloudKitRecord.
Field names match CKTeam keys in CKModels.swift:
- teamId, canonicalId: Unique identifiers
- name, abbreviation, city: Team info
- sport: Sport code uppercase (e.g., "NBA")
- stadiumCanonicalId: Home stadium canonical ID string
- logoURL: URL string for team logo
- primaryColor, secondaryColor: Hex color strings
"""
return CloudKitRecord(
record_name=team.id,
record_type=RecordType.TEAM,
fields={
"teamId": team.id,
"canonicalId": team.id,
"sport": team.sport.upper(),
"city": team.city,
"name": team.name,
"abbreviation": team.abbreviation,
"stadiumCanonicalId": team.stadium_id,
"logoURL": team.logo_url,
"primaryColor": team.primary_color,
"secondaryColor": team.secondary_color,
},
)
def _stadium_to_record(self, stadium: Stadium) -> CloudKitRecord:
"""Convert a Stadium to a CloudKitRecord.
Field names match CKStadium keys in CKModels.swift:
- stadiumId, canonicalId: Unique identifiers
- name, city, state: Location info
- location: CloudKit LOCATION type with latitude/longitude
- capacity: Seating capacity as int
- yearOpened: Year opened as int
- imageURL: URL string for stadium image
- sport: Sport code uppercase (e.g., "MLB")
- timezoneIdentifier: IANA timezone (e.g., "America/New_York")
"""
return CloudKitRecord(
record_name=stadium.id,
record_type=RecordType.STADIUM,
fields={
"stadiumId": stadium.id,
"canonicalId": stadium.id,
"sport": stadium.sport.upper(),
"name": stadium.name,
"city": stadium.city,
"state": stadium.state,
# CloudKit LOCATION type expects dict with latitude/longitude
"location": {
"latitude": stadium.latitude,
"longitude": stadium.longitude,
},
"capacity": stadium.capacity,
"yearOpened": stadium.opened_year,
"imageURL": stadium.image_url,
"timezoneIdentifier": stadium.timezone,
},
)
def _team_alias_to_record(self, alias: TeamAlias) -> CloudKitRecord:
"""Convert a TeamAlias to a CloudKitRecord.
Field names match CKTeamAlias keys in CKModels.swift:
- aliasId: Unique identifier
- teamCanonicalId: The canonical team this alias resolves to
- aliasType: Type of alias ("abbreviation", "name", "city")
- aliasValue: The alias value to match
- validFrom, validUntil: Optional date bounds
- schemaVersion, lastModified: Versioning fields
"""
return CloudKitRecord(
record_name=alias.id,
record_type=RecordType.TEAM_ALIAS,
fields={
"aliasId": alias.id,
"teamCanonicalId": alias.team_canonical_id,
"aliasType": alias.alias_type.value,
"aliasValue": alias.alias_value,
"validFrom": _date_to_datetime(alias.valid_from),
"validUntil": _date_to_datetime(alias.valid_until),
"schemaVersion": 1,
"lastModified": datetime.utcnow(),
},
)
def _stadium_alias_to_record(self, alias: StadiumAlias) -> CloudKitRecord:
"""Convert a StadiumAlias to a CloudKitRecord.
Field names match CKStadiumAlias keys in CKModels.swift:
- aliasName: The alias name (used as record name/primary key)
- stadiumCanonicalId: The canonical stadium this alias resolves to
- validFrom, validUntil: Optional date bounds
- schemaVersion, lastModified: Versioning fields
"""
# Record name must be unique - combine alias name with stadium ID
# to handle cases like "yankee stadium" mapping to both MLB and MLS stadiums
record_name = f"{alias.alias_name.lower()}|{alias.stadium_canonical_id}"
return CloudKitRecord(
record_name=record_name,
record_type=RecordType.STADIUM_ALIAS,
fields={
"aliasName": alias.alias_name.lower(),
"stadiumCanonicalId": alias.stadium_canonical_id,
"validFrom": _date_to_datetime(alias.valid_from),
"validUntil": _date_to_datetime(alias.valid_until),
"schemaVersion": 1,
"lastModified": datetime.utcnow(),
},
)
def _sport_to_record(self, sport: Sport) -> CloudKitRecord:
"""Convert a Sport to a CloudKitRecord.
Field names match CKSport keys in CKModels.swift:
- sportId: Unique identifier (e.g., 'MLB', 'NBA')
- abbreviation: Sport abbreviation
- displayName: Full display name
- iconName: SF Symbol name
- colorHex: Primary color as hex string
- seasonStartMonth, seasonEndMonth: Season boundary months (1-12)
- isActive: Whether sport is currently supported
- schemaVersion, lastModified: Versioning fields
"""
return CloudKitRecord(
record_name=sport.id,
record_type=RecordType.SPORT,
fields={
"sportId": sport.id,
"abbreviation": sport.abbreviation,
"displayName": sport.display_name,
"iconName": sport.icon_name,
"colorHex": sport.color_hex,
"seasonStartMonth": sport.season_start_month,
"seasonEndMonth": sport.season_end_month,
"isActive": sport.is_active,
"schemaVersion": 1,
"lastModified": datetime.utcnow(),
},
)
def _league_structure_to_record(self, structure: LeagueStructure) -> CloudKitRecord:
"""Convert a LeagueStructure to a CloudKitRecord.
Field names match CKLeagueStructure keys in CKModels.swift:
- structureId: Unique identifier (e.g., 'nba_eastern', 'mlb_al_east')
- sport: Sport code (e.g., 'NBA', 'MLB')
- type: Structure type ('conference', 'division', 'league')
- name: Full name
- abbreviation: Optional abbreviation
- parentId: Parent structure ID (e.g., division's parent is conference)
- displayOrder: Order for display (0-indexed)
- schemaVersion, lastModified: Versioning fields
"""
return CloudKitRecord(
record_name=structure.id,
record_type=RecordType.LEAGUE_STRUCTURE,
fields={
"structureId": structure.id,
"sport": structure.sport.upper(),
"type": structure.structure_type.value,
"name": structure.name,
"abbreviation": structure.abbreviation,
"parentId": structure.parent_id,
"displayOrder": structure.display_order,
"schemaVersion": 1,
"lastModified": datetime.utcnow(),
},
)
def game_to_cloudkit_record(game: Game) -> CloudKitRecord:
"""Convert a Game to a CloudKitRecord.
Convenience function for external use.
"""
differ = RecordDiffer()
return differ._game_to_record(game)
def team_to_cloudkit_record(team: Team) -> CloudKitRecord:
"""Convert a Team to a CloudKitRecord.
Convenience function for external use.
"""
differ = RecordDiffer()
return differ._team_to_record(team)
def stadium_to_cloudkit_record(stadium: Stadium) -> CloudKitRecord:
"""Convert a Stadium to a CloudKitRecord.
Convenience function for external use.
"""
differ = RecordDiffer()
return differ._stadium_to_record(stadium)
def team_alias_to_cloudkit_record(alias: TeamAlias) -> CloudKitRecord:
"""Convert a TeamAlias to a CloudKitRecord.
Convenience function for external use.
"""
differ = RecordDiffer()
return differ._team_alias_to_record(alias)
def stadium_alias_to_cloudkit_record(alias: StadiumAlias) -> CloudKitRecord:
"""Convert a StadiumAlias to a CloudKitRecord.
Convenience function for external use.
"""
differ = RecordDiffer()
return differ._stadium_alias_to_record(alias)
def sport_to_cloudkit_record(sport: Sport) -> CloudKitRecord:
"""Convert a Sport to a CloudKitRecord.
Convenience function for external use.
"""
differ = RecordDiffer()
return differ._sport_to_record(sport)
def league_structure_to_cloudkit_record(structure: LeagueStructure) -> CloudKitRecord:
"""Convert a LeagueStructure to a CloudKitRecord.
Convenience function for external use.
"""
differ = RecordDiffer()
return differ._league_structure_to_record(structure)