From c49206bb7cf97ff8e2b78a34e4804d0886dda799 Mon Sep 17 00:00:00 2001 From: Trey t Date: Tue, 20 Jan 2026 12:25:00 -0600 Subject: [PATCH] wip --- Scripts/.claude/settings.local.json | 3 +- Scripts/sportstime_parser/cli.py | 329 ++++++++++++++++++ Scripts/sportstime_parser/config.py | 2 + Scripts/sportstime_parser/models/game.py | 4 +- .../sportstime_parser/uploaders/cloudkit.py | 33 +- Scripts/sportstime_parser/uploaders/diff.py | 5 +- 6 files changed, 359 insertions(+), 17 deletions(-) diff --git a/Scripts/.claude/settings.local.json b/Scripts/.claude/settings.local.json index 87a44b5..784178b 100644 --- a/Scripts/.claude/settings.local.json +++ b/Scripts/.claude/settings.local.json @@ -18,7 +18,8 @@ "Bash(python -m sportstime_parser:*)", "Bash(python -m py_compile:*)", "WebFetch(domain:en.wikipedia.org)", - "Bash(tree:*)" + "Bash(tree:*)", + "Bash(python:*)" ] } } diff --git a/Scripts/sportstime_parser/cli.py b/Scripts/sportstime_parser/cli.py index a78cb50..fce3576 100644 --- a/Scripts/sportstime_parser/cli.py +++ b/Scripts/sportstime_parser/cli.py @@ -26,6 +26,9 @@ Examples: sportstime-parser validate nba --season 2025 sportstime-parser upload nba --season 2025 sportstime-parser status + sportstime-parser purge --environment development + sportstime-parser count --environment development + sportstime-parser upload-static --environment development """, ) @@ -179,6 +182,53 @@ Examples: ) clear_parser.set_defaults(func=cmd_clear) + # Purge subcommand + purge_parser = subparsers.add_parser( + "purge", + help="Delete all records from CloudKit (DESTRUCTIVE)", + description="Delete ALL records from CloudKit. This is destructive and cannot be undone.", + ) + purge_parser.add_argument( + "--environment", "-e", + choices=["development", "production"], + default=CLOUDKIT_ENVIRONMENT, + help=f"CloudKit environment (default: {CLOUDKIT_ENVIRONMENT})", + ) + purge_parser.add_argument( + "--yes", "-y", + action="store_true", + help="Skip confirmation prompt", + ) + purge_parser.set_defaults(func=cmd_purge) + + # Count subcommand + count_parser = subparsers.add_parser( + "count", + help="Count records in CloudKit by type", + description="Display count of all record types in CloudKit", + ) + count_parser.add_argument( + "--environment", "-e", + choices=["development", "production"], + default=CLOUDKIT_ENVIRONMENT, + help=f"CloudKit environment (default: {CLOUDKIT_ENVIRONMENT})", + ) + count_parser.set_defaults(func=cmd_count) + + # Upload-static subcommand + upload_static_parser = subparsers.add_parser( + "upload-static", + help="Upload static reference data to CloudKit", + description="Upload league structure, team aliases, stadium aliases, and sports to CloudKit", + ) + upload_static_parser.add_argument( + "--environment", "-e", + choices=["development", "production"], + default=CLOUDKIT_ENVIRONMENT, + help=f"CloudKit environment (default: {CLOUDKIT_ENVIRONMENT})", + ) + upload_static_parser.set_defaults(func=cmd_upload_static) + return parser @@ -950,6 +1000,285 @@ def cmd_clear(args: argparse.Namespace) -> int: return 0 +def cmd_purge(args: argparse.Namespace) -> int: + """Execute the purge command to delete all CloudKit records.""" + from .uploaders.cloudkit import CloudKitClient, RecordType + + logger = get_logger() + + # Check CloudKit configuration + client = CloudKitClient(environment=args.environment) + if not client.is_configured: + logger.error("CloudKit not configured. Check CLOUDKIT_KEY_ID and private key.") + return 1 + + # Confirmation prompt + if not args.yes: + logger.warning(f"[bold red]WARNING: This will delete ALL records from CloudKit ({args.environment})![/bold red]") + logger.warning("This action cannot be undone.") + logger.info("") + response = input(f"Type 'DELETE {args.environment.upper()}' to confirm: ") + if response != f"DELETE {args.environment.upper()}": + logger.info("Aborted.") + return 1 + + logger.info(f"Purging all records from CloudKit ({args.environment})...") + logger.info("") + + record_types = [ + RecordType.GAME, + RecordType.TEAM, + RecordType.STADIUM, + RecordType.TEAM_ALIAS, + RecordType.STADIUM_ALIAS, + RecordType.SPORT, + RecordType.LEAGUE_STRUCTURE, + ] + + total_deleted = 0 + total_failed = 0 + + for record_type in record_types: + logger.info(f"Fetching {record_type.value} records...") + try: + records = client.fetch_all_records(record_type) + except Exception as e: + logger.error(f" Failed to fetch: {e}") + continue + + if not records: + logger.info(f" No {record_type.value} records found") + continue + + logger.info(f" Deleting {len(records)} {record_type.value} records...") + + try: + result = client.delete_records(record_type, records) + total_deleted += result.success_count + total_failed += result.failure_count + logger.info(f" [green]✓[/green] Deleted: {result.success_count}, Failed: {result.failure_count}") + except Exception as e: + logger.error(f" Failed to delete: {e}") + total_failed += len(records) + + logger.info("") + logger.info(f"{'='*50}") + logger.info(f"Total deleted: {total_deleted}") + logger.info(f"Total failed: {total_failed}") + + return 0 if total_failed == 0 else 1 + + +def cmd_upload_static(args: argparse.Namespace) -> int: + """Execute the upload-static command to upload reference data to CloudKit.""" + import json + from rich.progress import Progress, SpinnerColumn, TextColumn + + from .uploaders.cloudkit import CloudKitClient, RecordType + from .uploaders.diff import RecordDiffer + from .models.aliases import TeamAlias, StadiumAlias + from .models.sport import Sport, LeagueStructure, LeagueStructureType + from .config import SCRIPTS_DIR + + logger = get_logger() + + # Check CloudKit configuration + client = CloudKitClient(environment=args.environment) + if not client.is_configured: + logger.error("CloudKit not configured. Check CLOUDKIT_KEY_ID and private key.") + return 1 + + logger.info(f"Uploading static reference data to CloudKit ({args.environment})") + logger.info(f"{'='*50}") + + differ = RecordDiffer() + total_uploaded = 0 + total_failed = 0 + + # Define sports (hardcoded since there's no sports.json) + sports = [ + Sport(id="MLB", abbreviation="MLB", display_name="Major League Baseball", + icon_name="baseball.fill", color_hex="#002D72", season_start_month=3, season_end_month=11), + Sport(id="NBA", abbreviation="NBA", display_name="National Basketball Association", + icon_name="basketball.fill", color_hex="#1D428A", season_start_month=10, season_end_month=6), + Sport(id="NFL", abbreviation="NFL", display_name="National Football League", + icon_name="football.fill", color_hex="#013369", season_start_month=9, season_end_month=2), + Sport(id="NHL", abbreviation="NHL", display_name="National Hockey League", + icon_name="hockey.puck.fill", color_hex="#000000", season_start_month=10, season_end_month=6), + Sport(id="MLS", abbreviation="MLS", display_name="Major League Soccer", + icon_name="soccerball", color_hex="#80A63A", season_start_month=2, season_end_month=11), + Sport(id="WNBA", abbreviation="WNBA", display_name="Women's National Basketball Association", + icon_name="basketball.fill", color_hex="#FF6600", season_start_month=5, season_end_month=10), + Sport(id="NWSL", abbreviation="NWSL", display_name="National Women's Soccer League", + icon_name="soccerball", color_hex="#003087", season_start_month=3, season_end_month=11), + ] + + # Upload Sports + logger.info("Uploading Sports...") + try: + remote_sports = client.fetch_all_records(RecordType.SPORT) + except Exception: + remote_sports = [] + + diff_result = differ.diff_sports(sports, remote_sports) + records_to_upload = diff_result.get_records_to_upload() + + if records_to_upload: + result = client.save_records(records_to_upload) + total_uploaded += result.success_count + total_failed += result.failure_count + logger.info(f" [green]✓[/green] Sports: {result.success_count} uploaded, {result.failure_count} failed") + else: + logger.info(f" [dim]-[/dim] Sports: No changes") + + # Load and upload League Structures + logger.info("Uploading League Structures...") + league_structure_file = SCRIPTS_DIR / "league_structure.json" + if league_structure_file.exists(): + with open(league_structure_file, "r") as f: + data = json.load(f) + + structures = [] + for d in data: + # Handle "type" vs "structure_type" field name + structure_type = d.get("structure_type") or d.get("type") + structures.append(LeagueStructure( + id=d["id"], + sport=d["sport"], + structure_type=LeagueStructureType(structure_type), + name=d["name"], + abbreviation=d.get("abbreviation"), + parent_id=d.get("parent_id"), + display_order=d.get("display_order", 0), + )) + + try: + remote_structures = client.fetch_all_records(RecordType.LEAGUE_STRUCTURE) + except Exception: + remote_structures = [] + + diff_result = differ.diff_league_structures(structures, remote_structures) + records_to_upload = diff_result.get_records_to_upload() + + if records_to_upload: + result = client.save_records(records_to_upload) + total_uploaded += result.success_count + total_failed += result.failure_count + logger.info(f" [green]✓[/green] League Structures: {result.success_count} uploaded, {result.failure_count} failed") + else: + logger.info(f" [dim]-[/dim] League Structures: No changes ({len(structures)} unchanged)") + else: + logger.warning(f" [yellow]![/yellow] league_structure.json not found") + + # Load and upload Team Aliases + logger.info("Uploading Team Aliases...") + team_aliases_file = SCRIPTS_DIR / "team_aliases.json" + if team_aliases_file.exists(): + with open(team_aliases_file, "r") as f: + data = json.load(f) + + aliases = [TeamAlias.from_dict(d) for d in data] + + try: + remote_aliases = client.fetch_all_records(RecordType.TEAM_ALIAS) + except Exception: + remote_aliases = [] + + diff_result = differ.diff_team_aliases(aliases, remote_aliases) + records_to_upload = diff_result.get_records_to_upload() + + if records_to_upload: + result = client.save_records(records_to_upload) + total_uploaded += result.success_count + total_failed += result.failure_count + logger.info(f" [green]✓[/green] Team Aliases: {result.success_count} uploaded, {result.failure_count} failed") + else: + logger.info(f" [dim]-[/dim] Team Aliases: No changes ({len(aliases)} unchanged)") + else: + logger.warning(f" [yellow]![/yellow] team_aliases.json not found") + + # Load and upload Stadium Aliases + logger.info("Uploading Stadium Aliases...") + stadium_aliases_file = SCRIPTS_DIR / "stadium_aliases.json" + if stadium_aliases_file.exists(): + with open(stadium_aliases_file, "r") as f: + data = json.load(f) + + aliases = [StadiumAlias.from_dict(d) for d in data] + + try: + remote_aliases = client.fetch_all_records(RecordType.STADIUM_ALIAS) + except Exception: + remote_aliases = [] + + diff_result = differ.diff_stadium_aliases(aliases, remote_aliases) + records_to_upload = diff_result.get_records_to_upload() + + if records_to_upload: + result = client.save_records(records_to_upload) + total_uploaded += result.success_count + total_failed += result.failure_count + logger.info(f" [green]✓[/green] Stadium Aliases: {result.success_count} uploaded, {result.failure_count} failed") + else: + logger.info(f" [dim]-[/dim] Stadium Aliases: No changes ({len(aliases)} unchanged)") + else: + logger.warning(f" [yellow]![/yellow] stadium_aliases.json not found") + + logger.info(f"{'='*50}") + logger.info(f"Total uploaded: {total_uploaded}") + logger.info(f"Total failed: {total_failed}") + + return 0 if total_failed == 0 else 1 + + +def cmd_count(args: argparse.Namespace) -> int: + """Execute the count command to show CloudKit record counts.""" + from .uploaders.cloudkit import CloudKitClient, RecordType + + logger = get_logger() + + # Check CloudKit configuration + client = CloudKitClient(environment=args.environment) + if not client.is_configured: + logger.error("CloudKit not configured. Check CLOUDKIT_KEY_ID and private key.") + return 1 + + logger.info(f"CloudKit record counts ({args.environment})") + logger.info(f"{'='*50}") + + record_types = [ + RecordType.GAME, + RecordType.TEAM, + RecordType.STADIUM, + RecordType.TEAM_ALIAS, + RecordType.STADIUM_ALIAS, + RecordType.SPORT, + RecordType.LEAGUE_STRUCTURE, + ] + + total = 0 + errors = [] + for record_type in record_types: + try: + records = client.fetch_all_records(record_type) + count = len(records) + total += count + logger.info(f" {record_type.value:<20} {count:>6}") + except Exception as e: + logger.error(f" {record_type.value:<20} [red]Not queryable[/red]") + errors.append(record_type.value) + + logger.info(f"{'='*50}") + logger.info(f" {'Total':<20} {total:>6}") + + if errors: + logger.info("") + logger.warning(f"[yellow]Records not queryable: {', '.join(errors)}[/yellow]") + logger.warning("[yellow]Enable QUERYABLE index in CloudKit Dashboard[/yellow]") + + return 0 + + def run_cli(argv: Optional[list[str]] = None) -> int: """Parse arguments and run the appropriate command.""" parser = create_parser() diff --git a/Scripts/sportstime_parser/config.py b/Scripts/sportstime_parser/config.py index f0a103b..6d9f3ec 100644 --- a/Scripts/sportstime_parser/config.py +++ b/Scripts/sportstime_parser/config.py @@ -31,6 +31,8 @@ DEFAULT_SEASON: int = 2025 CLOUDKIT_CONTAINER_ID: str = "iCloud.com.sportstime.app" CLOUDKIT_ENVIRONMENT: str = "development" CLOUDKIT_BATCH_SIZE: int = 200 +CLOUDKIT_KEY_ID: str = "152be0715e0276e31aaea5cbfe79dc872f298861a55c70fae14e5fe3e026cff9" +CLOUDKIT_PRIVATE_KEY_PATH: Path = SCRIPTS_DIR / "eckey.pem" # Rate limiting DEFAULT_REQUEST_DELAY: float = 1.0 # seconds between requests diff --git a/Scripts/sportstime_parser/models/game.py b/Scripts/sportstime_parser/models/game.py index 6d4acf5..0284539 100644 --- a/Scripts/sportstime_parser/models/game.py +++ b/Scripts/sportstime_parser/models/game.py @@ -138,7 +138,9 @@ class Game: @classmethod def from_canonical_dict(cls, data: dict) -> "Game": """Create a Game from a canonical dictionary (iOS app format).""" - game_date = datetime.fromisoformat(data["game_datetime_utc"]) + # Handle 'Z' suffix (fromisoformat doesn't support it before Python 3.11) + date_str = data["game_datetime_utc"].replace("Z", "+00:00") + game_date = datetime.fromisoformat(date_str) # Parse season string (e.g., "2025-26" -> 2025, or "2025" -> 2025) season_str = data["season"] diff --git a/Scripts/sportstime_parser/uploaders/cloudkit.py b/Scripts/sportstime_parser/uploaders/cloudkit.py index 7090975..57c9118 100644 --- a/Scripts/sportstime_parser/uploaders/cloudkit.py +++ b/Scripts/sportstime_parser/uploaders/cloudkit.py @@ -28,6 +28,8 @@ from ..config import ( CLOUDKIT_CONTAINER_ID, CLOUDKIT_ENVIRONMENT, CLOUDKIT_BATCH_SIZE, + CLOUDKIT_KEY_ID, + CLOUDKIT_PRIVATE_KEY_PATH, ) from ..utils.logging import get_logger @@ -86,14 +88,15 @@ class CloudKitRecord: def _format_field_value(self, value: Any) -> dict: """Format a single field value for CloudKit API.""" - if isinstance(value, str): + # Check bool BEFORE int (bool is a subclass of int in Python) + if isinstance(value, bool): + return {"value": 1 if value else 0, "type": "INT64"} + elif isinstance(value, str): return {"value": value, "type": "STRING"} elif isinstance(value, int): return {"value": value, "type": "INT64"} elif isinstance(value, float): return {"value": value, "type": "DOUBLE"} - elif isinstance(value, bool): - return {"value": 1 if value else 0, "type": "INT64"} elif isinstance(value, datetime): # CloudKit expects milliseconds since epoch timestamp_ms = int(value.timestamp() * 1000) @@ -182,8 +185,8 @@ class CloudKitClient: self.environment = environment self.logger = get_logger() - # Load authentication credentials - self.key_id = key_id or os.environ.get("CLOUDKIT_KEY_ID") + # Load authentication credentials (config defaults > env vars > None) + self.key_id = key_id or os.environ.get("CLOUDKIT_KEY_ID") or CLOUDKIT_KEY_ID if private_key: self._private_key_pem = private_key @@ -193,6 +196,8 @@ class CloudKitClient: self._private_key_pem = os.environ["CLOUDKIT_PRIVATE_KEY"] elif os.environ.get("CLOUDKIT_PRIVATE_KEY_PATH"): self._private_key_pem = Path(os.environ["CLOUDKIT_PRIVATE_KEY_PATH"]).read_text() + elif CLOUDKIT_PRIVATE_KEY_PATH.exists(): + self._private_key_pem = CLOUDKIT_PRIVATE_KEY_PATH.read_text() else: self._private_key_pem = None @@ -494,13 +499,13 @@ class CloudKitClient: def delete_records( self, record_type: RecordType, - record_names: list[str], + records: list[dict], ) -> BatchResult: """Delete records from CloudKit. Args: record_type: Type of records to delete - record_names: List of record names to delete + records: List of record dicts (must have recordName and recordChangeTag) Returns: BatchResult with success/failure details @@ -508,16 +513,16 @@ class CloudKitClient: result = BatchResult() # Process in batches - for i in range(0, len(record_names), CLOUDKIT_BATCH_SIZE): - batch = record_names[i:i + CLOUDKIT_BATCH_SIZE] + for i in range(0, len(records), CLOUDKIT_BATCH_SIZE): + batch = records[i:i + CLOUDKIT_BATCH_SIZE] operations = [] - for name in batch: + for record in batch: operations.append({ "operationType": "delete", "record": { - "recordName": name, - "recordType": record_type.value, + "recordName": record["recordName"], + "recordChangeTag": record.get("recordChangeTag"), }, }) @@ -526,9 +531,9 @@ class CloudKitClient: try: response = self._request("POST", "records/modify", body) except CloudKitError as e: - for name in batch: + for record in batch: result.failed.append(OperationResult( - record_name=name, + record_name=record["recordName"], success=False, error_message=str(e), )) diff --git a/Scripts/sportstime_parser/uploaders/diff.py b/Scripts/sportstime_parser/uploaders/diff.py index d2bf3e5..3bec2c1 100644 --- a/Scripts/sportstime_parser/uploaders/diff.py +++ b/Scripts/sportstime_parser/uploaders/diff.py @@ -602,8 +602,11 @@ class RecordDiffer: - validFrom, validUntil: Optional date bounds - schemaVersion, lastModified: Versioning fields """ + # Record name must be unique - combine alias name with stadium ID + # to handle cases like "yankee stadium" mapping to both MLB and MLS stadiums + record_name = f"{alias.alias_name.lower()}|{alias.stadium_canonical_id}" return CloudKitRecord( - record_name=alias.alias_name.lower(), + record_name=record_name, record_type=RecordType.STADIUM_ALIAS, fields={ "aliasName": alias.alias_name.lower(),