feat: add Django web app, CloudKit sync, dashboard, and game_datetime_utc export
Adds the full Django application layer on top of sportstime_parser: - core: Sport, Team, Stadium, Game models with aliases and league structure - scraper: orchestration engine, adapter, job management, Celery tasks - cloudkit: CloudKit sync client, sync state tracking, sync jobs - dashboard: staff dashboard for monitoring scrapers, sync, review queue - notifications: email reports for scrape/sync results - Docker setup for deployment (Dockerfile, docker-compose, entrypoint) Game exports now use game_datetime_utc (ISO 8601 UTC) instead of venue-local date+time strings, matching the canonical format used by the iOS app. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Trey t
1
cloudkit/__init__.py
Normal file
1
cloudkit/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
default_app_config = 'cloudkit.apps.CloudKitConfig'
|
||||
213
cloudkit/admin.py
Normal file
213
cloudkit/admin.py
Normal file
@@ -0,0 +1,213 @@
|
||||
from django.contrib import admin
|
||||
from django.utils.html import format_html
|
||||
from import_export.admin import ImportExportMixin, ImportExportModelAdmin
|
||||
from simple_history.admin import SimpleHistoryAdmin
|
||||
|
||||
from .models import CloudKitConfiguration, CloudKitSyncState, CloudKitSyncJob
|
||||
from .resources import CloudKitConfigurationResource, CloudKitSyncStateResource, CloudKitSyncJobResource
|
||||
|
||||
|
||||
@admin.register(CloudKitConfiguration)
|
||||
class CloudKitConfigurationAdmin(ImportExportMixin, SimpleHistoryAdmin):
|
||||
resource_class = CloudKitConfigurationResource
|
||||
list_display = [
|
||||
'name',
|
||||
'environment',
|
||||
'container_id',
|
||||
'is_active_badge',
|
||||
'auto_sync_after_scrape',
|
||||
'batch_size',
|
||||
]
|
||||
list_filter = ['environment', 'is_active']
|
||||
search_fields = ['name', 'container_id']
|
||||
readonly_fields = ['created_at', 'updated_at']
|
||||
|
||||
fieldsets = [
|
||||
(None, {
|
||||
'fields': ['name', 'environment', 'is_active']
|
||||
}),
|
||||
('CloudKit Credentials', {
|
||||
'fields': ['container_id', 'key_id', 'private_key', 'private_key_path'],
|
||||
'description': 'Enter your private key content directly OR provide a file path'
|
||||
}),
|
||||
('Sync Settings', {
|
||||
'fields': ['batch_size', 'auto_sync_after_scrape']
|
||||
}),
|
||||
('Metadata', {
|
||||
'fields': ['created_at', 'updated_at'],
|
||||
'classes': ['collapse']
|
||||
}),
|
||||
]
|
||||
|
||||
actions = ['run_sync', 'test_connection']
|
||||
|
||||
def is_active_badge(self, obj):
|
||||
if obj.is_active:
|
||||
return format_html(
|
||||
'<span style="color: green; font-weight: bold;">● ACTIVE</span>'
|
||||
)
|
||||
return format_html('<span style="color: gray;">○ Inactive</span>')
|
||||
is_active_badge.short_description = 'Status'
|
||||
|
||||
@admin.action(description='Run sync with selected configuration')
|
||||
def run_sync(self, request, queryset):
|
||||
from cloudkit.tasks import run_cloudkit_sync
|
||||
for config in queryset:
|
||||
run_cloudkit_sync.delay(config.id)
|
||||
self.message_user(request, f'Started {queryset.count()} sync jobs.')
|
||||
|
||||
@admin.action(description='Test CloudKit connection')
|
||||
def test_connection(self, request, queryset):
|
||||
from django.contrib import messages
|
||||
for config in queryset:
|
||||
try:
|
||||
client = config.get_client()
|
||||
if client.test_connection():
|
||||
self.message_user(
|
||||
request,
|
||||
f'✓ {config.name}: Connection successful!',
|
||||
messages.SUCCESS
|
||||
)
|
||||
else:
|
||||
self.message_user(
|
||||
request,
|
||||
f'✗ {config.name}: Connection failed',
|
||||
messages.ERROR
|
||||
)
|
||||
except Exception as e:
|
||||
self.message_user(
|
||||
request,
|
||||
f'✗ {config.name}: {str(e)}',
|
||||
messages.ERROR
|
||||
)
|
||||
|
||||
|
||||
@admin.register(CloudKitSyncState)
|
||||
class CloudKitSyncStateAdmin(ImportExportModelAdmin):
|
||||
resource_class = CloudKitSyncStateResource
|
||||
list_display = [
|
||||
'record_id',
|
||||
'record_type',
|
||||
'sync_status_badge',
|
||||
'last_synced',
|
||||
'retry_count',
|
||||
]
|
||||
list_filter = ['sync_status', 'record_type']
|
||||
search_fields = ['record_id', 'cloudkit_record_name']
|
||||
ordering = ['-updated_at']
|
||||
readonly_fields = [
|
||||
'record_type',
|
||||
'record_id',
|
||||
'cloudkit_record_name',
|
||||
'local_hash',
|
||||
'remote_change_tag',
|
||||
'last_synced',
|
||||
'last_error',
|
||||
'retry_count',
|
||||
'created_at',
|
||||
'updated_at',
|
||||
]
|
||||
|
||||
actions = ['mark_pending', 'retry_failed']
|
||||
|
||||
def has_add_permission(self, request):
|
||||
return False
|
||||
|
||||
def sync_status_badge(self, obj):
|
||||
colors = {
|
||||
'pending': '#f0ad4e',
|
||||
'synced': '#5cb85c',
|
||||
'failed': '#d9534f',
|
||||
'deleted': '#999',
|
||||
}
|
||||
color = colors.get(obj.sync_status, '#999')
|
||||
return format_html(
|
||||
'<span style="background-color: {}; color: white; padding: 3px 8px; '
|
||||
'border-radius: 3px; font-size: 11px;">{}</span>',
|
||||
color,
|
||||
obj.sync_status.upper()
|
||||
)
|
||||
sync_status_badge.short_description = 'Status'
|
||||
|
||||
@admin.action(description='Mark selected as pending sync')
|
||||
def mark_pending(self, request, queryset):
|
||||
updated = queryset.update(sync_status='pending')
|
||||
self.message_user(request, f'{updated} records marked as pending.')
|
||||
|
||||
@admin.action(description='Retry failed syncs')
|
||||
def retry_failed(self, request, queryset):
|
||||
updated = queryset.filter(sync_status='failed').update(
|
||||
sync_status='pending',
|
||||
retry_count=0
|
||||
)
|
||||
self.message_user(request, f'{updated} failed records queued for retry.')
|
||||
|
||||
|
||||
@admin.register(CloudKitSyncJob)
|
||||
class CloudKitSyncJobAdmin(ImportExportModelAdmin):
|
||||
resource_class = CloudKitSyncJobResource
|
||||
list_display = [
|
||||
'id',
|
||||
'configuration',
|
||||
'status_badge',
|
||||
'triggered_by',
|
||||
'started_at',
|
||||
'duration_display',
|
||||
'records_summary',
|
||||
]
|
||||
list_filter = ['status', 'configuration', 'triggered_by']
|
||||
date_hierarchy = 'created_at'
|
||||
ordering = ['-created_at']
|
||||
readonly_fields = [
|
||||
'configuration',
|
||||
'status',
|
||||
'triggered_by',
|
||||
'started_at',
|
||||
'finished_at',
|
||||
'duration_display',
|
||||
'records_synced',
|
||||
'records_created',
|
||||
'records_updated',
|
||||
'records_deleted',
|
||||
'records_failed',
|
||||
'sport_filter',
|
||||
'record_type_filter',
|
||||
'error_message',
|
||||
'celery_task_id',
|
||||
'created_at',
|
||||
'updated_at',
|
||||
]
|
||||
|
||||
def has_add_permission(self, request):
|
||||
return False
|
||||
|
||||
def has_change_permission(self, request, obj=None):
|
||||
return False
|
||||
|
||||
def status_badge(self, obj):
|
||||
colors = {
|
||||
'pending': '#999',
|
||||
'running': '#f0ad4e',
|
||||
'completed': '#5cb85c',
|
||||
'failed': '#d9534f',
|
||||
'cancelled': '#777',
|
||||
}
|
||||
color = colors.get(obj.status, '#999')
|
||||
return format_html(
|
||||
'<span style="background-color: {}; color: white; padding: 3px 8px; '
|
||||
'border-radius: 3px; font-size: 11px;">{}</span>',
|
||||
color,
|
||||
obj.status.upper()
|
||||
)
|
||||
status_badge.short_description = 'Status'
|
||||
|
||||
def records_summary(self, obj):
|
||||
if obj.records_synced == 0 and obj.status != 'completed':
|
||||
return '-'
|
||||
return format_html(
|
||||
'<span title="Created: {}, Updated: {}, Deleted: {}, Failed: {}">'
|
||||
'{} synced ({} new)</span>',
|
||||
obj.records_created, obj.records_updated, obj.records_deleted, obj.records_failed,
|
||||
obj.records_synced, obj.records_created
|
||||
)
|
||||
records_summary.short_description = 'Records'
|
||||
7
cloudkit/apps.py
Normal file
7
cloudkit/apps.py
Normal file
@@ -0,0 +1,7 @@
|
||||
from django.apps import AppConfig
|
||||
|
||||
|
||||
class CloudKitConfig(AppConfig):
|
||||
default_auto_field = 'django.db.models.BigAutoField'
|
||||
name = 'cloudkit'
|
||||
verbose_name = 'CloudKit Sync'
|
||||
385
cloudkit/client.py
Normal file
385
cloudkit/client.py
Normal file
@@ -0,0 +1,385 @@
|
||||
"""
|
||||
CloudKit Web Services API client.
|
||||
Adapted from existing sportstime_parser.uploaders.cloudkit
|
||||
"""
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import time
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import jwt
|
||||
import requests
|
||||
from cryptography.hazmat.primitives import hashes, serialization
|
||||
from cryptography.hazmat.primitives.asymmetric import ec
|
||||
from cryptography.hazmat.backends import default_backend
|
||||
|
||||
|
||||
class CloudKitClient:
|
||||
"""
|
||||
Client for CloudKit Web Services API.
|
||||
"""
|
||||
|
||||
BASE_URL = "https://api.apple-cloudkit.com"
|
||||
TOKEN_EXPIRY_SECONDS = 3600 # 1 hour
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
container_id: str,
|
||||
environment: str = 'development',
|
||||
key_id: str = '',
|
||||
private_key: str = '',
|
||||
private_key_path: str = '',
|
||||
):
|
||||
self.container_id = container_id
|
||||
self.environment = environment
|
||||
self.key_id = key_id
|
||||
self._private_key_pem = private_key
|
||||
self.private_key_path = private_key_path
|
||||
self._private_key = None
|
||||
self._token = None
|
||||
self._token_expiry = 0
|
||||
|
||||
# Load private key
|
||||
if not self._private_key_pem and private_key_path:
|
||||
key_path = Path(private_key_path)
|
||||
if key_path.exists():
|
||||
self._private_key_pem = key_path.read_text()
|
||||
|
||||
if self._private_key_pem:
|
||||
self._private_key = serialization.load_pem_private_key(
|
||||
self._private_key_pem.encode(),
|
||||
password=None,
|
||||
backend=default_backend(),
|
||||
)
|
||||
|
||||
@property
|
||||
def is_configured(self) -> bool:
|
||||
"""Check if the client has valid authentication credentials."""
|
||||
return bool(self.key_id and self._private_key)
|
||||
|
||||
def _get_api_path(self, operation: str) -> str:
|
||||
"""Build the full API path for an operation."""
|
||||
return f"/database/1/{self.container_id}/{self.environment}/public/{operation}"
|
||||
|
||||
def _get_token(self) -> str:
|
||||
"""Get a valid JWT token, generating a new one if needed."""
|
||||
if not self.is_configured:
|
||||
raise ValueError("CloudKit credentials not configured")
|
||||
|
||||
now = time.time()
|
||||
|
||||
# Return cached token if still valid (with 5 min buffer)
|
||||
if self._token and (self._token_expiry - now) > 300:
|
||||
return self._token
|
||||
|
||||
# Generate new token
|
||||
expiry = now + self.TOKEN_EXPIRY_SECONDS
|
||||
|
||||
payload = {
|
||||
'iss': self.key_id,
|
||||
'iat': int(now),
|
||||
'exp': int(expiry),
|
||||
'sub': self.container_id,
|
||||
}
|
||||
|
||||
self._token = jwt.encode(
|
||||
payload,
|
||||
self._private_key,
|
||||
algorithm='ES256',
|
||||
)
|
||||
self._token_expiry = expiry
|
||||
|
||||
return self._token
|
||||
|
||||
def _sign_request(self, method: str, path: str, body: Optional[bytes] = None) -> dict:
|
||||
"""Generate request headers with authentication.
|
||||
|
||||
Args:
|
||||
method: HTTP method
|
||||
path: API path
|
||||
body: Request body bytes
|
||||
|
||||
Returns:
|
||||
Dictionary of headers to include in the request
|
||||
"""
|
||||
token = self._get_token()
|
||||
|
||||
# CloudKit uses date in ISO format
|
||||
date_str = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
# Calculate body hash
|
||||
if body:
|
||||
body_hash = base64.b64encode(
|
||||
hashlib.sha256(body).digest()
|
||||
).decode()
|
||||
else:
|
||||
body_hash = base64.b64encode(
|
||||
hashlib.sha256(b"").digest()
|
||||
).decode()
|
||||
|
||||
# Build the message to sign
|
||||
# Format: date:body_hash:path
|
||||
message = f"{date_str}:{body_hash}:{path}"
|
||||
|
||||
# Sign the message
|
||||
signature = self._private_key.sign(
|
||||
message.encode(),
|
||||
ec.ECDSA(hashes.SHA256()),
|
||||
)
|
||||
signature_b64 = base64.b64encode(signature).decode()
|
||||
|
||||
return {
|
||||
'Authorization': f'Bearer {token}',
|
||||
'X-Apple-CloudKit-Request-KeyID': self.key_id,
|
||||
'X-Apple-CloudKit-Request-ISO8601Date': date_str,
|
||||
'X-Apple-CloudKit-Request-SignatureV1': signature_b64,
|
||||
'Content-Type': 'application/json',
|
||||
}
|
||||
|
||||
def _request(self, method: str, operation: str, body: Optional[dict] = None) -> dict:
|
||||
"""Make a request to the CloudKit API."""
|
||||
path = self._get_api_path(operation)
|
||||
url = f"{self.BASE_URL}{path}"
|
||||
|
||||
body_bytes = json.dumps(body).encode() if body else None
|
||||
headers = self._sign_request(method, path, body_bytes)
|
||||
|
||||
response = requests.request(
|
||||
method=method,
|
||||
url=url,
|
||||
headers=headers,
|
||||
data=body_bytes,
|
||||
)
|
||||
|
||||
if response.status_code == 200:
|
||||
return response.json()
|
||||
else:
|
||||
response.raise_for_status()
|
||||
|
||||
def _get_url(self, path: str) -> str:
|
||||
"""Build full API URL."""
|
||||
return f"{self.BASE_URL}/database/1/{self.container_id}/{self.environment}/public{path}"
|
||||
|
||||
def fetch_records(
|
||||
self,
|
||||
record_type: str,
|
||||
filter_by: Optional[dict] = None,
|
||||
sort_by: Optional[str] = None,
|
||||
limit: int = 200,
|
||||
) -> list:
|
||||
"""
|
||||
Fetch records from CloudKit.
|
||||
"""
|
||||
query = {
|
||||
'recordType': record_type,
|
||||
}
|
||||
|
||||
if filter_by:
|
||||
query['filterBy'] = filter_by
|
||||
|
||||
if sort_by:
|
||||
query['sortBy'] = [{'fieldName': sort_by}]
|
||||
|
||||
payload = {
|
||||
'query': query,
|
||||
'resultsLimit': limit,
|
||||
}
|
||||
|
||||
data = self._request('POST', 'records/query', payload)
|
||||
return data.get('records', [])
|
||||
|
||||
def save_records(self, records: list) -> dict:
|
||||
"""
|
||||
Save records to CloudKit.
|
||||
"""
|
||||
operations = []
|
||||
for record in records:
|
||||
op = {
|
||||
'operationType': 'forceReplace',
|
||||
'record': record,
|
||||
}
|
||||
operations.append(op)
|
||||
|
||||
payload = {
|
||||
'operations': operations,
|
||||
}
|
||||
|
||||
return self._request('POST', 'records/modify', payload)
|
||||
|
||||
def delete_records(self, record_names: list, record_type: str) -> dict:
|
||||
"""
|
||||
Delete records from CloudKit.
|
||||
"""
|
||||
operations = []
|
||||
for name in record_names:
|
||||
op = {
|
||||
'operationType': 'delete',
|
||||
'record': {
|
||||
'recordName': name,
|
||||
'recordType': record_type,
|
||||
},
|
||||
}
|
||||
operations.append(op)
|
||||
|
||||
payload = {
|
||||
'operations': operations,
|
||||
}
|
||||
|
||||
return self._request('POST', 'records/modify', payload)
|
||||
|
||||
def to_cloudkit_record(self, record_type: str, data: dict) -> dict:
|
||||
"""
|
||||
Convert local data dict to CloudKit record format.
|
||||
Field names must match existing CloudKit schema.
|
||||
"""
|
||||
record = {
|
||||
'recordType': record_type,
|
||||
'recordName': data['id'],
|
||||
'fields': {},
|
||||
}
|
||||
|
||||
if record_type == 'Sport':
|
||||
fields = record['fields']
|
||||
fields['sportId'] = {'value': data['id'], 'type': 'STRING'}
|
||||
fields['abbreviation'] = {'value': data['abbreviation'].upper(), 'type': 'STRING'}
|
||||
fields['displayName'] = {'value': data['displayName'], 'type': 'STRING'}
|
||||
fields['iconName'] = {'value': data.get('iconName', ''), 'type': 'STRING'}
|
||||
fields['colorHex'] = {'value': data.get('colorHex', ''), 'type': 'STRING'}
|
||||
fields['seasonStartMonth'] = {'value': data.get('seasonStartMonth', 1), 'type': 'INT64'}
|
||||
fields['seasonEndMonth'] = {'value': data.get('seasonEndMonth', 12), 'type': 'INT64'}
|
||||
fields['isActive'] = {'value': 1 if data.get('isActive') else 0, 'type': 'INT64'}
|
||||
|
||||
elif record_type == 'Game':
|
||||
# Match existing CloudKit Game schema
|
||||
fields = record['fields']
|
||||
fields['gameId'] = {'value': data['id'], 'type': 'STRING'}
|
||||
fields['canonicalId'] = {'value': data['id'], 'type': 'STRING'}
|
||||
fields['sport'] = {'value': data['sport'].upper(), 'type': 'STRING'}
|
||||
fields['season'] = {'value': str(data['season']), 'type': 'STRING'}
|
||||
fields['homeTeamCanonicalId'] = {'value': data['homeTeamId'], 'type': 'STRING'}
|
||||
fields['awayTeamCanonicalId'] = {'value': data['awayTeamId'], 'type': 'STRING'}
|
||||
if data.get('stadiumId'):
|
||||
fields['stadiumCanonicalId'] = {'value': data['stadiumId'], 'type': 'STRING'}
|
||||
if data.get('gameDate'):
|
||||
dt = datetime.fromisoformat(data['gameDate'].replace('Z', '+00:00'))
|
||||
fields['dateTime'] = {'value': int(dt.timestamp() * 1000), 'type': 'TIMESTAMP'}
|
||||
fields['isPlayoff'] = {'value': 1 if data.get('isPlayoff') else 0, 'type': 'INT64'}
|
||||
|
||||
elif record_type == 'Team':
|
||||
# Match existing CloudKit Team schema
|
||||
fields = record['fields']
|
||||
fields['teamId'] = {'value': data['id'], 'type': 'STRING'}
|
||||
fields['canonicalId'] = {'value': data['id'], 'type': 'STRING'}
|
||||
fields['sport'] = {'value': data['sport'].upper(), 'type': 'STRING'}
|
||||
fields['city'] = {'value': data.get('city', ''), 'type': 'STRING'}
|
||||
fields['name'] = {'value': data.get('name', ''), 'type': 'STRING'}
|
||||
fields['abbreviation'] = {'value': data.get('abbreviation', ''), 'type': 'STRING'}
|
||||
if data.get('homeStadiumId'):
|
||||
fields['stadiumCanonicalId'] = {'value': data['homeStadiumId'], 'type': 'STRING'}
|
||||
if data.get('primaryColor'):
|
||||
fields['primaryColor'] = {'value': data['primaryColor'], 'type': 'STRING'}
|
||||
if data.get('secondaryColor'):
|
||||
fields['secondaryColor'] = {'value': data['secondaryColor'], 'type': 'STRING'}
|
||||
if data.get('logoUrl'):
|
||||
fields['logoUrl'] = {'value': data['logoUrl'], 'type': 'STRING'}
|
||||
if data.get('divisionId'):
|
||||
fields['divisionCanonicalId'] = {'value': data['divisionId'], 'type': 'STRING'}
|
||||
if data.get('conferenceId'):
|
||||
fields['conferenceCanonicalId'] = {'value': data['conferenceId'], 'type': 'STRING'}
|
||||
|
||||
elif record_type == 'Stadium':
|
||||
# Match existing CloudKit Stadium schema
|
||||
fields = record['fields']
|
||||
fields['stadiumId'] = {'value': data['id'], 'type': 'STRING'}
|
||||
fields['canonicalId'] = {'value': data['id'], 'type': 'STRING'}
|
||||
fields['sport'] = {'value': data['sport'].upper(), 'type': 'STRING'}
|
||||
fields['name'] = {'value': data.get('name', ''), 'type': 'STRING'}
|
||||
fields['city'] = {'value': data.get('city', ''), 'type': 'STRING'}
|
||||
if data.get('state'):
|
||||
fields['state'] = {'value': data['state'], 'type': 'STRING'}
|
||||
# Use LOCATION type for coordinates
|
||||
if data.get('latitude') is not None and data.get('longitude') is not None:
|
||||
fields['location'] = {
|
||||
'value': {
|
||||
'latitude': float(data['latitude']),
|
||||
'longitude': float(data['longitude']),
|
||||
},
|
||||
'type': 'LOCATION'
|
||||
}
|
||||
if data.get('capacity'):
|
||||
fields['capacity'] = {'value': data['capacity'], 'type': 'INT64'}
|
||||
if data.get('yearOpened'):
|
||||
fields['yearOpened'] = {'value': data['yearOpened'], 'type': 'INT64'}
|
||||
if data.get('imageUrl'):
|
||||
fields['imageURL'] = {'value': data['imageUrl'], 'type': 'STRING'}
|
||||
if data.get('timezone'):
|
||||
fields['timezoneIdentifier'] = {'value': data['timezone'], 'type': 'STRING'}
|
||||
|
||||
elif record_type == 'Conference':
|
||||
fields = record['fields']
|
||||
fields['conferenceId'] = {'value': data['id'], 'type': 'STRING'}
|
||||
fields['canonicalId'] = {'value': data['id'], 'type': 'STRING'}
|
||||
fields['sport'] = {'value': data['sport'].upper(), 'type': 'STRING'}
|
||||
fields['name'] = {'value': data.get('name', ''), 'type': 'STRING'}
|
||||
fields['shortName'] = {'value': data.get('shortName', ''), 'type': 'STRING'}
|
||||
fields['order'] = {'value': data.get('order', 0), 'type': 'INT64'}
|
||||
|
||||
elif record_type == 'Division':
|
||||
fields = record['fields']
|
||||
fields['divisionId'] = {'value': data['id'], 'type': 'STRING'}
|
||||
fields['canonicalId'] = {'value': data['id'], 'type': 'STRING'}
|
||||
fields['conferenceCanonicalId'] = {'value': data['conferenceId'], 'type': 'STRING'}
|
||||
fields['sport'] = {'value': data['sport'].upper(), 'type': 'STRING'}
|
||||
fields['name'] = {'value': data.get('name', ''), 'type': 'STRING'}
|
||||
fields['shortName'] = {'value': data.get('shortName', ''), 'type': 'STRING'}
|
||||
fields['order'] = {'value': data.get('order', 0), 'type': 'INT64'}
|
||||
|
||||
elif record_type == 'TeamAlias':
|
||||
fields = record['fields']
|
||||
fields['aliasId'] = {'value': data['id'], 'type': 'STRING'}
|
||||
fields['teamCanonicalId'] = {'value': data['teamId'], 'type': 'STRING'}
|
||||
fields['aliasValue'] = {'value': data.get('alias', ''), 'type': 'STRING'}
|
||||
fields['aliasType'] = {'value': data.get('aliasType', ''), 'type': 'STRING'}
|
||||
if data.get('validFrom'):
|
||||
dt = datetime.fromisoformat(data['validFrom'])
|
||||
fields['validFrom'] = {'value': int(dt.timestamp() * 1000), 'type': 'TIMESTAMP'}
|
||||
if data.get('validUntil'):
|
||||
dt = datetime.fromisoformat(data['validUntil'])
|
||||
fields['validUntil'] = {'value': int(dt.timestamp() * 1000), 'type': 'TIMESTAMP'}
|
||||
|
||||
elif record_type == 'StadiumAlias':
|
||||
fields = record['fields']
|
||||
fields['stadiumCanonicalId'] = {'value': data['stadiumId'], 'type': 'STRING'}
|
||||
fields['aliasName'] = {'value': data.get('alias', ''), 'type': 'STRING'}
|
||||
if data.get('validFrom'):
|
||||
dt = datetime.fromisoformat(data['validFrom'])
|
||||
fields['validFrom'] = {'value': int(dt.timestamp() * 1000), 'type': 'TIMESTAMP'}
|
||||
if data.get('validUntil'):
|
||||
dt = datetime.fromisoformat(data['validUntil'])
|
||||
fields['validUntil'] = {'value': int(dt.timestamp() * 1000), 'type': 'TIMESTAMP'}
|
||||
|
||||
elif record_type == 'LeagueStructure':
|
||||
fields = record['fields']
|
||||
fields['structureId'] = {'value': data['id'], 'type': 'STRING'}
|
||||
fields['sport'] = {'value': data['sport'].upper(), 'type': 'STRING'}
|
||||
fields['type'] = {'value': data['type'], 'type': 'STRING'}
|
||||
fields['name'] = {'value': data.get('name', ''), 'type': 'STRING'}
|
||||
fields['abbreviation'] = {'value': data.get('abbreviation', ''), 'type': 'STRING'}
|
||||
fields['parentId'] = {'value': data.get('parentId', ''), 'type': 'STRING'}
|
||||
fields['displayOrder'] = {'value': data.get('displayOrder', 0), 'type': 'INT64'}
|
||||
|
||||
return record
|
||||
|
||||
def test_connection(self) -> bool:
|
||||
"""
|
||||
Test the CloudKit connection.
|
||||
"""
|
||||
try:
|
||||
# Try to fetch a small query
|
||||
self.fetch_records('Team', limit=1)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
120
cloudkit/migrations/0001_initial.py
Normal file
120
cloudkit/migrations/0001_initial.py
Normal file
@@ -0,0 +1,120 @@
|
||||
# Generated by Django 5.1.15 on 2026-01-26 08:59
|
||||
|
||||
import django.db.models.deletion
|
||||
import simple_history.models
|
||||
from django.conf import settings
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
initial = True
|
||||
|
||||
dependencies = [
|
||||
('core', '0001_initial'),
|
||||
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name='CloudKitConfiguration',
|
||||
fields=[
|
||||
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
||||
('name', models.CharField(help_text='Configuration name (e.g., "Production", "Development")', max_length=100, unique=True)),
|
||||
('environment', models.CharField(choices=[('development', 'Development'), ('production', 'Production')], default='development', max_length=20)),
|
||||
('container_id', models.CharField(default='iCloud.com.sportstime.app', help_text='CloudKit container ID (e.g., iCloud.com.sportstime.app)', max_length=200)),
|
||||
('key_id', models.CharField(blank=True, help_text='CloudKit API key ID', max_length=200)),
|
||||
('private_key', models.TextField(blank=True, help_text='EC P-256 private key content (PEM format). Paste key here OR use path below.')),
|
||||
('private_key_path', models.CharField(blank=True, help_text='Path to EC P-256 private key file (alternative to pasting key above)', max_length=500)),
|
||||
('is_active', models.BooleanField(default=False, help_text='Whether this configuration is active for syncing')),
|
||||
('batch_size', models.PositiveIntegerField(default=200, help_text='Maximum records per batch upload')),
|
||||
('auto_sync_after_scrape', models.BooleanField(default=False, help_text='Automatically sync after scraper jobs complete')),
|
||||
('created_at', models.DateTimeField(auto_now_add=True)),
|
||||
('updated_at', models.DateTimeField(auto_now=True)),
|
||||
],
|
||||
options={
|
||||
'verbose_name': 'CloudKit Configuration',
|
||||
'verbose_name_plural': 'CloudKit Configurations',
|
||||
},
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name='CloudKitSyncJob',
|
||||
fields=[
|
||||
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
||||
('status', models.CharField(choices=[('pending', 'Pending'), ('running', 'Running'), ('completed', 'Completed'), ('failed', 'Failed'), ('cancelled', 'Cancelled')], default='pending', max_length=20)),
|
||||
('triggered_by', models.CharField(default='manual', help_text='How the sync was triggered', max_length=50)),
|
||||
('started_at', models.DateTimeField(blank=True, null=True)),
|
||||
('finished_at', models.DateTimeField(blank=True, null=True)),
|
||||
('records_synced', models.PositiveIntegerField(default=0)),
|
||||
('records_created', models.PositiveIntegerField(default=0)),
|
||||
('records_updated', models.PositiveIntegerField(default=0)),
|
||||
('records_deleted', models.PositiveIntegerField(default=0)),
|
||||
('records_failed', models.PositiveIntegerField(default=0)),
|
||||
('record_type_filter', models.CharField(blank=True, help_text='Only sync this record type (all if blank)', max_length=20)),
|
||||
('error_message', models.TextField(blank=True)),
|
||||
('celery_task_id', models.CharField(blank=True, max_length=255)),
|
||||
('created_at', models.DateTimeField(auto_now_add=True)),
|
||||
('updated_at', models.DateTimeField(auto_now=True)),
|
||||
('configuration', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='sync_jobs', to='cloudkit.cloudkitconfiguration')),
|
||||
('sport_filter', models.ForeignKey(blank=True, help_text='Only sync this sport (all if blank)', null=True, on_delete=django.db.models.deletion.SET_NULL, to='core.sport')),
|
||||
],
|
||||
options={
|
||||
'verbose_name': 'CloudKit Sync Job',
|
||||
'verbose_name_plural': 'CloudKit Sync Jobs',
|
||||
'ordering': ['-created_at'],
|
||||
},
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name='CloudKitSyncState',
|
||||
fields=[
|
||||
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
||||
('record_type', models.CharField(choices=[('Game', 'Game'), ('Team', 'Team'), ('Stadium', 'Stadium')], max_length=20)),
|
||||
('record_id', models.CharField(help_text='Local record ID (canonical ID)', max_length=100)),
|
||||
('cloudkit_record_name', models.CharField(blank=True, help_text='CloudKit record name (may differ from local ID)', max_length=200)),
|
||||
('local_hash', models.CharField(blank=True, help_text='Hash of local record data for change detection', max_length=64)),
|
||||
('remote_change_tag', models.CharField(blank=True, help_text='CloudKit change tag for conflict detection', max_length=200)),
|
||||
('sync_status', models.CharField(choices=[('pending', 'Pending Sync'), ('synced', 'Synced'), ('failed', 'Failed'), ('deleted', 'Deleted')], default='pending', max_length=20)),
|
||||
('last_synced', models.DateTimeField(blank=True, null=True)),
|
||||
('last_error', models.TextField(blank=True, help_text='Last sync error message')),
|
||||
('retry_count', models.PositiveSmallIntegerField(default=0)),
|
||||
('created_at', models.DateTimeField(auto_now_add=True)),
|
||||
('updated_at', models.DateTimeField(auto_now=True)),
|
||||
],
|
||||
options={
|
||||
'verbose_name': 'CloudKit Sync State',
|
||||
'verbose_name_plural': 'CloudKit Sync States',
|
||||
'ordering': ['-updated_at'],
|
||||
'indexes': [models.Index(fields=['sync_status', 'record_type'], name='cloudkit_cl_sync_st_cc8bf6_idx'), models.Index(fields=['record_type', 'last_synced'], name='cloudkit_cl_record__d82278_idx')],
|
||||
'unique_together': {('record_type', 'record_id')},
|
||||
},
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name='HistoricalCloudKitConfiguration',
|
||||
fields=[
|
||||
('id', models.BigIntegerField(auto_created=True, blank=True, db_index=True, verbose_name='ID')),
|
||||
('name', models.CharField(db_index=True, help_text='Configuration name (e.g., "Production", "Development")', max_length=100)),
|
||||
('environment', models.CharField(choices=[('development', 'Development'), ('production', 'Production')], default='development', max_length=20)),
|
||||
('container_id', models.CharField(default='iCloud.com.sportstime.app', help_text='CloudKit container ID (e.g., iCloud.com.sportstime.app)', max_length=200)),
|
||||
('key_id', models.CharField(blank=True, help_text='CloudKit API key ID', max_length=200)),
|
||||
('private_key', models.TextField(blank=True, help_text='EC P-256 private key content (PEM format). Paste key here OR use path below.')),
|
||||
('private_key_path', models.CharField(blank=True, help_text='Path to EC P-256 private key file (alternative to pasting key above)', max_length=500)),
|
||||
('is_active', models.BooleanField(default=False, help_text='Whether this configuration is active for syncing')),
|
||||
('batch_size', models.PositiveIntegerField(default=200, help_text='Maximum records per batch upload')),
|
||||
('auto_sync_after_scrape', models.BooleanField(default=False, help_text='Automatically sync after scraper jobs complete')),
|
||||
('created_at', models.DateTimeField(blank=True, editable=False)),
|
||||
('updated_at', models.DateTimeField(blank=True, editable=False)),
|
||||
('history_id', models.AutoField(primary_key=True, serialize=False)),
|
||||
('history_date', models.DateTimeField(db_index=True)),
|
||||
('history_change_reason', models.CharField(max_length=100, null=True)),
|
||||
('history_type', models.CharField(choices=[('+', 'Created'), ('~', 'Changed'), ('-', 'Deleted')], max_length=1)),
|
||||
('history_user', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to=settings.AUTH_USER_MODEL)),
|
||||
],
|
||||
options={
|
||||
'verbose_name': 'historical CloudKit Configuration',
|
||||
'verbose_name_plural': 'historical CloudKit Configurations',
|
||||
'ordering': ('-history_date', '-history_id'),
|
||||
'get_latest_by': ('history_date', 'history_id'),
|
||||
},
|
||||
bases=(simple_history.models.HistoricalChanges, models.Model),
|
||||
),
|
||||
]
|
||||
63
cloudkit/migrations/0002_add_sync_progress_fields.py
Normal file
63
cloudkit/migrations/0002_add_sync_progress_fields.py
Normal file
@@ -0,0 +1,63 @@
|
||||
# Generated by Django 5.1.15 on 2026-01-26 13:46
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('cloudkit', '0001_initial'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name='cloudkitsyncjob',
|
||||
name='current_record_type',
|
||||
field=models.CharField(blank=True, help_text='Currently syncing record type', max_length=20),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='cloudkitsyncjob',
|
||||
name='games_failed',
|
||||
field=models.PositiveIntegerField(default=0),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='cloudkitsyncjob',
|
||||
name='games_synced',
|
||||
field=models.PositiveIntegerField(default=0),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='cloudkitsyncjob',
|
||||
name='games_total',
|
||||
field=models.PositiveIntegerField(default=0),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='cloudkitsyncjob',
|
||||
name='stadiums_failed',
|
||||
field=models.PositiveIntegerField(default=0),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='cloudkitsyncjob',
|
||||
name='stadiums_synced',
|
||||
field=models.PositiveIntegerField(default=0),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='cloudkitsyncjob',
|
||||
name='stadiums_total',
|
||||
field=models.PositiveIntegerField(default=0),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='cloudkitsyncjob',
|
||||
name='teams_failed',
|
||||
field=models.PositiveIntegerField(default=0),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='cloudkitsyncjob',
|
||||
name='teams_synced',
|
||||
field=models.PositiveIntegerField(default=0),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='cloudkitsyncjob',
|
||||
name='teams_total',
|
||||
field=models.PositiveIntegerField(default=0),
|
||||
),
|
||||
]
|
||||
29
cloudkit/migrations/0003_alter_cloudkitsyncjob_status.py
Normal file
29
cloudkit/migrations/0003_alter_cloudkitsyncjob_status.py
Normal file
@@ -0,0 +1,29 @@
|
||||
# Generated manually
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('cloudkit', '0002_add_sync_progress_fields'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name='cloudkitsyncjob',
|
||||
name='status',
|
||||
field=models.CharField(
|
||||
choices=[
|
||||
('pending', 'Pending'),
|
||||
('running', 'Running'),
|
||||
('completed', 'Completed'),
|
||||
('completed_with_errors', 'Completed with Errors'),
|
||||
('failed', 'Failed'),
|
||||
('cancelled', 'Cancelled'),
|
||||
],
|
||||
default='pending',
|
||||
max_length=25,
|
||||
),
|
||||
),
|
||||
]
|
||||
28
cloudkit/migrations/0004_cloudkitsyncjob_sport_progress.py
Normal file
28
cloudkit/migrations/0004_cloudkitsyncjob_sport_progress.py
Normal file
@@ -0,0 +1,28 @@
|
||||
# Generated manually
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('cloudkit', '0003_alter_cloudkitsyncjob_status'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name='cloudkitsyncjob',
|
||||
name='sports_total',
|
||||
field=models.PositiveIntegerField(default=0),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='cloudkitsyncjob',
|
||||
name='sports_synced',
|
||||
field=models.PositiveIntegerField(default=0),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='cloudkitsyncjob',
|
||||
name='sports_failed',
|
||||
field=models.PositiveIntegerField(default=0),
|
||||
),
|
||||
]
|
||||
@@ -0,0 +1,78 @@
|
||||
# Generated by Django 5.1.4 on 2026-02-06 02:21
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('cloudkit', '0004_cloudkitsyncjob_sport_progress'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name='cloudkitsyncjob',
|
||||
name='conferences_failed',
|
||||
field=models.PositiveIntegerField(default=0),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='cloudkitsyncjob',
|
||||
name='conferences_synced',
|
||||
field=models.PositiveIntegerField(default=0),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='cloudkitsyncjob',
|
||||
name='conferences_total',
|
||||
field=models.PositiveIntegerField(default=0),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='cloudkitsyncjob',
|
||||
name='divisions_failed',
|
||||
field=models.PositiveIntegerField(default=0),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='cloudkitsyncjob',
|
||||
name='divisions_synced',
|
||||
field=models.PositiveIntegerField(default=0),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='cloudkitsyncjob',
|
||||
name='divisions_total',
|
||||
field=models.PositiveIntegerField(default=0),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='cloudkitsyncjob',
|
||||
name='stadium_aliases_failed',
|
||||
field=models.PositiveIntegerField(default=0),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='cloudkitsyncjob',
|
||||
name='stadium_aliases_synced',
|
||||
field=models.PositiveIntegerField(default=0),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='cloudkitsyncjob',
|
||||
name='stadium_aliases_total',
|
||||
field=models.PositiveIntegerField(default=0),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='cloudkitsyncjob',
|
||||
name='team_aliases_failed',
|
||||
field=models.PositiveIntegerField(default=0),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='cloudkitsyncjob',
|
||||
name='team_aliases_synced',
|
||||
field=models.PositiveIntegerField(default=0),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='cloudkitsyncjob',
|
||||
name='team_aliases_total',
|
||||
field=models.PositiveIntegerField(default=0),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='cloudkitsyncstate',
|
||||
name='record_type',
|
||||
field=models.CharField(choices=[('Sport', 'Sport'), ('Conference', 'Conference'), ('Division', 'Division'), ('Team', 'Team'), ('Stadium', 'Stadium'), ('TeamAlias', 'Team Alias'), ('StadiumAlias', 'Stadium Alias'), ('Game', 'Game')], max_length=20),
|
||||
),
|
||||
]
|
||||
0
cloudkit/migrations/__init__.py
Normal file
0
cloudkit/migrations/__init__.py
Normal file
394
cloudkit/models.py
Normal file
394
cloudkit/models.py
Normal file
@@ -0,0 +1,394 @@
|
||||
from django.db import models
|
||||
from django.conf import settings
|
||||
from simple_history.models import HistoricalRecords
|
||||
|
||||
|
||||
class CloudKitConfiguration(models.Model):
|
||||
"""
|
||||
CloudKit configuration for syncing.
|
||||
"""
|
||||
ENVIRONMENT_CHOICES = [
|
||||
('development', 'Development'),
|
||||
('production', 'Production'),
|
||||
]
|
||||
|
||||
name = models.CharField(
|
||||
max_length=100,
|
||||
unique=True,
|
||||
help_text='Configuration name (e.g., "Production", "Development")'
|
||||
)
|
||||
environment = models.CharField(
|
||||
max_length=20,
|
||||
choices=ENVIRONMENT_CHOICES,
|
||||
default='development'
|
||||
)
|
||||
container_id = models.CharField(
|
||||
max_length=200,
|
||||
default=settings.CLOUDKIT_CONTAINER,
|
||||
help_text='CloudKit container ID (e.g., iCloud.com.sportstime.app)'
|
||||
)
|
||||
key_id = models.CharField(
|
||||
max_length=200,
|
||||
blank=True,
|
||||
help_text='CloudKit API key ID'
|
||||
)
|
||||
private_key = models.TextField(
|
||||
blank=True,
|
||||
help_text='EC P-256 private key content (PEM format). Paste key here OR use path below.'
|
||||
)
|
||||
private_key_path = models.CharField(
|
||||
max_length=500,
|
||||
blank=True,
|
||||
help_text='Path to EC P-256 private key file (alternative to pasting key above)'
|
||||
)
|
||||
is_active = models.BooleanField(
|
||||
default=False,
|
||||
help_text='Whether this configuration is active for syncing'
|
||||
)
|
||||
|
||||
# Sync settings
|
||||
batch_size = models.PositiveIntegerField(
|
||||
default=200,
|
||||
help_text='Maximum records per batch upload'
|
||||
)
|
||||
auto_sync_after_scrape = models.BooleanField(
|
||||
default=False,
|
||||
help_text='Automatically sync after scraper jobs complete'
|
||||
)
|
||||
|
||||
# Metadata
|
||||
created_at = models.DateTimeField(auto_now_add=True)
|
||||
updated_at = models.DateTimeField(auto_now=True)
|
||||
|
||||
# Audit trail
|
||||
history = HistoricalRecords()
|
||||
|
||||
class Meta:
|
||||
verbose_name = 'CloudKit Configuration'
|
||||
verbose_name_plural = 'CloudKit Configurations'
|
||||
|
||||
def __str__(self):
|
||||
return f"{self.name} ({self.environment})"
|
||||
|
||||
def save(self, *args, **kwargs):
|
||||
# Ensure only one active configuration
|
||||
if self.is_active:
|
||||
CloudKitConfiguration.objects.filter(is_active=True).exclude(pk=self.pk).update(is_active=False)
|
||||
super().save(*args, **kwargs)
|
||||
|
||||
def get_client(self):
|
||||
"""Create a CloudKitClient from this configuration."""
|
||||
from cloudkit.client import CloudKitClient
|
||||
return CloudKitClient(
|
||||
container_id=self.container_id,
|
||||
environment=self.environment,
|
||||
key_id=self.key_id,
|
||||
private_key=self.private_key,
|
||||
private_key_path=self.private_key_path,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_active(cls):
|
||||
"""Get the active CloudKit configuration."""
|
||||
return cls.objects.filter(is_active=True).first()
|
||||
|
||||
|
||||
class CloudKitSyncState(models.Model):
|
||||
"""
|
||||
Tracks sync state for individual records.
|
||||
"""
|
||||
RECORD_TYPE_CHOICES = [
|
||||
('Sport', 'Sport'),
|
||||
('Conference', 'Conference'),
|
||||
('Division', 'Division'),
|
||||
('Team', 'Team'),
|
||||
('Stadium', 'Stadium'),
|
||||
('TeamAlias', 'Team Alias'),
|
||||
('StadiumAlias', 'Stadium Alias'),
|
||||
('Game', 'Game'),
|
||||
]
|
||||
|
||||
SYNC_STATUS_CHOICES = [
|
||||
('pending', 'Pending Sync'),
|
||||
('synced', 'Synced'),
|
||||
('failed', 'Failed'),
|
||||
('deleted', 'Deleted'),
|
||||
]
|
||||
|
||||
record_type = models.CharField(
|
||||
max_length=20,
|
||||
choices=RECORD_TYPE_CHOICES
|
||||
)
|
||||
record_id = models.CharField(
|
||||
max_length=100,
|
||||
help_text='Local record ID (canonical ID)'
|
||||
)
|
||||
cloudkit_record_name = models.CharField(
|
||||
max_length=200,
|
||||
blank=True,
|
||||
help_text='CloudKit record name (may differ from local ID)'
|
||||
)
|
||||
local_hash = models.CharField(
|
||||
max_length=64,
|
||||
blank=True,
|
||||
help_text='Hash of local record data for change detection'
|
||||
)
|
||||
remote_change_tag = models.CharField(
|
||||
max_length=200,
|
||||
blank=True,
|
||||
help_text='CloudKit change tag for conflict detection'
|
||||
)
|
||||
sync_status = models.CharField(
|
||||
max_length=20,
|
||||
choices=SYNC_STATUS_CHOICES,
|
||||
default='pending'
|
||||
)
|
||||
last_synced = models.DateTimeField(
|
||||
null=True,
|
||||
blank=True
|
||||
)
|
||||
last_error = models.TextField(
|
||||
blank=True,
|
||||
help_text='Last sync error message'
|
||||
)
|
||||
retry_count = models.PositiveSmallIntegerField(
|
||||
default=0
|
||||
)
|
||||
|
||||
# Metadata
|
||||
created_at = models.DateTimeField(auto_now_add=True)
|
||||
updated_at = models.DateTimeField(auto_now=True)
|
||||
|
||||
class Meta:
|
||||
ordering = ['-updated_at']
|
||||
unique_together = ['record_type', 'record_id']
|
||||
verbose_name = 'CloudKit Sync State'
|
||||
verbose_name_plural = 'CloudKit Sync States'
|
||||
indexes = [
|
||||
models.Index(fields=['sync_status', 'record_type']),
|
||||
models.Index(fields=['record_type', 'last_synced']),
|
||||
]
|
||||
|
||||
def __str__(self):
|
||||
return f"{self.record_type}:{self.record_id} ({self.sync_status})"
|
||||
|
||||
def mark_synced(self, change_tag=''):
|
||||
"""Mark record as successfully synced."""
|
||||
from django.utils import timezone
|
||||
self.sync_status = 'synced'
|
||||
self.remote_change_tag = change_tag
|
||||
self.last_synced = timezone.now()
|
||||
self.last_error = ''
|
||||
self.retry_count = 0
|
||||
self.save()
|
||||
|
||||
def mark_failed(self, error_message):
|
||||
"""Mark record as failed to sync."""
|
||||
self.sync_status = 'failed'
|
||||
self.last_error = error_message
|
||||
self.retry_count += 1
|
||||
self.save()
|
||||
|
||||
def mark_pending(self, new_hash=''):
|
||||
"""Mark record as pending sync (e.g., after local change)."""
|
||||
self.sync_status = 'pending'
|
||||
if new_hash:
|
||||
self.local_hash = new_hash
|
||||
self.save()
|
||||
|
||||
|
||||
class CloudKitSyncJob(models.Model):
|
||||
"""
|
||||
Record of a CloudKit sync job execution.
|
||||
"""
|
||||
STATUS_CHOICES = [
|
||||
('pending', 'Pending'),
|
||||
('running', 'Running'),
|
||||
('completed', 'Completed'),
|
||||
('completed_with_errors', 'Completed with Errors'),
|
||||
('failed', 'Failed'),
|
||||
('cancelled', 'Cancelled'),
|
||||
]
|
||||
|
||||
configuration = models.ForeignKey(
|
||||
CloudKitConfiguration,
|
||||
on_delete=models.CASCADE,
|
||||
related_name='sync_jobs'
|
||||
)
|
||||
status = models.CharField(
|
||||
max_length=25,
|
||||
choices=STATUS_CHOICES,
|
||||
default='pending'
|
||||
)
|
||||
triggered_by = models.CharField(
|
||||
max_length=50,
|
||||
default='manual',
|
||||
help_text='How the sync was triggered'
|
||||
)
|
||||
|
||||
# Timing
|
||||
started_at = models.DateTimeField(null=True, blank=True)
|
||||
finished_at = models.DateTimeField(null=True, blank=True)
|
||||
|
||||
# Results
|
||||
records_synced = models.PositiveIntegerField(default=0)
|
||||
records_created = models.PositiveIntegerField(default=0)
|
||||
records_updated = models.PositiveIntegerField(default=0)
|
||||
records_deleted = models.PositiveIntegerField(default=0)
|
||||
records_failed = models.PositiveIntegerField(default=0)
|
||||
|
||||
# Filter (optional - sync specific records)
|
||||
sport_filter = models.ForeignKey(
|
||||
'core.Sport',
|
||||
on_delete=models.SET_NULL,
|
||||
null=True,
|
||||
blank=True,
|
||||
help_text='Only sync this sport (all if blank)'
|
||||
)
|
||||
record_type_filter = models.CharField(
|
||||
max_length=20,
|
||||
blank=True,
|
||||
help_text='Only sync this record type (all if blank)'
|
||||
)
|
||||
|
||||
# Error tracking
|
||||
error_message = models.TextField(blank=True)
|
||||
|
||||
# Progress tracking
|
||||
current_record_type = models.CharField(
|
||||
max_length=20,
|
||||
blank=True,
|
||||
help_text='Currently syncing record type'
|
||||
)
|
||||
sports_total = models.PositiveIntegerField(default=0)
|
||||
sports_synced = models.PositiveIntegerField(default=0)
|
||||
sports_failed = models.PositiveIntegerField(default=0)
|
||||
teams_total = models.PositiveIntegerField(default=0)
|
||||
teams_synced = models.PositiveIntegerField(default=0)
|
||||
teams_failed = models.PositiveIntegerField(default=0)
|
||||
stadiums_total = models.PositiveIntegerField(default=0)
|
||||
stadiums_synced = models.PositiveIntegerField(default=0)
|
||||
stadiums_failed = models.PositiveIntegerField(default=0)
|
||||
conferences_total = models.PositiveIntegerField(default=0)
|
||||
conferences_synced = models.PositiveIntegerField(default=0)
|
||||
conferences_failed = models.PositiveIntegerField(default=0)
|
||||
divisions_total = models.PositiveIntegerField(default=0)
|
||||
divisions_synced = models.PositiveIntegerField(default=0)
|
||||
divisions_failed = models.PositiveIntegerField(default=0)
|
||||
team_aliases_total = models.PositiveIntegerField(default=0)
|
||||
team_aliases_synced = models.PositiveIntegerField(default=0)
|
||||
team_aliases_failed = models.PositiveIntegerField(default=0)
|
||||
stadium_aliases_total = models.PositiveIntegerField(default=0)
|
||||
stadium_aliases_synced = models.PositiveIntegerField(default=0)
|
||||
stadium_aliases_failed = models.PositiveIntegerField(default=0)
|
||||
games_total = models.PositiveIntegerField(default=0)
|
||||
games_synced = models.PositiveIntegerField(default=0)
|
||||
games_failed = models.PositiveIntegerField(default=0)
|
||||
|
||||
# Celery task ID
|
||||
celery_task_id = models.CharField(
|
||||
max_length=255,
|
||||
blank=True
|
||||
)
|
||||
|
||||
# Metadata
|
||||
created_at = models.DateTimeField(auto_now_add=True)
|
||||
updated_at = models.DateTimeField(auto_now=True)
|
||||
|
||||
class Meta:
|
||||
ordering = ['-created_at']
|
||||
verbose_name = 'CloudKit Sync Job'
|
||||
verbose_name_plural = 'CloudKit Sync Jobs'
|
||||
|
||||
def __str__(self):
|
||||
return f"Sync {self.configuration.name} - {self.created_at.strftime('%Y-%m-%d %H:%M')}"
|
||||
|
||||
@property
|
||||
def duration(self):
|
||||
if self.started_at and self.finished_at:
|
||||
return self.finished_at - self.started_at
|
||||
return None
|
||||
|
||||
@property
|
||||
def duration_display(self):
|
||||
duration = self.duration
|
||||
if duration:
|
||||
total_seconds = int(duration.total_seconds())
|
||||
minutes, seconds = divmod(total_seconds, 60)
|
||||
if minutes > 0:
|
||||
return f"{minutes}m {seconds}s"
|
||||
return f"{seconds}s"
|
||||
return '-'
|
||||
|
||||
def get_progress(self):
|
||||
"""Get progress data for API/display."""
|
||||
total = (self.sports_total + self.conferences_total + self.divisions_total
|
||||
+ self.teams_total + self.stadiums_total
|
||||
+ self.team_aliases_total + self.stadium_aliases_total
|
||||
+ self.games_total)
|
||||
synced = (self.sports_synced + self.conferences_synced + self.divisions_synced
|
||||
+ self.teams_synced + self.stadiums_synced
|
||||
+ self.team_aliases_synced + self.stadium_aliases_synced
|
||||
+ self.games_synced)
|
||||
failed = (self.sports_failed + self.conferences_failed + self.divisions_failed
|
||||
+ self.teams_failed + self.stadiums_failed
|
||||
+ self.team_aliases_failed + self.stadium_aliases_failed
|
||||
+ self.games_failed)
|
||||
|
||||
return {
|
||||
'status': self.status,
|
||||
'current_type': self.current_record_type,
|
||||
'total': total,
|
||||
'synced': synced,
|
||||
'failed': failed,
|
||||
'remaining': total - synced - failed,
|
||||
'percent': round((synced + failed) / total * 100) if total > 0 else 0,
|
||||
'sports': {
|
||||
'total': self.sports_total,
|
||||
'synced': self.sports_synced,
|
||||
'failed': self.sports_failed,
|
||||
'remaining': self.sports_total - self.sports_synced - self.sports_failed,
|
||||
},
|
||||
'conferences': {
|
||||
'total': self.conferences_total,
|
||||
'synced': self.conferences_synced,
|
||||
'failed': self.conferences_failed,
|
||||
'remaining': self.conferences_total - self.conferences_synced - self.conferences_failed,
|
||||
},
|
||||
'divisions': {
|
||||
'total': self.divisions_total,
|
||||
'synced': self.divisions_synced,
|
||||
'failed': self.divisions_failed,
|
||||
'remaining': self.divisions_total - self.divisions_synced - self.divisions_failed,
|
||||
},
|
||||
'teams': {
|
||||
'total': self.teams_total,
|
||||
'synced': self.teams_synced,
|
||||
'failed': self.teams_failed,
|
||||
'remaining': self.teams_total - self.teams_synced - self.teams_failed,
|
||||
},
|
||||
'stadiums': {
|
||||
'total': self.stadiums_total,
|
||||
'synced': self.stadiums_synced,
|
||||
'failed': self.stadiums_failed,
|
||||
'remaining': self.stadiums_total - self.stadiums_synced - self.stadiums_failed,
|
||||
},
|
||||
'team_aliases': {
|
||||
'total': self.team_aliases_total,
|
||||
'synced': self.team_aliases_synced,
|
||||
'failed': self.team_aliases_failed,
|
||||
'remaining': self.team_aliases_total - self.team_aliases_synced - self.team_aliases_failed,
|
||||
},
|
||||
'stadium_aliases': {
|
||||
'total': self.stadium_aliases_total,
|
||||
'synced': self.stadium_aliases_synced,
|
||||
'failed': self.stadium_aliases_failed,
|
||||
'remaining': self.stadium_aliases_total - self.stadium_aliases_synced - self.stadium_aliases_failed,
|
||||
},
|
||||
'games': {
|
||||
'total': self.games_total,
|
||||
'synced': self.games_synced,
|
||||
'failed': self.games_failed,
|
||||
'remaining': self.games_total - self.games_synced - self.games_failed,
|
||||
},
|
||||
}
|
||||
49
cloudkit/resources.py
Normal file
49
cloudkit/resources.py
Normal file
@@ -0,0 +1,49 @@
|
||||
"""Import/Export resources for cloudkit models."""
|
||||
from import_export import resources, fields
|
||||
from import_export.widgets import ForeignKeyWidget
|
||||
|
||||
from .models import CloudKitConfiguration, CloudKitSyncState, CloudKitSyncJob
|
||||
|
||||
|
||||
class CloudKitConfigurationResource(resources.ModelResource):
|
||||
class Meta:
|
||||
model = CloudKitConfiguration
|
||||
import_id_fields = ['name']
|
||||
fields = [
|
||||
'name', 'environment', 'container_id', 'key_id',
|
||||
'is_active', 'batch_size', 'auto_sync_after_scrape',
|
||||
]
|
||||
export_order = fields
|
||||
# Exclude private_key for security
|
||||
exclude = ['private_key', 'private_key_path']
|
||||
|
||||
|
||||
class CloudKitSyncStateResource(resources.ModelResource):
|
||||
class Meta:
|
||||
model = CloudKitSyncState
|
||||
import_id_fields = ['record_type', 'record_id']
|
||||
fields = [
|
||||
'record_type', 'record_id', 'cloudkit_record_name',
|
||||
'sync_status', 'local_hash', 'remote_change_tag',
|
||||
'last_synced', 'last_error', 'retry_count',
|
||||
]
|
||||
export_order = fields
|
||||
|
||||
|
||||
class CloudKitSyncJobResource(resources.ModelResource):
|
||||
configuration = fields.Field(
|
||||
column_name='configuration',
|
||||
attribute='configuration',
|
||||
widget=ForeignKeyWidget(CloudKitConfiguration, 'name')
|
||||
)
|
||||
|
||||
class Meta:
|
||||
model = CloudKitSyncJob
|
||||
fields = [
|
||||
'id', 'configuration', 'status', 'triggered_by',
|
||||
'started_at', 'finished_at',
|
||||
'records_synced', 'records_created', 'records_updated',
|
||||
'records_deleted', 'records_failed',
|
||||
'error_message', 'created_at',
|
||||
]
|
||||
export_order = fields
|
||||
701
cloudkit/tasks.py
Normal file
701
cloudkit/tasks.py
Normal file
@@ -0,0 +1,701 @@
|
||||
import logging
|
||||
import traceback
|
||||
|
||||
from celery import shared_task
|
||||
from django.utils import timezone
|
||||
|
||||
logger = logging.getLogger('cloudkit')
|
||||
|
||||
|
||||
@shared_task(bind=True, max_retries=3)
|
||||
def run_cloudkit_sync(self, config_id: int, triggered_by: str = 'manual',
|
||||
sport_code: str = None, record_type: str = None):
|
||||
"""
|
||||
Run a CloudKit sync job.
|
||||
"""
|
||||
from cloudkit.models import CloudKitConfiguration, CloudKitSyncJob, CloudKitSyncState
|
||||
from notifications.tasks import send_sync_notification
|
||||
|
||||
# Get configuration
|
||||
try:
|
||||
config = CloudKitConfiguration.objects.get(id=config_id)
|
||||
except CloudKitConfiguration.DoesNotExist:
|
||||
logger.error(f"CloudKitConfiguration {config_id} not found")
|
||||
return {'error': 'Configuration not found'}
|
||||
|
||||
# Create job record
|
||||
job = CloudKitSyncJob.objects.create(
|
||||
configuration=config,
|
||||
status='running',
|
||||
triggered_by=triggered_by,
|
||||
started_at=timezone.now(),
|
||||
celery_task_id=self.request.id,
|
||||
sport_filter_id=sport_code,
|
||||
record_type_filter=record_type or '',
|
||||
)
|
||||
|
||||
try:
|
||||
logger.info(f'Starting CloudKit sync to {config.environment}')
|
||||
|
||||
# Run sync
|
||||
result = perform_sync(config, job, sport_code, record_type)
|
||||
|
||||
# Update job with results
|
||||
job.finished_at = timezone.now()
|
||||
job.records_synced = result.get('synced', 0)
|
||||
job.records_created = result.get('created', 0)
|
||||
job.records_updated = result.get('updated', 0)
|
||||
job.records_deleted = result.get('deleted', 0)
|
||||
job.records_failed = result.get('failed', 0)
|
||||
|
||||
# Set status based on results
|
||||
if job.records_failed > 0 and job.records_synced == 0:
|
||||
job.status = 'failed'
|
||||
job.error_message = f'All {job.records_failed} records failed to sync'
|
||||
logger.error(f'Sync failed: {job.records_failed} failed, 0 synced')
|
||||
elif job.records_failed > 0:
|
||||
job.status = 'completed_with_errors'
|
||||
logger.warning(f'Sync completed with errors: {job.records_synced} synced, {job.records_failed} failed')
|
||||
else:
|
||||
job.status = 'completed'
|
||||
logger.info(f'Sync completed: {job.records_synced} synced')
|
||||
job.save()
|
||||
|
||||
# Send notification if configured
|
||||
send_sync_notification.delay(job.id)
|
||||
|
||||
return {
|
||||
'job_id': job.id,
|
||||
'status': 'completed',
|
||||
'records_synced': job.records_synced,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
error_msg = str(e)
|
||||
error_tb = traceback.format_exc()
|
||||
|
||||
job.status = 'failed'
|
||||
job.finished_at = timezone.now()
|
||||
job.error_message = error_msg
|
||||
job.save()
|
||||
|
||||
logger.error(f'Sync failed: {error_msg}')
|
||||
|
||||
# Send failure notification
|
||||
send_sync_notification.delay(job.id)
|
||||
|
||||
# Retry if applicable
|
||||
if self.request.retries < self.max_retries:
|
||||
raise self.retry(exc=e, countdown=60 * (self.request.retries + 1))
|
||||
|
||||
return {
|
||||
'job_id': job.id,
|
||||
'status': 'failed',
|
||||
'error': error_msg,
|
||||
}
|
||||
|
||||
|
||||
def perform_sync(config, job, sport_code=None, record_type=None):
|
||||
"""
|
||||
Perform the actual CloudKit sync.
|
||||
Syncs ALL local records to CloudKit (creates new, updates existing).
|
||||
"""
|
||||
from cloudkit.client import CloudKitClient
|
||||
from cloudkit.models import CloudKitSyncState
|
||||
from core.models import Sport, Conference, Division, Game, Team, Stadium, TeamAlias, StadiumAlias
|
||||
|
||||
# Initialize CloudKit client from config
|
||||
client = config.get_client()
|
||||
|
||||
# Test connection first
|
||||
try:
|
||||
client._get_token()
|
||||
except Exception as e:
|
||||
logger.error(f'CloudKit authentication failed: {e}')
|
||||
raise ValueError(f'CloudKit authentication failed: {e}')
|
||||
|
||||
results = {
|
||||
'synced': 0,
|
||||
'created': 0,
|
||||
'updated': 0,
|
||||
'deleted': 0,
|
||||
'failed': 0,
|
||||
}
|
||||
|
||||
batch_size = config.batch_size
|
||||
|
||||
# Sync Sports first (no dependencies)
|
||||
if not record_type or record_type == 'Sport':
|
||||
sports = Sport.objects.filter(is_active=True)
|
||||
job.sports_total = sports.count()
|
||||
job.current_record_type = 'Sport'
|
||||
job.save(update_fields=['sports_total', 'current_record_type'])
|
||||
|
||||
sport_results = sync_model_records(client, 'Sport', sports, sport_to_dict, batch_size, job)
|
||||
results['synced'] += sport_results['synced']
|
||||
results['failed'] += sport_results['failed']
|
||||
|
||||
# Sync Conferences (FK to Sport)
|
||||
if not record_type or record_type == 'Conference':
|
||||
conferences = Conference.objects.select_related('sport').all()
|
||||
job.conferences_total = conferences.count()
|
||||
job.current_record_type = 'Conference'
|
||||
job.save(update_fields=['conferences_total', 'current_record_type'])
|
||||
|
||||
conf_results = sync_model_records(client, 'Conference', conferences, conference_to_dict, batch_size, job)
|
||||
results['synced'] += conf_results['synced']
|
||||
results['failed'] += conf_results['failed']
|
||||
|
||||
# Sync Divisions (FK to Conference)
|
||||
if not record_type or record_type == 'Division':
|
||||
divisions = Division.objects.select_related('conference', 'conference__sport').all()
|
||||
job.divisions_total = divisions.count()
|
||||
job.current_record_type = 'Division'
|
||||
job.save(update_fields=['divisions_total', 'current_record_type'])
|
||||
|
||||
div_results = sync_model_records(client, 'Division', divisions, division_to_dict, batch_size, job)
|
||||
results['synced'] += div_results['synced']
|
||||
results['failed'] += div_results['failed']
|
||||
|
||||
# Sync Teams (dependencies for Games, TeamAliases)
|
||||
if not record_type or record_type == 'Team':
|
||||
teams = Team.objects.select_related('sport', 'home_stadium', 'division', 'division__conference').all()
|
||||
job.teams_total = teams.count()
|
||||
job.current_record_type = 'Team'
|
||||
job.save(update_fields=['teams_total', 'current_record_type'])
|
||||
|
||||
team_results = sync_model_records(client, 'Team', teams, team_to_dict, batch_size, job)
|
||||
results['synced'] += team_results['synced']
|
||||
results['failed'] += team_results['failed']
|
||||
|
||||
# Sync Stadiums (dependencies for Games, StadiumAliases)
|
||||
if not record_type or record_type == 'Stadium':
|
||||
stadiums = Stadium.objects.select_related('sport').all()
|
||||
job.stadiums_total = stadiums.count()
|
||||
job.current_record_type = 'Stadium'
|
||||
job.save(update_fields=['stadiums_total', 'current_record_type'])
|
||||
|
||||
stadium_results = sync_model_records(client, 'Stadium', stadiums, stadium_to_dict, batch_size, job)
|
||||
results['synced'] += stadium_results['synced']
|
||||
results['failed'] += stadium_results['failed']
|
||||
|
||||
# Sync TeamAliases (FK to Team)
|
||||
if not record_type or record_type == 'TeamAlias':
|
||||
team_aliases = TeamAlias.objects.select_related('team').all()
|
||||
job.team_aliases_total = team_aliases.count()
|
||||
job.current_record_type = 'TeamAlias'
|
||||
job.save(update_fields=['team_aliases_total', 'current_record_type'])
|
||||
|
||||
ta_results = sync_model_records(client, 'TeamAlias', team_aliases, team_alias_to_dict, batch_size, job)
|
||||
results['synced'] += ta_results['synced']
|
||||
results['failed'] += ta_results['failed']
|
||||
|
||||
# Sync StadiumAliases (FK to Stadium)
|
||||
if not record_type or record_type == 'StadiumAlias':
|
||||
stadium_aliases = StadiumAlias.objects.select_related('stadium').all()
|
||||
job.stadium_aliases_total = stadium_aliases.count()
|
||||
job.current_record_type = 'StadiumAlias'
|
||||
job.save(update_fields=['stadium_aliases_total', 'current_record_type'])
|
||||
|
||||
sa_results = sync_model_records(client, 'StadiumAlias', stadium_aliases, stadium_alias_to_dict, batch_size, job)
|
||||
results['synced'] += sa_results['synced']
|
||||
results['failed'] += sa_results['failed']
|
||||
|
||||
# Sync LeagueStructure (flattened hierarchy: league + conference + division)
|
||||
if not record_type or record_type == 'LeagueStructure':
|
||||
ls_records = build_league_structure_records()
|
||||
job.current_record_type = 'LeagueStructure'
|
||||
job.save(update_fields=['current_record_type'])
|
||||
|
||||
ls_results = sync_dict_records(client, 'LeagueStructure', ls_records, batch_size, job)
|
||||
results['synced'] += ls_results['synced']
|
||||
results['failed'] += ls_results['failed']
|
||||
|
||||
# Sync Games (depends on Teams, Stadiums)
|
||||
if not record_type or record_type == 'Game':
|
||||
games = Game.objects.select_related('home_team', 'away_team', 'stadium', 'sport').all()
|
||||
job.games_total = games.count()
|
||||
job.current_record_type = 'Game'
|
||||
job.save(update_fields=['games_total', 'current_record_type'])
|
||||
|
||||
game_results = sync_model_records(client, 'Game', games, game_to_dict, batch_size, job)
|
||||
results['synced'] += game_results['synced']
|
||||
results['failed'] += game_results['failed']
|
||||
|
||||
job.current_record_type = ''
|
||||
job.save(update_fields=['current_record_type'])
|
||||
return results
|
||||
|
||||
|
||||
def sync_model_records(client, record_type, queryset, to_dict_func, batch_size, job=None):
|
||||
"""
|
||||
Sync all records from a queryset to CloudKit.
|
||||
Updates progress frequently for real-time UI feedback.
|
||||
"""
|
||||
results = {'synced': 0, 'failed': 0}
|
||||
|
||||
records = list(queryset)
|
||||
total = len(records)
|
||||
|
||||
logger.info(f'[{record_type}] Starting sync: {total} total records')
|
||||
|
||||
# Field names for job updates
|
||||
field_map = {
|
||||
'Sport': ('sports_synced', 'sports_failed'),
|
||||
'Conference': ('conferences_synced', 'conferences_failed'),
|
||||
'Division': ('divisions_synced', 'divisions_failed'),
|
||||
'Team': ('teams_synced', 'teams_failed'),
|
||||
'Stadium': ('stadiums_synced', 'stadiums_failed'),
|
||||
'TeamAlias': ('team_aliases_synced', 'team_aliases_failed'),
|
||||
'StadiumAlias': ('stadium_aliases_synced', 'stadium_aliases_failed'),
|
||||
'Game': ('games_synced', 'games_failed'),
|
||||
}
|
||||
synced_field, failed_field = field_map.get(record_type, (None, None))
|
||||
|
||||
# Use smaller batches for more frequent progress updates
|
||||
# CloudKit API batch size vs progress update frequency
|
||||
api_batch_size = min(batch_size, 50) # Max 50 per API call for frequent updates
|
||||
progress_update_interval = 10 # Update DB every 10 records
|
||||
records_since_last_update = 0
|
||||
|
||||
for i in range(0, total, api_batch_size):
|
||||
batch = records[i:i + api_batch_size]
|
||||
batch_num = (i // api_batch_size) + 1
|
||||
total_batches = (total + api_batch_size - 1) // api_batch_size
|
||||
|
||||
# Convert to CloudKit format
|
||||
cloudkit_records = []
|
||||
for record in batch:
|
||||
try:
|
||||
data = to_dict_func(record)
|
||||
ck_record = client.to_cloudkit_record(record_type, data)
|
||||
cloudkit_records.append(ck_record)
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to convert {record_type}:{record.id}: {e}')
|
||||
results['failed'] += 1
|
||||
records_since_last_update += 1
|
||||
|
||||
if cloudkit_records:
|
||||
try:
|
||||
response = client.save_records(cloudkit_records)
|
||||
response_records = response.get('records', [])
|
||||
|
||||
batch_synced = 0
|
||||
batch_failed = 0
|
||||
for rec in response_records:
|
||||
if 'serverErrorCode' in rec:
|
||||
logger.error(f'CloudKit error for {rec.get("recordName")}: {rec.get("reason")}')
|
||||
results['failed'] += 1
|
||||
batch_failed += 1
|
||||
else:
|
||||
results['synced'] += 1
|
||||
batch_synced += 1
|
||||
records_since_last_update += 1
|
||||
|
||||
# Update progress frequently for real-time UI
|
||||
if job and synced_field and records_since_last_update >= progress_update_interval:
|
||||
setattr(job, synced_field, results['synced'])
|
||||
setattr(job, failed_field, results['failed'])
|
||||
job.save(update_fields=[synced_field, failed_field])
|
||||
records_since_last_update = 0
|
||||
|
||||
# Always update after each batch completes
|
||||
if job and synced_field:
|
||||
setattr(job, synced_field, results['synced'])
|
||||
setattr(job, failed_field, results['failed'])
|
||||
job.save(update_fields=[synced_field, failed_field])
|
||||
records_since_last_update = 0
|
||||
|
||||
# Log progress after each batch
|
||||
remaining = total - (results['synced'] + results['failed'])
|
||||
logger.info(
|
||||
f'[{record_type}] Batch {batch_num}/{total_batches}: '
|
||||
f'+{batch_synced} synced, +{batch_failed} failed | '
|
||||
f'Progress: {results["synced"]}/{total} synced, {remaining} remaining'
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f'Batch save failed: {e}')
|
||||
results['failed'] += len(cloudkit_records)
|
||||
|
||||
# Update job progress
|
||||
if job and failed_field:
|
||||
setattr(job, failed_field, results['failed'])
|
||||
job.save(update_fields=[failed_field])
|
||||
|
||||
remaining = total - (results['synced'] + results['failed'])
|
||||
logger.info(
|
||||
f'[{record_type}] Batch {batch_num}/{total_batches} FAILED | '
|
||||
f'Progress: {results["synced"]}/{total} synced, {remaining} remaining'
|
||||
)
|
||||
|
||||
logger.info(f'[{record_type}] Complete: {results["synced"]} synced, {results["failed"]} failed')
|
||||
return results
|
||||
|
||||
|
||||
def build_league_structure_records():
|
||||
"""Build flat LeagueStructure dicts from Sport, Conference, Division models."""
|
||||
from core.models import Sport, Conference, Division
|
||||
|
||||
records = []
|
||||
|
||||
for sport in Sport.objects.filter(is_active=True).order_by('code'):
|
||||
league_id = f'ls_{sport.code}_league'
|
||||
records.append({
|
||||
'id': league_id,
|
||||
'structureId': league_id,
|
||||
'sport': sport.code,
|
||||
'type': 'league',
|
||||
'name': sport.name,
|
||||
'abbreviation': sport.short_name,
|
||||
'parentId': '',
|
||||
'displayOrder': 0,
|
||||
})
|
||||
|
||||
for conf in Conference.objects.filter(sport=sport).order_by('order', 'name'):
|
||||
raw_conf_id = conf.canonical_id or f'conf_{conf.id}'
|
||||
conf_id = f'ls_{raw_conf_id}'
|
||||
records.append({
|
||||
'id': conf_id,
|
||||
'structureId': conf_id,
|
||||
'sport': sport.code,
|
||||
'type': 'conference',
|
||||
'name': conf.name,
|
||||
'abbreviation': conf.short_name or '',
|
||||
'parentId': league_id,
|
||||
'displayOrder': conf.order,
|
||||
})
|
||||
|
||||
for div in Division.objects.filter(conference=conf).order_by('order', 'name'):
|
||||
raw_div_id = div.canonical_id or f'div_{div.id}'
|
||||
div_id = f'ls_{raw_div_id}'
|
||||
records.append({
|
||||
'id': div_id,
|
||||
'structureId': div_id,
|
||||
'sport': sport.code,
|
||||
'type': 'division',
|
||||
'name': div.name,
|
||||
'abbreviation': div.short_name or '',
|
||||
'parentId': conf_id,
|
||||
'displayOrder': div.order,
|
||||
})
|
||||
|
||||
return records
|
||||
|
||||
|
||||
def sync_dict_records(client, record_type, dict_records, batch_size, job=None):
|
||||
"""Sync pre-built dict records to CloudKit (no model/queryset needed)."""
|
||||
results = {'synced': 0, 'failed': 0}
|
||||
total = len(dict_records)
|
||||
|
||||
logger.info(f'[{record_type}] Starting sync: {total} total records')
|
||||
|
||||
api_batch_size = min(batch_size, 50)
|
||||
|
||||
for i in range(0, total, api_batch_size):
|
||||
batch = dict_records[i:i + api_batch_size]
|
||||
batch_num = (i // api_batch_size) + 1
|
||||
total_batches = (total + api_batch_size - 1) // api_batch_size
|
||||
|
||||
cloudkit_records = []
|
||||
for data in batch:
|
||||
try:
|
||||
ck_record = client.to_cloudkit_record(record_type, data)
|
||||
cloudkit_records.append(ck_record)
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to convert {record_type}:{data.get("id")}: {e}')
|
||||
results['failed'] += 1
|
||||
|
||||
if cloudkit_records:
|
||||
try:
|
||||
response = client.save_records(cloudkit_records)
|
||||
batch_synced = 0
|
||||
batch_failed = 0
|
||||
for rec in response.get('records', []):
|
||||
if 'serverErrorCode' in rec:
|
||||
logger.error(f'CloudKit error for {rec.get("recordName")}: {rec.get("reason")}')
|
||||
results['failed'] += 1
|
||||
batch_failed += 1
|
||||
else:
|
||||
results['synced'] += 1
|
||||
batch_synced += 1
|
||||
|
||||
remaining = total - (results['synced'] + results['failed'])
|
||||
logger.info(
|
||||
f'[{record_type}] Batch {batch_num}/{total_batches}: '
|
||||
f'+{batch_synced} synced, +{batch_failed} failed | '
|
||||
f'Progress: {results["synced"]}/{total} synced, {remaining} remaining'
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f'Batch save failed: {e}')
|
||||
results['failed'] += len(cloudkit_records)
|
||||
|
||||
logger.info(f'[{record_type}] Complete: {results["synced"]} synced, {results["failed"]} failed')
|
||||
return results
|
||||
|
||||
|
||||
def sync_batch(client, states):
|
||||
"""
|
||||
Sync a batch of records to CloudKit.
|
||||
"""
|
||||
from core.models import Game, Team, Stadium
|
||||
|
||||
result = {'synced': 0, 'created': 0, 'updated': 0, 'failed': 0}
|
||||
|
||||
records_to_save = []
|
||||
|
||||
for state in states:
|
||||
try:
|
||||
# Get the local record
|
||||
record_data = get_record_data(state.record_type, state.record_id)
|
||||
if record_data:
|
||||
records_to_save.append({
|
||||
'state': state,
|
||||
'data': record_data,
|
||||
})
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to get record {state.record_type}:{state.record_id}: {e}')
|
||||
state.mark_failed(str(e))
|
||||
result['failed'] += 1
|
||||
|
||||
if records_to_save:
|
||||
# Convert to CloudKit format and upload
|
||||
cloudkit_records = [
|
||||
client.to_cloudkit_record(r['state'].record_type, r['data'])
|
||||
for r in records_to_save
|
||||
]
|
||||
|
||||
try:
|
||||
response = client.save_records(cloudkit_records)
|
||||
|
||||
for i, r in enumerate(records_to_save):
|
||||
if i < len(response.get('records', [])):
|
||||
change_tag = response['records'][i].get('recordChangeTag', '')
|
||||
r['state'].mark_synced(change_tag)
|
||||
result['synced'] += 1
|
||||
if r['state'].cloudkit_record_name:
|
||||
result['updated'] += 1
|
||||
else:
|
||||
result['created'] += 1
|
||||
else:
|
||||
r['state'].mark_failed('No response for record')
|
||||
result['failed'] += 1
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f'CloudKit save failed: {e}')
|
||||
for r in records_to_save:
|
||||
r['state'].mark_failed(str(e))
|
||||
result['failed'] += len(records_to_save)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def get_record_data(record_type, record_id):
|
||||
"""
|
||||
Get the local record data for a given type and ID.
|
||||
"""
|
||||
from core.models import Sport, Conference, Division, Game, Team, Stadium, TeamAlias, StadiumAlias
|
||||
|
||||
if record_type == 'Sport':
|
||||
try:
|
||||
sport = Sport.objects.get(code=record_id)
|
||||
return sport_to_dict(sport)
|
||||
except Sport.DoesNotExist:
|
||||
return None
|
||||
|
||||
elif record_type == 'Conference':
|
||||
try:
|
||||
conf = Conference.objects.select_related('sport').get(id=record_id)
|
||||
return conference_to_dict(conf)
|
||||
except Conference.DoesNotExist:
|
||||
return None
|
||||
|
||||
elif record_type == 'Division':
|
||||
try:
|
||||
div = Division.objects.select_related('conference', 'conference__sport').get(id=record_id)
|
||||
return division_to_dict(div)
|
||||
except Division.DoesNotExist:
|
||||
return None
|
||||
|
||||
elif record_type == 'Game':
|
||||
try:
|
||||
game = Game.objects.select_related(
|
||||
'home_team', 'away_team', 'stadium', 'sport'
|
||||
).get(id=record_id)
|
||||
return game_to_dict(game)
|
||||
except Game.DoesNotExist:
|
||||
return None
|
||||
|
||||
elif record_type == 'Team':
|
||||
try:
|
||||
team = Team.objects.select_related('sport', 'home_stadium').get(id=record_id)
|
||||
return team_to_dict(team)
|
||||
except Team.DoesNotExist:
|
||||
return None
|
||||
|
||||
elif record_type == 'Stadium':
|
||||
try:
|
||||
stadium = Stadium.objects.select_related('sport').get(id=record_id)
|
||||
return stadium_to_dict(stadium)
|
||||
except Stadium.DoesNotExist:
|
||||
return None
|
||||
|
||||
elif record_type == 'TeamAlias':
|
||||
try:
|
||||
alias = TeamAlias.objects.select_related('team').get(id=record_id)
|
||||
return team_alias_to_dict(alias)
|
||||
except TeamAlias.DoesNotExist:
|
||||
return None
|
||||
|
||||
elif record_type == 'StadiumAlias':
|
||||
try:
|
||||
alias = StadiumAlias.objects.select_related('stadium').get(id=record_id)
|
||||
return stadium_alias_to_dict(alias)
|
||||
except StadiumAlias.DoesNotExist:
|
||||
return None
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def sport_to_dict(sport):
|
||||
"""Convert Sport model to dict for CloudKit."""
|
||||
return {
|
||||
'id': sport.code,
|
||||
'abbreviation': sport.short_name,
|
||||
'displayName': sport.name,
|
||||
'iconName': sport.icon_name,
|
||||
'colorHex': sport.color_hex,
|
||||
'seasonStartMonth': sport.season_start_month,
|
||||
'seasonEndMonth': sport.season_end_month,
|
||||
'isActive': sport.is_active,
|
||||
}
|
||||
|
||||
|
||||
def game_to_dict(game):
|
||||
"""Convert Game model to dict for CloudKit."""
|
||||
return {
|
||||
'id': game.id,
|
||||
'sport': game.sport.code,
|
||||
'season': game.season,
|
||||
'homeTeamId': game.home_team_id,
|
||||
'awayTeamId': game.away_team_id,
|
||||
'stadiumId': game.stadium_id,
|
||||
'gameDate': game.game_date.isoformat(),
|
||||
'gameNumber': game.game_number,
|
||||
'homeScore': game.home_score,
|
||||
'awayScore': game.away_score,
|
||||
'status': game.status,
|
||||
'isNeutralSite': game.is_neutral_site,
|
||||
'isPlayoff': game.is_playoff,
|
||||
'playoffRound': game.playoff_round,
|
||||
}
|
||||
|
||||
|
||||
def team_to_dict(team):
|
||||
"""Convert Team model to dict for CloudKit."""
|
||||
division_id = None
|
||||
conference_id = None
|
||||
if team.division:
|
||||
division_id = team.division.canonical_id or f'div_{team.division.id}'
|
||||
conference_id = team.division.conference.canonical_id or f'conf_{team.division.conference.id}'
|
||||
return {
|
||||
'id': team.id,
|
||||
'sport': team.sport.code,
|
||||
'city': team.city,
|
||||
'name': team.name,
|
||||
'fullName': team.full_name,
|
||||
'abbreviation': team.abbreviation,
|
||||
'homeStadiumId': team.home_stadium_id,
|
||||
'primaryColor': team.primary_color,
|
||||
'secondaryColor': team.secondary_color,
|
||||
'logoUrl': team.logo_url,
|
||||
'divisionId': division_id,
|
||||
'conferenceId': conference_id,
|
||||
}
|
||||
|
||||
|
||||
def stadium_to_dict(stadium):
|
||||
"""Convert Stadium model to dict for CloudKit."""
|
||||
return {
|
||||
'id': stadium.id,
|
||||
'sport': stadium.sport.code,
|
||||
'name': stadium.name,
|
||||
'city': stadium.city,
|
||||
'state': stadium.state,
|
||||
'country': stadium.country,
|
||||
'latitude': float(stadium.latitude) if stadium.latitude else None,
|
||||
'longitude': float(stadium.longitude) if stadium.longitude else None,
|
||||
'capacity': stadium.capacity,
|
||||
'yearOpened': stadium.opened_year,
|
||||
'imageUrl': stadium.image_url,
|
||||
'surface': stadium.surface,
|
||||
'roofType': stadium.roof_type,
|
||||
'timezone': stadium.timezone,
|
||||
}
|
||||
|
||||
|
||||
def conference_to_dict(conf):
|
||||
"""Convert Conference model to dict for CloudKit."""
|
||||
return {
|
||||
'id': conf.canonical_id or f'conf_{conf.id}',
|
||||
'sport': conf.sport.code,
|
||||
'name': conf.name,
|
||||
'shortName': conf.short_name,
|
||||
'order': conf.order,
|
||||
}
|
||||
|
||||
|
||||
def division_to_dict(div):
|
||||
"""Convert Division model to dict for CloudKit."""
|
||||
return {
|
||||
'id': div.canonical_id or f'div_{div.id}',
|
||||
'conferenceId': div.conference.canonical_id or f'conf_{div.conference.id}',
|
||||
'sport': div.conference.sport.code,
|
||||
'name': div.name,
|
||||
'shortName': div.short_name,
|
||||
'order': div.order,
|
||||
}
|
||||
|
||||
|
||||
def team_alias_to_dict(alias):
|
||||
"""Convert TeamAlias model to dict for CloudKit."""
|
||||
return {
|
||||
'id': f'team_alias_{alias.id}',
|
||||
'teamId': alias.team.id,
|
||||
'alias': alias.alias,
|
||||
'aliasType': alias.alias_type,
|
||||
'validFrom': alias.valid_from.isoformat() if alias.valid_from else None,
|
||||
'validUntil': alias.valid_until.isoformat() if alias.valid_until else None,
|
||||
'isPrimary': alias.is_primary,
|
||||
}
|
||||
|
||||
|
||||
def stadium_alias_to_dict(alias):
|
||||
"""Convert StadiumAlias model to dict for CloudKit."""
|
||||
return {
|
||||
'id': f'stadium_alias_{alias.id}',
|
||||
'stadiumId': alias.stadium.id,
|
||||
'alias': alias.alias,
|
||||
'aliasType': alias.alias_type,
|
||||
'validFrom': alias.valid_from.isoformat() if alias.valid_from else None,
|
||||
'validUntil': alias.valid_until.isoformat() if alias.valid_until else None,
|
||||
'isPrimary': alias.is_primary,
|
||||
}
|
||||
|
||||
|
||||
@shared_task
|
||||
def mark_records_for_sync(record_type: str, record_ids: list):
|
||||
"""
|
||||
Mark records as needing sync after local changes.
|
||||
"""
|
||||
from cloudkit.models import CloudKitSyncState
|
||||
|
||||
for record_id in record_ids:
|
||||
state, created = CloudKitSyncState.objects.get_or_create(
|
||||
record_type=record_type,
|
||||
record_id=record_id,
|
||||
)
|
||||
state.mark_pending()
|
||||
|
||||
return {'marked': len(record_ids)}
|
||||
Reference in New Issue
Block a user