Refactor travel segments and simplify trip options
Travel segment architecture: - Remove departureTime/arrivalTime from TravelSegment (location-based, not date-based) - Fix travel sections appearing after destination instead of between cities - Fix missing travel segments when revisiting same city (consecutive grouping) - Remove unwanted rest day at end of trip Planning engine fixes: - All three planners now group only consecutive games at same stadium - Visiting A → B → A creates 3 stops with proper travel between UI simplification: - Remove redundant sort options (mostDriving/leastDriving, mostCities/leastCities) - Remove unused "Find Other Sports Along Route" toggle (was dead code) 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Trey t
Binary file not shown.
|
After Width: | Height: | Size: 540 KiB |
@@ -35,6 +35,22 @@ HOST = "https://api.apple-cloudkit.com"
|
||||
BATCH_SIZE = 200
|
||||
|
||||
|
||||
def deterministic_uuid(string: str) -> str:
|
||||
"""
|
||||
Generate a deterministic UUID from a string using SHA256.
|
||||
Matches the StubDataProvider.deterministicUUID() implementation in Swift.
|
||||
"""
|
||||
# SHA256 hash of the string
|
||||
hash_bytes = hashlib.sha256(string.encode('utf-8')).digest()
|
||||
# Use first 16 bytes
|
||||
uuid_bytes = bytearray(hash_bytes[:16])
|
||||
# Set UUID version (4) and variant bits to match Swift implementation
|
||||
uuid_bytes[6] = (uuid_bytes[6] & 0x0F) | 0x40
|
||||
uuid_bytes[8] = (uuid_bytes[8] & 0x3F) | 0x80
|
||||
# Format as UUID string
|
||||
return f"{uuid_bytes[0:4].hex()}-{uuid_bytes[4:6].hex()}-{uuid_bytes[6:8].hex()}-{uuid_bytes[8:10].hex()}-{uuid_bytes[10:16].hex()}".upper()
|
||||
|
||||
|
||||
class CloudKit:
|
||||
def __init__(self, key_id, private_key, container, env):
|
||||
self.key_id = key_id
|
||||
@@ -69,7 +85,7 @@ class CloudKit:
|
||||
except:
|
||||
return {'error': f"{r.status_code}: {r.text[:200]}"}
|
||||
|
||||
def query(self, record_type, limit=200):
|
||||
def query(self, record_type, limit=200, verbose=False):
|
||||
"""Query records of a given type."""
|
||||
path = f"{self.path_base}/records/query"
|
||||
body = json.dumps({
|
||||
@@ -83,16 +99,28 @@ class CloudKit:
|
||||
'X-Apple-CloudKit-Request-ISO8601Date': date,
|
||||
'X-Apple-CloudKit-Request-SignatureV1': self._sign(date, body, path),
|
||||
}
|
||||
r = requests.post(f"{HOST}{path}", headers=headers, data=body, timeout=60)
|
||||
if r.status_code == 200:
|
||||
return r.json()
|
||||
return {'error': f"{r.status_code}: {r.text[:200]}"}
|
||||
if verbose:
|
||||
print(f" Querying {record_type}...")
|
||||
try:
|
||||
r = requests.post(f"{HOST}{path}", headers=headers, data=body, timeout=30)
|
||||
if verbose:
|
||||
print(f" Response: {r.status_code}")
|
||||
if r.status_code == 200:
|
||||
result = r.json()
|
||||
if verbose:
|
||||
print(f" Found {len(result.get('records', []))} records")
|
||||
return result
|
||||
return {'error': f"{r.status_code}: {r.text[:200]}"}
|
||||
except requests.exceptions.Timeout:
|
||||
return {'error': 'Request timed out after 30s'}
|
||||
except Exception as e:
|
||||
return {'error': f"Request failed: {e}"}
|
||||
|
||||
def delete_all(self, record_type, verbose=False):
|
||||
"""Delete all records of a given type."""
|
||||
total_deleted = 0
|
||||
while True:
|
||||
result = self.query(record_type)
|
||||
result = self.query(record_type, verbose=verbose)
|
||||
if 'error' in result:
|
||||
print(f" Query error: {result['error']}")
|
||||
break
|
||||
@@ -101,21 +129,38 @@ class CloudKit:
|
||||
if not records:
|
||||
break
|
||||
|
||||
# Build delete operations
|
||||
# Build delete operations (recordChangeTag required for delete)
|
||||
ops = [{
|
||||
'operationType': 'delete',
|
||||
'record': {'recordName': r['recordName'], 'recordType': record_type}
|
||||
'record': {
|
||||
'recordName': r['recordName'],
|
||||
'recordType': record_type,
|
||||
'recordChangeTag': r.get('recordChangeTag', '')
|
||||
}
|
||||
} for r in records]
|
||||
|
||||
if verbose:
|
||||
print(f" Sending delete for {len(ops)} records...")
|
||||
|
||||
delete_result = self.modify(ops)
|
||||
|
||||
if verbose:
|
||||
print(f" Delete response: {json.dumps(delete_result)[:500]}")
|
||||
|
||||
if 'error' in delete_result:
|
||||
print(f" Delete error: {delete_result['error']}")
|
||||
break
|
||||
|
||||
deleted = len(delete_result.get('records', []))
|
||||
total_deleted += deleted
|
||||
if verbose:
|
||||
print(f" Deleted {deleted} {record_type} records...")
|
||||
# Check for individual record errors
|
||||
result_records = delete_result.get('records', [])
|
||||
successful = [r for r in result_records if 'serverErrorCode' not in r]
|
||||
failed = [r for r in result_records if 'serverErrorCode' in r]
|
||||
|
||||
if failed and verbose:
|
||||
print(f" Failed: {failed[0]}")
|
||||
|
||||
total_deleted += len(successful)
|
||||
print(f" Deleted {len(successful)} {record_type} records" + (f" ({len(failed)} failed)" if failed else ""))
|
||||
|
||||
time.sleep(0.5)
|
||||
|
||||
@@ -222,13 +267,16 @@ def main():
|
||||
stats = {'stadiums': 0, 'teams': 0, 'games': 0}
|
||||
team_map = {}
|
||||
|
||||
# Build stadium UUID lookup (stadium string ID -> UUID)
|
||||
stadium_uuid_map = {s['id']: deterministic_uuid(s['id']) for s in stadiums}
|
||||
|
||||
# Import stadiums & teams
|
||||
if not args.games_only:
|
||||
print("--- Stadiums ---")
|
||||
recs = [{
|
||||
'recordType': 'Stadium', 'recordName': s['id'],
|
||||
'recordType': 'Stadium', 'recordName': stadium_uuid_map[s['id']],
|
||||
'fields': {
|
||||
'stadiumId': {'value': s['id']}, 'name': {'value': s['name']},
|
||||
'stadiumId': {'value': stadium_uuid_map[s['id']]}, 'name': {'value': s['name']},
|
||||
'city': {'value': s['city']}, 'state': {'value': s.get('state', '')},
|
||||
'sport': {'value': s['sport']}, 'source': {'value': s.get('source', '')},
|
||||
'teamAbbrevs': {'value': s.get('team_abbrevs', [])},
|
||||
@@ -243,25 +291,39 @@ def main():
|
||||
teams = {}
|
||||
for s in stadiums:
|
||||
for abbr in s.get('team_abbrevs', []):
|
||||
if abbr not in teams:
|
||||
teams[abbr] = {'city': s['city'], 'sport': s['sport']}
|
||||
team_map[abbr] = f"team_{abbr.lower()}"
|
||||
team_key = f"{s['sport']}_{abbr}" # Match Swift: "{sport.rawValue}_{abbrev}"
|
||||
if team_key not in teams:
|
||||
teams[team_key] = {'abbr': abbr, 'city': s['city'], 'sport': s['sport']}
|
||||
team_uuid = deterministic_uuid(team_key)
|
||||
team_map[(s['sport'], abbr)] = team_uuid
|
||||
|
||||
recs = [{
|
||||
'recordType': 'Team', 'recordName': f"team_{abbr.lower()}",
|
||||
'recordType': 'Team', 'recordName': deterministic_uuid(team_key),
|
||||
'fields': {
|
||||
'teamId': {'value': f"team_{abbr.lower()}"}, 'abbreviation': {'value': abbr},
|
||||
'name': {'value': abbr}, 'city': {'value': info['city']}, 'sport': {'value': info['sport']},
|
||||
'teamId': {'value': deterministic_uuid(team_key)},
|
||||
'abbreviation': {'value': info['abbr']},
|
||||
'name': {'value': info['abbr']},
|
||||
'city': {'value': info['city']},
|
||||
'sport': {'value': info['sport']},
|
||||
}
|
||||
} for abbr, info in teams.items()]
|
||||
} for team_key, info in teams.items()]
|
||||
stats['teams'] = import_data(ck, recs, 'teams', args.dry_run, args.verbose)
|
||||
|
||||
# Import games
|
||||
if not args.stadiums_only and games:
|
||||
# Rebuild team_map if only importing games (--games-only flag)
|
||||
if not team_map:
|
||||
for s in stadiums:
|
||||
for abbr in s.get('team_abbrevs', []):
|
||||
team_map[abbr] = f"team_{abbr.lower()}"
|
||||
team_key = f"{s['sport']}_{abbr}"
|
||||
team_map[(s['sport'], abbr)] = deterministic_uuid(team_key)
|
||||
|
||||
# Build team -> stadium map for stadiumRef
|
||||
team_stadium_map = {}
|
||||
for s in stadiums:
|
||||
stadium_uuid = stadium_uuid_map[s['id']]
|
||||
for abbr in s.get('team_abbrevs', []):
|
||||
team_stadium_map[(s['sport'], abbr)] = stadium_uuid
|
||||
|
||||
print("--- Games ---")
|
||||
|
||||
@@ -278,20 +340,51 @@ def main():
|
||||
|
||||
recs = []
|
||||
for g in unique_games:
|
||||
game_uuid = deterministic_uuid(g['id'])
|
||||
sport = g['sport']
|
||||
|
||||
fields = {
|
||||
'gameId': {'value': g['id']}, 'sport': {'value': g['sport']},
|
||||
'gameId': {'value': game_uuid}, 'sport': {'value': sport},
|
||||
'season': {'value': g.get('season', '')}, 'source': {'value': g.get('source', '')},
|
||||
}
|
||||
if g.get('date'):
|
||||
try:
|
||||
dt = datetime.strptime(f"{g['date']} {g.get('time', '19:00')}", '%Y-%m-%d %H:%M')
|
||||
fields['dateTime'] = {'value': int(dt.timestamp() * 1000)}
|
||||
except: pass
|
||||
if g.get('home_team_abbrev') in team_map:
|
||||
fields['homeTeamRef'] = {'value': {'recordName': team_map[g['home_team_abbrev']], 'action': 'NONE'}}
|
||||
if g.get('away_team_abbrev') in team_map:
|
||||
fields['awayTeamRef'] = {'value': {'recordName': team_map[g['away_team_abbrev']], 'action': 'NONE'}}
|
||||
recs.append({'recordType': 'Game', 'recordName': g['id'], 'fields': fields})
|
||||
# Parse time like "7:30p" or "10:00a"
|
||||
time_str = g.get('time', '7:00p')
|
||||
hour, minute = 19, 0
|
||||
if time_str:
|
||||
clean_time = time_str.lower().replace(' ', '')
|
||||
is_pm = 'p' in clean_time
|
||||
time_parts = clean_time.replace('p', '').replace('a', '').split(':')
|
||||
if time_parts:
|
||||
hour = int(time_parts[0])
|
||||
if is_pm and hour != 12:
|
||||
hour += 12
|
||||
elif not is_pm and hour == 12:
|
||||
hour = 0
|
||||
if len(time_parts) > 1:
|
||||
minute = int(time_parts[1])
|
||||
dt = datetime.strptime(f"{g['date']} {hour:02d}:{minute:02d}", '%Y-%m-%d %H:%M')
|
||||
# CloudKit expects TIMESTAMP type with milliseconds since epoch
|
||||
fields['dateTime'] = {'value': int(dt.timestamp() * 1000), 'type': 'TIMESTAMP'}
|
||||
except Exception as e:
|
||||
if args.verbose:
|
||||
print(f" Warning: Failed to parse date/time for {g['id']}: {e}")
|
||||
|
||||
# Team references - use (sport, abbrev) tuple for lookup
|
||||
home_team_key = f"{sport}_{g.get('home_team_abbrev', '')}"
|
||||
away_team_key = f"{sport}_{g.get('away_team_abbrev', '')}"
|
||||
home_team_uuid = deterministic_uuid(home_team_key)
|
||||
away_team_uuid = deterministic_uuid(away_team_key)
|
||||
fields['homeTeamRef'] = {'value': {'recordName': home_team_uuid, 'action': 'NONE'}}
|
||||
fields['awayTeamRef'] = {'value': {'recordName': away_team_uuid, 'action': 'NONE'}}
|
||||
|
||||
# Stadium reference - look up by home team abbrev
|
||||
stadium_uuid = team_stadium_map.get((sport, g.get('home_team_abbrev', '')))
|
||||
if stadium_uuid:
|
||||
fields['stadiumRef'] = {'value': {'recordName': stadium_uuid, 'action': 'NONE'}}
|
||||
|
||||
recs.append({'recordType': 'Game', 'recordName': game_uuid, 'fields': fields})
|
||||
|
||||
stats['games'] = import_data(ck, recs, 'games', args.dry_run, args.verbose)
|
||||
|
||||
|
||||
10070
Scripts/data/games.csv
10070
Scripts/data/games.csv
File diff suppressed because it is too large
Load Diff
138738
Scripts/data/games.json
138738
Scripts/data/games.json
File diff suppressed because it is too large
Load Diff
@@ -4,8 +4,8 @@ Sports Schedule Scraper for SportsTime App
|
||||
Scrapes NBA, MLB, NHL schedules from multiple sources for cross-validation.
|
||||
|
||||
Usage:
|
||||
python scrape_schedules.py --sport nba --season 2025
|
||||
python scrape_schedules.py --sport all --season 2025
|
||||
python scrape_schedules.py --sport nba --season 2026
|
||||
python scrape_schedules.py --sport all --season 2026
|
||||
python scrape_schedules.py --stadiums-only
|
||||
"""
|
||||
|
||||
@@ -435,7 +435,7 @@ def scrape_mlb_statsapi(season: int) -> list[Game]:
|
||||
time_str = None
|
||||
|
||||
game = Game(
|
||||
id=f"mlb_{game_data.get('gamePk', '')}",
|
||||
id='', # Will be assigned by assign_stable_ids
|
||||
sport='MLB',
|
||||
season=str(season),
|
||||
date=game_date,
|
||||
@@ -786,28 +786,34 @@ def generate_stadiums_from_teams() -> list[Stadium]:
|
||||
|
||||
def assign_stable_ids(games: list[Game], sport: str, season: str) -> list[Game]:
|
||||
"""
|
||||
Assign stable IDs based on matchup + occurrence number within season.
|
||||
Format: {sport}_{season}_{away}_{home}_{num}
|
||||
Assign IDs based on matchup + date.
|
||||
Format: {sport}_{season}_{away}_{home}_{MMDD} (or {MMDD}_2 for doubleheaders)
|
||||
|
||||
This ensures IDs don't change when games are rescheduled.
|
||||
When games are rescheduled, the old ID becomes orphaned and a new one is created.
|
||||
Use --delete-all before import to clean up orphaned records.
|
||||
"""
|
||||
from collections import defaultdict
|
||||
|
||||
# Group games by matchup (away @ home)
|
||||
matchups = defaultdict(list)
|
||||
for game in games:
|
||||
key = f"{game.away_team_abbrev}_{game.home_team_abbrev}"
|
||||
matchups[key].append(game)
|
||||
season_str = season.replace('-', '')
|
||||
|
||||
# Sort each matchup by date and assign occurrence number
|
||||
for key, matchup_games in matchups.items():
|
||||
matchup_games.sort(key=lambda g: g.date)
|
||||
for i, game in enumerate(matchup_games, 1):
|
||||
away = game.away_team_abbrev.lower()
|
||||
home = game.home_team_abbrev.lower()
|
||||
# Normalize season format (e.g., "2024-25" -> "2024-25", "2025" -> "2025")
|
||||
season_str = season.replace('-', '')
|
||||
game.id = f"{sport.lower()}_{season_str}_{away}_{home}_{i}"
|
||||
# Track how many times we've seen each base ID (for doubleheaders)
|
||||
id_counts = defaultdict(int)
|
||||
|
||||
for game in games:
|
||||
away = game.away_team_abbrev.lower()
|
||||
home = game.home_team_abbrev.lower()
|
||||
# Extract MMDD from date (YYYY-MM-DD)
|
||||
date_parts = game.date.split('-')
|
||||
mmdd = f"{date_parts[1]}{date_parts[2]}" if len(date_parts) == 3 else "0000"
|
||||
|
||||
base_id = f"{sport.lower()}_{season_str}_{away}_{home}_{mmdd}"
|
||||
id_counts[base_id] += 1
|
||||
|
||||
# Add suffix for doubleheaders (game 2+)
|
||||
if id_counts[base_id] > 1:
|
||||
game.id = f"{base_id}_{id_counts[base_id]}"
|
||||
else:
|
||||
game.id = base_id
|
||||
|
||||
return games
|
||||
|
||||
@@ -892,7 +898,7 @@ def export_to_json(games: list[Game], stadiums: list[Stadium], output_dir: Path)
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='Scrape sports schedules')
|
||||
parser.add_argument('--sport', choices=['nba', 'mlb', 'nhl', 'all'], default='all')
|
||||
parser.add_argument('--season', type=int, default=2025, help='Season year (ending year)')
|
||||
parser.add_argument('--season', type=int, default=2026, help='Season year (ending year)')
|
||||
parser.add_argument('--stadiums-only', action='store_true', help='Only scrape stadium data')
|
||||
parser.add_argument('--output', type=str, default='./data', help='Output directory')
|
||||
|
||||
@@ -931,7 +937,7 @@ def main():
|
||||
print("="*60)
|
||||
|
||||
mlb_games_api = scrape_mlb_statsapi(args.season)
|
||||
# MLB API uses official gamePk which is already stable - no reassignment needed
|
||||
mlb_games_api = assign_stable_ids(mlb_games_api, 'MLB', str(args.season))
|
||||
all_games.extend(mlb_games_api)
|
||||
|
||||
if args.sport in ['nhl', 'all']:
|
||||
|
||||
Reference in New Issue
Block a user