37 lines
903 B
Python
37 lines
903 B
Python
from celery import Celery
|
|
|
|
from app.config import get_settings
|
|
|
|
settings = get_settings()
|
|
|
|
celery_app = Celery(
|
|
"plant_scraper",
|
|
broker=settings.redis_url,
|
|
backend=settings.redis_url,
|
|
include=[
|
|
"app.workers.scrape_tasks",
|
|
"app.workers.quality_tasks",
|
|
"app.workers.export_tasks",
|
|
"app.workers.stats_tasks",
|
|
],
|
|
)
|
|
|
|
celery_app.conf.update(
|
|
task_serializer="json",
|
|
accept_content=["json"],
|
|
result_serializer="json",
|
|
timezone="UTC",
|
|
enable_utc=True,
|
|
task_track_started=True,
|
|
task_time_limit=3600 * 24, # 24 hour max per task
|
|
worker_prefetch_multiplier=1,
|
|
task_acks_late=True,
|
|
beat_schedule={
|
|
"refresh-stats-every-5min": {
|
|
"task": "app.workers.stats_tasks.refresh_stats",
|
|
"schedule": 300.0, # Every 5 minutes
|
|
},
|
|
},
|
|
beat_schedule_filename="/tmp/celerybeat-schedule",
|
|
)
|