Initial commit — PlantGuideScraper project

backend/alembic/versions/001_initial.py

@@ -0,0 +1,112 @@
+"""Initial migration
+
+Revision ID: 001
+Revises:
+Create Date: 2024-01-01
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = '001'
+down_revision: Union[str, None] = None
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # Species table
+    op.create_table(
+        'species',
+        sa.Column('id', sa.Integer(), primary_key=True),
+        sa.Column('scientific_name', sa.String(), nullable=False, unique=True),
+        sa.Column('common_name', sa.String(), nullable=True),
+        sa.Column('genus', sa.String(), nullable=True),
+        sa.Column('family', sa.String(), nullable=True),
+        sa.Column('created_at', sa.DateTime(), server_default=sa.func.now()),
+    )
+    op.create_index('ix_species_scientific_name', 'species', ['scientific_name'])
+    op.create_index('ix_species_genus', 'species', ['genus'])
+
+    # API Keys table
+    op.create_table(
+        'api_keys',
+        sa.Column('id', sa.Integer(), primary_key=True),
+        sa.Column('source', sa.String(), nullable=False, unique=True),
+        sa.Column('api_key', sa.String(), nullable=False),
+        sa.Column('api_secret', sa.String(), nullable=True),
+        sa.Column('rate_limit_per_sec', sa.Float(), default=1.0),
+        sa.Column('enabled', sa.Boolean(), default=True),
+    )
+
+    # Images table
+    op.create_table(
+        'images',
+        sa.Column('id', sa.Integer(), primary_key=True),
+        sa.Column('species_id', sa.Integer(), sa.ForeignKey('species.id'), nullable=False),
+        sa.Column('source', sa.String(), nullable=False),
+        sa.Column('source_id', sa.String(), nullable=True),
+        sa.Column('url', sa.String(), nullable=False),
+        sa.Column('local_path', sa.String(), nullable=True),
+        sa.Column('license', sa.String(), nullable=False),
+        sa.Column('attribution', sa.String(), nullable=True),
+        sa.Column('width', sa.Integer(), nullable=True),
+        sa.Column('height', sa.Integer(), nullable=True),
+        sa.Column('phash', sa.String(), nullable=True),
+        sa.Column('quality_score', sa.Float(), nullable=True),
+        sa.Column('status', sa.String(), default='pending'),
+        sa.Column('created_at', sa.DateTime(), server_default=sa.func.now()),
+    )
+    op.create_index('ix_images_species_id', 'images', ['species_id'])
+    op.create_index('ix_images_source', 'images', ['source'])
+    op.create_index('ix_images_status', 'images', ['status'])
+    op.create_index('ix_images_phash', 'images', ['phash'])
+    op.create_unique_constraint('uq_source_source_id', 'images', ['source', 'source_id'])
+
+    # Jobs table
+    op.create_table(
+        'jobs',
+        sa.Column('id', sa.Integer(), primary_key=True),
+        sa.Column('name', sa.String(), nullable=False),
+        sa.Column('source', sa.String(), nullable=False),
+        sa.Column('species_filter', sa.Text(), nullable=True),
+        sa.Column('status', sa.String(), default='pending'),
+        sa.Column('progress_current', sa.Integer(), default=0),
+        sa.Column('progress_total', sa.Integer(), default=0),
+        sa.Column('images_downloaded', sa.Integer(), default=0),
+        sa.Column('images_rejected', sa.Integer(), default=0),
+        sa.Column('celery_task_id', sa.String(), nullable=True),
+        sa.Column('started_at', sa.DateTime(), nullable=True),
+        sa.Column('completed_at', sa.DateTime(), nullable=True),
+        sa.Column('error_message', sa.Text(), nullable=True),
+        sa.Column('created_at', sa.DateTime(), server_default=sa.func.now()),
+    )
+    op.create_index('ix_jobs_status', 'jobs', ['status'])
+
+    # Exports table
+    op.create_table(
+        'exports',
+        sa.Column('id', sa.Integer(), primary_key=True),
+        sa.Column('name', sa.String(), nullable=False),
+        sa.Column('filter_criteria', sa.Text(), nullable=True),
+        sa.Column('train_split', sa.Float(), default=0.8),
+        sa.Column('status', sa.String(), default='pending'),
+        sa.Column('file_path', sa.String(), nullable=True),
+        sa.Column('file_size', sa.Integer(), nullable=True),
+        sa.Column('species_count', sa.Integer(), nullable=True),
+        sa.Column('image_count', sa.Integer(), nullable=True),
+        sa.Column('celery_task_id', sa.String(), nullable=True),
+        sa.Column('created_at', sa.DateTime(), server_default=sa.func.now()),
+        sa.Column('completed_at', sa.DateTime(), nullable=True),
+        sa.Column('error_message', sa.Text(), nullable=True),
+    )
+
+
+def downgrade() -> None:
+    op.drop_table('exports')
+    op.drop_table('jobs')
+    op.drop_table('images')
+    op.drop_table('api_keys')
+    op.drop_table('species')

backend/alembic/versions/002_add_cached_stats_and_indexes.py

@@ -0,0 +1,53 @@
+"""Add cached_stats table and license index
+
+Revision ID: 002
+Revises: 001
+Create Date: 2025-01-25
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = '002'
+down_revision: Union[str, None] = '001'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # Cached stats table for pre-calculated dashboard statistics
+    op.create_table(
+        'cached_stats',
+        sa.Column('id', sa.Integer(), primary_key=True),
+        sa.Column('key', sa.String(50), nullable=False, unique=True),
+        sa.Column('value', sa.Text(), nullable=False),
+        sa.Column('updated_at', sa.DateTime(), server_default=sa.func.now()),
+    )
+    op.create_index('ix_cached_stats_key', 'cached_stats', ['key'])
+
+    # Add license index to images table (if not exists)
+    # Using batch mode for SQLite compatibility
+    try:
+        op.create_index('ix_images_license', 'images', ['license'])
+    except Exception:
+        pass  # Index may already exist
+
+    # Add only_without_images column to jobs if it doesn't exist
+    try:
+        op.add_column('jobs', sa.Column('only_without_images', sa.Boolean(), default=False))
+    except Exception:
+        pass  # Column may already exist
+
+
+def downgrade() -> None:
+    try:
+        op.drop_index('ix_images_license', 'images')
+    except Exception:
+        pass
+    try:
+        op.drop_column('jobs', 'only_without_images')
+    except Exception:
+        pass
+    op.drop_table('cached_stats')

backend/alembic/versions/003_add_job_max_images.py

@@ -0,0 +1,31 @@
+"""Add max_images column to jobs table
+
+Revision ID: 003
+Revises: 002
+Create Date: 2025-01-25
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = '003'
+down_revision: Union[str, None] = '002'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # Add max_images column to jobs table
+    try:
+        op.add_column('jobs', sa.Column('max_images', sa.Integer(), nullable=True))
+    except Exception:
+        pass  # Column may already exist
+
+
+def downgrade() -> None:
+    try:
+        op.drop_column('jobs', 'max_images')
+    except Exception:
+        pass