database fixes / simplification

Reviewed-on: #10

Dockerfile

@@ -39,13 +39,12 @@ COPY app/web/ ./web/
 COPY app/migrations/ ./migrations/
 COPY app/alembic.ini .
 COPY app/init_db.py .
-COPY app/docker-entrypoint.sh /docker-entrypoint.sh
 
 # Create required directories
 RUN mkdir -p /app/output /app/logs
 
 # Make scripts executable
-RUN chmod +x /app/src/scanner.py /app/init_db.py /docker-entrypoint.sh
+RUN chmod +x /app/src/scanner.py /app/init_db.py
 
 # Force Python unbuffered output
 ENV PYTHONUNBUFFERED=1

app/migrations/env.py

@@ -69,8 +69,12 @@ def run_migrations_online() -> None:
     )
 
     with connectable.connect() as connection:
+        # Enable batch mode for SQLite to support ALTER TABLE operations
+        # like DROP COLUMN which SQLite doesn't natively support
         context.configure(
-            connection=connection, target_metadata=target_metadata
+            connection=connection,
+            target_metadata=target_metadata,
+            render_as_batch=True
         )
 
         with context.begin_transaction():

app/migrations/versions/001_initial_schema.py

@@ -1,125 +1,214 @@
-"""Initial database schema for SneakyScanner
+"""Initial schema for SneakyScanner
 
 Revision ID: 001
-Revises:
-Create Date: 2025-11-13 18:00:00.000000
+Revises: None
+Create Date: 2025-12-24
 
+This is the complete initial schema for SneakyScanner. All tables are created
+in the correct order to satisfy foreign key constraints.
 """
 from alembic import op
 import sqlalchemy as sa
 
 
-# revision identifiers, used by Alembic.
+# revision identifiers, used by Alembic
 revision = '001'
 down_revision = None
 branch_labels = None
 depends_on = None
 
 
-def upgrade() -> None:
-    """Create all initial tables for SneakyScanner."""
+def upgrade():
+    """Create all tables for SneakyScanner."""
 
-    # Create schedules table first (referenced by scans)
-    op.create_table('schedules',
+    # =========================================================================
+    # Settings Table (no dependencies)
+    # =========================================================================
+    op.create_table(
+        'settings',
         sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
-        sa.Column('name', sa.String(length=255), nullable=False, comment='Schedule name (e.g., \'Daily prod scan\')'),
-        sa.Column('config_file', sa.Text(), nullable=False, comment='Path to YAML config'),
-        sa.Column('cron_expression', sa.String(length=100), nullable=False, comment='Cron-like schedule (e.g., \'0 2 * * *\')'),
-        sa.Column('enabled', sa.Boolean(), nullable=False, comment='Is schedule active?'),
-        sa.Column('last_run', sa.DateTime(), nullable=True, comment='Last execution time'),
-        sa.Column('next_run', sa.DateTime(), nullable=True, comment='Next scheduled execution'),
-        sa.Column('created_at', sa.DateTime(), nullable=False, comment='Schedule creation time'),
+        sa.Column('key', sa.String(length=255), nullable=False, comment='Setting key'),
+        sa.Column('value', sa.Text(), nullable=True, comment='Setting value (JSON for complex values)'),
+        sa.Column('updated_at', sa.DateTime(), nullable=False, comment='Last modification time'),
+        sa.PrimaryKeyConstraint('id'),
+        sa.UniqueConstraint('key')
+    )
+    op.create_index('ix_settings_key', 'settings', ['key'], unique=True)
+
+    # =========================================================================
+    # Reusable Site Definition Tables
+    # =========================================================================
+    op.create_table(
+        'sites',
+        sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
+        sa.Column('name', sa.String(length=255), nullable=False, comment='Unique site name'),
+        sa.Column('description', sa.Text(), nullable=True, comment='Site description'),
+        sa.Column('created_at', sa.DateTime(), nullable=False, comment='Site creation time'),
+        sa.Column('updated_at', sa.DateTime(), nullable=False, comment='Last modification time'),
+        sa.PrimaryKeyConstraint('id'),
+        sa.UniqueConstraint('name')
+    )
+    op.create_index('ix_sites_name', 'sites', ['name'], unique=True)
+
+    op.create_table(
+        'site_ips',
+        sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
+        sa.Column('site_id', sa.Integer(), nullable=False, comment='FK to sites'),
+        sa.Column('ip_address', sa.String(length=45), nullable=False, comment='IPv4 or IPv6 address'),
+        sa.Column('expected_ping', sa.Boolean(), nullable=True, comment='Expected ping response'),
+        sa.Column('expected_tcp_ports', sa.Text(), nullable=True, comment='JSON array of expected TCP ports'),
+        sa.Column('expected_udp_ports', sa.Text(), nullable=True, comment='JSON array of expected UDP ports'),
+        sa.Column('created_at', sa.DateTime(), nullable=False, comment='IP creation time'),
+        sa.ForeignKeyConstraint(['site_id'], ['sites.id']),
+        sa.PrimaryKeyConstraint('id'),
+        sa.UniqueConstraint('site_id', 'ip_address', name='uix_site_ip_address')
+    )
+    op.create_index('ix_site_ips_site_id', 'site_ips', ['site_id'])
+
+    # =========================================================================
+    # Scan Configuration Tables
+    # =========================================================================
+    op.create_table(
+        'scan_configs',
+        sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
+        sa.Column('title', sa.String(length=255), nullable=False, comment='Configuration title'),
+        sa.Column('description', sa.Text(), nullable=True, comment='Configuration description'),
+        sa.Column('created_at', sa.DateTime(), nullable=False, comment='Config creation time'),
         sa.Column('updated_at', sa.DateTime(), nullable=False, comment='Last modification time'),
         sa.PrimaryKeyConstraint('id')
     )
 
-    # Create scans table
-    op.create_table('scans',
+    op.create_table(
+        'scan_config_sites',
+        sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
+        sa.Column('config_id', sa.Integer(), nullable=False),
+        sa.Column('site_id', sa.Integer(), nullable=False),
+        sa.Column('created_at', sa.DateTime(), nullable=False, comment='Association creation time'),
+        sa.ForeignKeyConstraint(['config_id'], ['scan_configs.id']),
+        sa.ForeignKeyConstraint(['site_id'], ['sites.id']),
+        sa.PrimaryKeyConstraint('id'),
+        sa.UniqueConstraint('config_id', 'site_id', name='uix_config_site')
+    )
+    op.create_index('ix_scan_config_sites_config_id', 'scan_config_sites', ['config_id'])
+    op.create_index('ix_scan_config_sites_site_id', 'scan_config_sites', ['site_id'])
+
+    # =========================================================================
+    # Scheduling Tables
+    # =========================================================================
+    op.create_table(
+        'schedules',
+        sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
+        sa.Column('name', sa.String(length=255), nullable=False, comment='Schedule name'),
+        sa.Column('config_id', sa.Integer(), nullable=True, comment='FK to scan_configs table'),
+        sa.Column('cron_expression', sa.String(length=100), nullable=False, comment='Cron-like schedule'),
+        sa.Column('enabled', sa.Boolean(), nullable=False, default=True, comment='Is schedule active?'),
+        sa.Column('last_run', sa.DateTime(), nullable=True, comment='Last execution time'),
+        sa.Column('next_run', sa.DateTime(), nullable=True, comment='Next scheduled execution'),
+        sa.Column('created_at', sa.DateTime(), nullable=False, comment='Schedule creation time'),
+        sa.Column('updated_at', sa.DateTime(), nullable=False, comment='Last modification time'),
+        sa.ForeignKeyConstraint(['config_id'], ['scan_configs.id']),
+        sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index('ix_schedules_config_id', 'schedules', ['config_id'])
+
+    # =========================================================================
+    # Core Scan Tables
+    # =========================================================================
+    op.create_table(
+        'scans',
         sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
         sa.Column('timestamp', sa.DateTime(), nullable=False, comment='Scan start time (UTC)'),
         sa.Column('duration', sa.Float(), nullable=True, comment='Total scan duration in seconds'),
-        sa.Column('status', sa.String(length=20), nullable=False, comment='running, completed, failed'),
-        sa.Column('config_file', sa.Text(), nullable=True, comment='Path to YAML config used'),
+        sa.Column('status', sa.String(length=20), nullable=False, default='running', comment='running, finalizing, completed, failed, cancelled'),
+        sa.Column('config_id', sa.Integer(), nullable=True, comment='FK to scan_configs table'),
         sa.Column('title', sa.Text(), nullable=True, comment='Scan title from config'),
         sa.Column('json_path', sa.Text(), nullable=True, comment='Path to JSON report'),
         sa.Column('html_path', sa.Text(), nullable=True, comment='Path to HTML report'),
         sa.Column('zip_path', sa.Text(), nullable=True, comment='Path to ZIP archive'),
         sa.Column('screenshot_dir', sa.Text(), nullable=True, comment='Path to screenshot directory'),
         sa.Column('created_at', sa.DateTime(), nullable=False, comment='Record creation time'),
-        sa.Column('triggered_by', sa.String(length=50), nullable=False, comment='manual, scheduled, api'),
+        sa.Column('triggered_by', sa.String(length=50), nullable=False, default='manual', comment='manual, scheduled, api'),
         sa.Column('schedule_id', sa.Integer(), nullable=True, comment='FK to schedules if triggered by schedule'),
-        sa.ForeignKeyConstraint(['schedule_id'], ['schedules.id'], ),
+        sa.Column('started_at', sa.DateTime(), nullable=True, comment='Scan execution start time'),
+        sa.Column('completed_at', sa.DateTime(), nullable=True, comment='Scan execution completion time'),
+        sa.Column('error_message', sa.Text(), nullable=True, comment='Error message if scan failed'),
+        sa.Column('current_phase', sa.String(length=50), nullable=True, comment='Current scan phase'),
+        sa.Column('total_ips', sa.Integer(), nullable=True, comment='Total number of IPs to scan'),
+        sa.Column('completed_ips', sa.Integer(), nullable=True, default=0, comment='Number of IPs completed'),
+        sa.ForeignKeyConstraint(['config_id'], ['scan_configs.id']),
+        sa.ForeignKeyConstraint(['schedule_id'], ['schedules.id']),
         sa.PrimaryKeyConstraint('id')
     )
-    op.create_index(op.f('ix_scans_timestamp'), 'scans', ['timestamp'], unique=False)
+    op.create_index('ix_scans_timestamp', 'scans', ['timestamp'])
+    op.create_index('ix_scans_config_id', 'scans', ['config_id'])
 
-    # Create scan_sites table
-    op.create_table('scan_sites',
+    op.create_table(
+        'scan_sites',
         sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
-        sa.Column('scan_id', sa.Integer(), nullable=False, comment='FK to scans'),
+        sa.Column('scan_id', sa.Integer(), nullable=False),
         sa.Column('site_name', sa.String(length=255), nullable=False, comment='Site name from config'),
-        sa.ForeignKeyConstraint(['scan_id'], ['scans.id'], ),
+        sa.ForeignKeyConstraint(['scan_id'], ['scans.id']),
         sa.PrimaryKeyConstraint('id')
     )
-    op.create_index(op.f('ix_scan_sites_scan_id'), 'scan_sites', ['scan_id'], unique=False)
+    op.create_index('ix_scan_sites_scan_id', 'scan_sites', ['scan_id'])
 
-    # Create scan_ips table
-    op.create_table('scan_ips',
+    op.create_table(
+        'scan_ips',
         sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
-        sa.Column('scan_id', sa.Integer(), nullable=False, comment='FK to scans'),
-        sa.Column('site_id', sa.Integer(), nullable=False, comment='FK to scan_sites'),
+        sa.Column('scan_id', sa.Integer(), nullable=False),
+        sa.Column('site_id', sa.Integer(), nullable=False),
         sa.Column('ip_address', sa.String(length=45), nullable=False, comment='IPv4 or IPv6 address'),
         sa.Column('ping_expected', sa.Boolean(), nullable=True, comment='Expected ping response'),
         sa.Column('ping_actual', sa.Boolean(), nullable=True, comment='Actual ping response'),
-        sa.ForeignKeyConstraint(['scan_id'], ['scans.id'], ),
-        sa.ForeignKeyConstraint(['site_id'], ['scan_sites.id'], ),
+        sa.ForeignKeyConstraint(['scan_id'], ['scans.id']),
+        sa.ForeignKeyConstraint(['site_id'], ['scan_sites.id']),
         sa.PrimaryKeyConstraint('id'),
         sa.UniqueConstraint('scan_id', 'ip_address', name='uix_scan_ip')
     )
-    op.create_index(op.f('ix_scan_ips_scan_id'), 'scan_ips', ['scan_id'], unique=False)
-    op.create_index(op.f('ix_scan_ips_site_id'), 'scan_ips', ['site_id'], unique=False)
+    op.create_index('ix_scan_ips_scan_id', 'scan_ips', ['scan_id'])
+    op.create_index('ix_scan_ips_site_id', 'scan_ips', ['site_id'])
 
-    # Create scan_ports table
-    op.create_table('scan_ports',
+    op.create_table(
+        'scan_ports',
         sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
-        sa.Column('scan_id', sa.Integer(), nullable=False, comment='FK to scans'),
-        sa.Column('ip_id', sa.Integer(), nullable=False, comment='FK to scan_ips'),
+        sa.Column('scan_id', sa.Integer(), nullable=False),
+        sa.Column('ip_id', sa.Integer(), nullable=False),
         sa.Column('port', sa.Integer(), nullable=False, comment='Port number (1-65535)'),
         sa.Column('protocol', sa.String(length=10), nullable=False, comment='tcp or udp'),
         sa.Column('expected', sa.Boolean(), nullable=True, comment='Was this port expected?'),
-        sa.Column('state', sa.String(length=20), nullable=False, comment='open, closed, filtered'),
-        sa.ForeignKeyConstraint(['ip_id'], ['scan_ips.id'], ),
-        sa.ForeignKeyConstraint(['scan_id'], ['scans.id'], ),
+        sa.Column('state', sa.String(length=20), nullable=False, default='open', comment='open, closed, filtered'),
+        sa.ForeignKeyConstraint(['scan_id'], ['scans.id']),
+        sa.ForeignKeyConstraint(['ip_id'], ['scan_ips.id']),
         sa.PrimaryKeyConstraint('id'),
         sa.UniqueConstraint('scan_id', 'ip_id', 'port', 'protocol', name='uix_scan_ip_port')
     )
-    op.create_index(op.f('ix_scan_ports_ip_id'), 'scan_ports', ['ip_id'], unique=False)
-    op.create_index(op.f('ix_scan_ports_scan_id'), 'scan_ports', ['scan_id'], unique=False)
+    op.create_index('ix_scan_ports_scan_id', 'scan_ports', ['scan_id'])
+    op.create_index('ix_scan_ports_ip_id', 'scan_ports', ['ip_id'])
 
-    # Create scan_services table
-    op.create_table('scan_services',
+    op.create_table(
+        'scan_services',
         sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
-        sa.Column('scan_id', sa.Integer(), nullable=False, comment='FK to scans'),
-        sa.Column('port_id', sa.Integer(), nullable=False, comment='FK to scan_ports'),
-        sa.Column('service_name', sa.String(length=100), nullable=True, comment='Service name (e.g., ssh, http)'),
-        sa.Column('product', sa.String(length=255), nullable=True, comment='Product name (e.g., OpenSSH)'),
+        sa.Column('scan_id', sa.Integer(), nullable=False),
+        sa.Column('port_id', sa.Integer(), nullable=False),
+        sa.Column('service_name', sa.String(length=100), nullable=True, comment='Service name'),
+        sa.Column('product', sa.String(length=255), nullable=True, comment='Product name'),
         sa.Column('version', sa.String(length=100), nullable=True, comment='Version string'),
         sa.Column('extrainfo', sa.Text(), nullable=True, comment='Additional nmap info'),
         sa.Column('ostype', sa.String(length=100), nullable=True, comment='OS type if detected'),
-        sa.Column('http_protocol', sa.String(length=10), nullable=True, comment='http or https (if web service)'),
+        sa.Column('http_protocol', sa.String(length=10), nullable=True, comment='http or https'),
         sa.Column('screenshot_path', sa.Text(), nullable=True, comment='Relative path to screenshot'),
-        sa.ForeignKeyConstraint(['port_id'], ['scan_ports.id'], ),
-        sa.ForeignKeyConstraint(['scan_id'], ['scans.id'], ),
+        sa.ForeignKeyConstraint(['scan_id'], ['scans.id']),
+        sa.ForeignKeyConstraint(['port_id'], ['scan_ports.id']),
         sa.PrimaryKeyConstraint('id')
     )
-    op.create_index(op.f('ix_scan_services_port_id'), 'scan_services', ['port_id'], unique=False)
-    op.create_index(op.f('ix_scan_services_scan_id'), 'scan_services', ['scan_id'], unique=False)
+    op.create_index('ix_scan_services_scan_id', 'scan_services', ['scan_id'])
+    op.create_index('ix_scan_services_port_id', 'scan_services', ['port_id'])
 
-    # Create scan_certificates table
-    op.create_table('scan_certificates',
+    op.create_table(
+        'scan_certificates',
         sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
-        sa.Column('scan_id', sa.Integer(), nullable=False, comment='FK to scans'),
-        sa.Column('service_id', sa.Integer(), nullable=False, comment='FK to scan_services'),
+        sa.Column('scan_id', sa.Integer(), nullable=False),
+        sa.Column('service_id', sa.Integer(), nullable=False),
         sa.Column('subject', sa.Text(), nullable=True, comment='Certificate subject (CN)'),
         sa.Column('issuer', sa.Text(), nullable=True, comment='Certificate issuer'),
         sa.Column('serial_number', sa.Text(), nullable=True, comment='Serial number'),
@@ -127,95 +216,177 @@ def upgrade() -> None:
         sa.Column('not_valid_after', sa.DateTime(), nullable=True, comment='Validity end date'),
         sa.Column('days_until_expiry', sa.Integer(), nullable=True, comment='Days until expiration'),
         sa.Column('sans', sa.Text(), nullable=True, comment='JSON array of SANs'),
-        sa.Column('is_self_signed', sa.Boolean(), nullable=True, comment='Self-signed certificate flag'),
-        sa.ForeignKeyConstraint(['scan_id'], ['scans.id'], ),
-        sa.ForeignKeyConstraint(['service_id'], ['scan_services.id'], ),
-        sa.PrimaryKeyConstraint('id'),
-        comment='Index on expiration date for alert queries'
+        sa.Column('is_self_signed', sa.Boolean(), nullable=True, default=False, comment='Self-signed flag'),
+        sa.ForeignKeyConstraint(['scan_id'], ['scans.id']),
+        sa.ForeignKeyConstraint(['service_id'], ['scan_services.id']),
+        sa.PrimaryKeyConstraint('id')
     )
-    op.create_index(op.f('ix_scan_certificates_scan_id'), 'scan_certificates', ['scan_id'], unique=False)
-    op.create_index(op.f('ix_scan_certificates_service_id'), 'scan_certificates', ['service_id'], unique=False)
+    op.create_index('ix_scan_certificates_scan_id', 'scan_certificates', ['scan_id'])
+    op.create_index('ix_scan_certificates_service_id', 'scan_certificates', ['service_id'])
 
-    # Create scan_tls_versions table
-    op.create_table('scan_tls_versions',
+    op.create_table(
+        'scan_tls_versions',
         sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
-        sa.Column('scan_id', sa.Integer(), nullable=False, comment='FK to scans'),
-        sa.Column('certificate_id', sa.Integer(), nullable=False, comment='FK to scan_certificates'),
-        sa.Column('tls_version', sa.String(length=20), nullable=False, comment='TLS 1.0, TLS 1.1, TLS 1.2, TLS 1.3'),
+        sa.Column('scan_id', sa.Integer(), nullable=False),
+        sa.Column('certificate_id', sa.Integer(), nullable=False),
+        sa.Column('tls_version', sa.String(length=20), nullable=False, comment='TLS 1.0, 1.1, 1.2, 1.3'),
         sa.Column('supported', sa.Boolean(), nullable=False, comment='Is this version supported?'),
         sa.Column('cipher_suites', sa.Text(), nullable=True, comment='JSON array of cipher suites'),
-        sa.ForeignKeyConstraint(['certificate_id'], ['scan_certificates.id'], ),
-        sa.ForeignKeyConstraint(['scan_id'], ['scans.id'], ),
+        sa.ForeignKeyConstraint(['scan_id'], ['scans.id']),
+        sa.ForeignKeyConstraint(['certificate_id'], ['scan_certificates.id']),
         sa.PrimaryKeyConstraint('id')
     )
-    op.create_index(op.f('ix_scan_tls_versions_certificate_id'), 'scan_tls_versions', ['certificate_id'], unique=False)
-    op.create_index(op.f('ix_scan_tls_versions_scan_id'), 'scan_tls_versions', ['scan_id'], unique=False)
+    op.create_index('ix_scan_tls_versions_scan_id', 'scan_tls_versions', ['scan_id'])
+    op.create_index('ix_scan_tls_versions_certificate_id', 'scan_tls_versions', ['certificate_id'])
 
-    # Create alerts table
-    op.create_table('alerts',
+    op.create_table(
+        'scan_progress',
         sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
-        sa.Column('scan_id', sa.Integer(), nullable=False, comment='FK to scans'),
-        sa.Column('alert_type', sa.String(length=50), nullable=False, comment='new_port, cert_expiry, service_change, ping_failed'),
-        sa.Column('severity', sa.String(length=20), nullable=False, comment='info, warning, critical'),
-        sa.Column('message', sa.Text(), nullable=False, comment='Human-readable alert message'),
-        sa.Column('ip_address', sa.String(length=45), nullable=True, comment='Related IP (optional)'),
-        sa.Column('port', sa.Integer(), nullable=True, comment='Related port (optional)'),
-        sa.Column('email_sent', sa.Boolean(), nullable=False, comment='Was email notification sent?'),
-        sa.Column('email_sent_at', sa.DateTime(), nullable=True, comment='Email send timestamp'),
-        sa.Column('created_at', sa.DateTime(), nullable=False, comment='Alert creation time'),
-        sa.ForeignKeyConstraint(['scan_id'], ['scans.id'], ),
+        sa.Column('scan_id', sa.Integer(), nullable=False),
+        sa.Column('ip_address', sa.String(length=45), nullable=False, comment='IP address being scanned'),
+        sa.Column('site_name', sa.String(length=255), nullable=True, comment='Site name'),
+        sa.Column('phase', sa.String(length=50), nullable=False, comment='Phase: ping, tcp_scan, etc.'),
+        sa.Column('status', sa.String(length=20), nullable=False, default='pending', comment='pending, in_progress, completed, failed'),
+        sa.Column('ping_result', sa.Boolean(), nullable=True, comment='Ping response result'),
+        sa.Column('tcp_ports', sa.Text(), nullable=True, comment='JSON array of TCP ports'),
+        sa.Column('udp_ports', sa.Text(), nullable=True, comment='JSON array of UDP ports'),
+        sa.Column('services', sa.Text(), nullable=True, comment='JSON array of services'),
+        sa.Column('created_at', sa.DateTime(), nullable=False, comment='Entry creation time'),
+        sa.Column('updated_at', sa.DateTime(), nullable=False, comment='Last update time'),
+        sa.ForeignKeyConstraint(['scan_id'], ['scans.id']),
         sa.PrimaryKeyConstraint('id'),
-        comment='Indexes for alert filtering'
+        sa.UniqueConstraint('scan_id', 'ip_address', name='uix_scan_progress_ip')
     )
-    op.create_index(op.f('ix_alerts_scan_id'), 'alerts', ['scan_id'], unique=False)
+    op.create_index('ix_scan_progress_scan_id', 'scan_progress', ['scan_id'])
 
-    # Create alert_rules table
-    op.create_table('alert_rules',
+    op.create_table(
+        'scan_site_associations',
         sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
-        sa.Column('rule_type', sa.String(length=50), nullable=False, comment='unexpected_port, cert_expiry, service_down, etc.'),
-        sa.Column('enabled', sa.Boolean(), nullable=False, comment='Is rule active?'),
-        sa.Column('threshold', sa.Integer(), nullable=True, comment='Threshold value (e.g., days for cert expiry)'),
-        sa.Column('email_enabled', sa.Boolean(), nullable=False, comment='Send email for this rule?'),
+        sa.Column('scan_id', sa.Integer(), nullable=False),
+        sa.Column('site_id', sa.Integer(), nullable=False),
+        sa.Column('created_at', sa.DateTime(), nullable=False, comment='Association creation time'),
+        sa.ForeignKeyConstraint(['scan_id'], ['scans.id']),
+        sa.ForeignKeyConstraint(['site_id'], ['sites.id']),
+        sa.PrimaryKeyConstraint('id'),
+        sa.UniqueConstraint('scan_id', 'site_id', name='uix_scan_site')
+    )
+    op.create_index('ix_scan_site_associations_scan_id', 'scan_site_associations', ['scan_id'])
+    op.create_index('ix_scan_site_associations_site_id', 'scan_site_associations', ['site_id'])
+
+    # =========================================================================
+    # Alert Tables
+    # =========================================================================
+    op.create_table(
+        'alert_rules',
+        sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
+        sa.Column('name', sa.String(length=255), nullable=True, comment='User-friendly rule name'),
+        sa.Column('rule_type', sa.String(length=50), nullable=False, comment='unexpected_port, cert_expiry, etc.'),
+        sa.Column('enabled', sa.Boolean(), nullable=False, default=True, comment='Is rule active?'),
+        sa.Column('threshold', sa.Integer(), nullable=True, comment='Threshold value'),
+        sa.Column('email_enabled', sa.Boolean(), nullable=False, default=False, comment='Send email?'),
+        sa.Column('webhook_enabled', sa.Boolean(), nullable=False, default=False, comment='Send webhook?'),
+        sa.Column('severity', sa.String(length=20), nullable=True, comment='critical, warning, info'),
+        sa.Column('filter_conditions', sa.Text(), nullable=True, comment='JSON filter conditions'),
+        sa.Column('config_id', sa.Integer(), nullable=True, comment='Optional: specific config this rule applies to'),
         sa.Column('created_at', sa.DateTime(), nullable=False, comment='Rule creation time'),
+        sa.Column('updated_at', sa.DateTime(), nullable=True, comment='Last update time'),
+        sa.ForeignKeyConstraint(['config_id'], ['scan_configs.id']),
+        sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index('ix_alert_rules_config_id', 'alert_rules', ['config_id'])
+
+    op.create_table(
+        'alerts',
+        sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
+        sa.Column('scan_id', sa.Integer(), nullable=False),
+        sa.Column('rule_id', sa.Integer(), nullable=True, comment='Associated alert rule'),
+        sa.Column('alert_type', sa.String(length=50), nullable=False, comment='Alert type'),
+        sa.Column('severity', sa.String(length=20), nullable=False, comment='info, warning, critical'),
+        sa.Column('message', sa.Text(), nullable=False, comment='Human-readable message'),
+        sa.Column('ip_address', sa.String(length=45), nullable=True, comment='Related IP'),
+        sa.Column('port', sa.Integer(), nullable=True, comment='Related port'),
+        sa.Column('email_sent', sa.Boolean(), nullable=False, default=False, comment='Was email sent?'),
+        sa.Column('email_sent_at', sa.DateTime(), nullable=True, comment='Email send timestamp'),
+        sa.Column('webhook_sent', sa.Boolean(), nullable=False, default=False, comment='Was webhook sent?'),
+        sa.Column('webhook_sent_at', sa.DateTime(), nullable=True, comment='Webhook send timestamp'),
+        sa.Column('acknowledged', sa.Boolean(), nullable=False, default=False, comment='Was alert acknowledged?'),
+        sa.Column('acknowledged_at', sa.DateTime(), nullable=True, comment='Acknowledgment timestamp'),
+        sa.Column('acknowledged_by', sa.String(length=255), nullable=True, comment='User who acknowledged'),
+        sa.Column('created_at', sa.DateTime(), nullable=False, comment='Alert creation time'),
+        sa.ForeignKeyConstraint(['scan_id'], ['scans.id']),
+        sa.ForeignKeyConstraint(['rule_id'], ['alert_rules.id']),
+        sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index('ix_alerts_scan_id', 'alerts', ['scan_id'])
+    op.create_index('ix_alerts_rule_id', 'alerts', ['rule_id'])
+    op.create_index('ix_alerts_acknowledged', 'alerts', ['acknowledged'])
+
+    # =========================================================================
+    # Webhook Tables
+    # =========================================================================
+    op.create_table(
+        'webhooks',
+        sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
+        sa.Column('name', sa.String(length=255), nullable=False, comment='Webhook name'),
+        sa.Column('url', sa.Text(), nullable=False, comment='Webhook URL'),
+        sa.Column('enabled', sa.Boolean(), nullable=False, default=True, comment='Is webhook enabled?'),
+        sa.Column('auth_type', sa.String(length=20), nullable=True, comment='none, bearer, basic, custom'),
+        sa.Column('auth_token', sa.Text(), nullable=True, comment='Encrypted auth token'),
+        sa.Column('custom_headers', sa.Text(), nullable=True, comment='JSON custom headers'),
+        sa.Column('alert_types', sa.Text(), nullable=True, comment='JSON array of alert types'),
+        sa.Column('severity_filter', sa.Text(), nullable=True, comment='JSON array of severities'),
+        sa.Column('timeout', sa.Integer(), nullable=True, default=10, comment='Request timeout'),
+        sa.Column('retry_count', sa.Integer(), nullable=True, default=3, comment='Retry attempts'),
+        sa.Column('template', sa.Text(), nullable=True, comment='Jinja2 template for payload'),
+        sa.Column('template_format', sa.String(length=20), nullable=True, default='json', comment='json, text'),
+        sa.Column('content_type_override', sa.String(length=100), nullable=True, comment='Custom Content-Type'),
+        sa.Column('created_at', sa.DateTime(), nullable=False, comment='Creation time'),
+        sa.Column('updated_at', sa.DateTime(), nullable=False, comment='Last update time'),
         sa.PrimaryKeyConstraint('id')
     )
 
-    # Create settings table
-    op.create_table('settings',
+    op.create_table(
+        'webhook_delivery_log',
         sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
-        sa.Column('key', sa.String(length=255), nullable=False, comment='Setting key (e.g., smtp_server)'),
-        sa.Column('value', sa.Text(), nullable=True, comment='Setting value (JSON for complex values)'),
-        sa.Column('updated_at', sa.DateTime(), nullable=False, comment='Last modification time'),
-        sa.PrimaryKeyConstraint('id'),
-        sa.UniqueConstraint('key')
+        sa.Column('webhook_id', sa.Integer(), nullable=False, comment='Associated webhook'),
+        sa.Column('alert_id', sa.Integer(), nullable=False, comment='Associated alert'),
+        sa.Column('status', sa.String(length=20), nullable=True, comment='success, failed, retrying'),
+        sa.Column('response_code', sa.Integer(), nullable=True, comment='HTTP response code'),
+        sa.Column('response_body', sa.Text(), nullable=True, comment='Response body'),
+        sa.Column('error_message', sa.Text(), nullable=True, comment='Error message if failed'),
+        sa.Column('attempt_number', sa.Integer(), nullable=True, comment='Which attempt'),
+        sa.Column('delivered_at', sa.DateTime(), nullable=False, comment='Delivery timestamp'),
+        sa.ForeignKeyConstraint(['webhook_id'], ['webhooks.id']),
+        sa.ForeignKeyConstraint(['alert_id'], ['alerts.id']),
+        sa.PrimaryKeyConstraint('id')
     )
-    op.create_index(op.f('ix_settings_key'), 'settings', ['key'], unique=True)
+    op.create_index('ix_webhook_delivery_log_webhook_id', 'webhook_delivery_log', ['webhook_id'])
+    op.create_index('ix_webhook_delivery_log_alert_id', 'webhook_delivery_log', ['alert_id'])
+    op.create_index('ix_webhook_delivery_log_status', 'webhook_delivery_log', ['status'])
+
+    print("\n✓ Initial schema created successfully")
 
 
-def downgrade() -> None:
-    """Drop all tables."""
-    op.drop_index(op.f('ix_settings_key'), table_name='settings')
-    op.drop_table('settings')
-    op.drop_table('alert_rules')
-    op.drop_index(op.f('ix_alerts_scan_id'), table_name='alerts')
+def downgrade():
+    """Drop all tables in reverse order."""
+    op.drop_table('webhook_delivery_log')
+    op.drop_table('webhooks')
     op.drop_table('alerts')
-    op.drop_index(op.f('ix_scan_tls_versions_scan_id'), table_name='scan_tls_versions')
-    op.drop_index(op.f('ix_scan_tls_versions_certificate_id'), table_name='scan_tls_versions')
+    op.drop_table('alert_rules')
+    op.drop_table('scan_site_associations')
+    op.drop_table('scan_progress')
     op.drop_table('scan_tls_versions')
-    op.drop_index(op.f('ix_scan_certificates_service_id'), table_name='scan_certificates')
-    op.drop_index(op.f('ix_scan_certificates_scan_id'), table_name='scan_certificates')
     op.drop_table('scan_certificates')
-    op.drop_index(op.f('ix_scan_services_scan_id'), table_name='scan_services')
-    op.drop_index(op.f('ix_scan_services_port_id'), table_name='scan_services')
     op.drop_table('scan_services')
-    op.drop_index(op.f('ix_scan_ports_scan_id'), table_name='scan_ports')
-    op.drop_index(op.f('ix_scan_ports_ip_id'), table_name='scan_ports')
     op.drop_table('scan_ports')
-    op.drop_index(op.f('ix_scan_ips_site_id'), table_name='scan_ips')
-    op.drop_index(op.f('ix_scan_ips_scan_id'), table_name='scan_ips')
     op.drop_table('scan_ips')
-    op.drop_index(op.f('ix_scan_sites_scan_id'), table_name='scan_sites')
     op.drop_table('scan_sites')
-    op.drop_index(op.f('ix_scans_timestamp'), table_name='scans')
     op.drop_table('scans')
     op.drop_table('schedules')
+    op.drop_table('scan_config_sites')
+    op.drop_table('scan_configs')
+    op.drop_table('site_ips')
+    op.drop_table('sites')
+    op.drop_table('settings')
+
+    print("\n✓ All tables dropped")

app/migrations/versions/002_add_scan_indexes.py

@@ -1,28 +0,0 @@
-"""Add indexes for scan queries
-
-Revision ID: 002
-Revises: 001
-Create Date: 2025-11-14 00:30:00.000000
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = '002'
-down_revision = '001'
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    """Add database indexes for better query performance."""
-    # Add index on scans.status for filtering
-    # Note: index on scans.timestamp already exists from migration 001
-    op.create_index('ix_scans_status', 'scans', ['status'], unique=False)
-
-
-def downgrade() -> None:
-    """Remove indexes."""
-    op.drop_index('ix_scans_status', table_name='scans')

app/migrations/versions/003_add_scan_timing_fields.py

@@ -1,39 +0,0 @@
-"""Add timing and error fields to scans table
-
-Revision ID: 003
-Revises: 002
-Create Date: 2025-11-14
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic
-revision = '003'
-down_revision = '002'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    """
-    Add fields for tracking scan execution timing and errors.
-
-    New fields:
-    - started_at: When scan execution actually started
-    - completed_at: When scan execution finished (success or failure)
-    - error_message: Error message if scan failed
-    """
-    with op.batch_alter_table('scans') as batch_op:
-        batch_op.add_column(sa.Column('started_at', sa.DateTime(), nullable=True, comment='Scan execution start time'))
-        batch_op.add_column(sa.Column('completed_at', sa.DateTime(), nullable=True, comment='Scan execution completion time'))
-        batch_op.add_column(sa.Column('error_message', sa.Text(), nullable=True, comment='Error message if scan failed'))
-
-
-def downgrade():
-    """Remove the timing and error fields."""
-    with op.batch_alter_table('scans') as batch_op:
-        batch_op.drop_column('error_message')
-        batch_op.drop_column('completed_at')
-        batch_op.drop_column('started_at')

app/migrations/versions/004_add_alert_rule_enhancements.py

@@ -1,120 +0,0 @@
-"""Add enhanced alert features for Phase 5
-
-Revision ID: 004
-Revises: 003
-Create Date: 2025-11-18
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic
-revision = '004'
-down_revision = '003'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    """
-    Add enhancements for Phase 5 Alert Rule Engine:
-    - Enhanced alert_rules fields
-    - Enhanced alerts fields
-    - New webhooks table
-    - New webhook_delivery_log table
-    """
-
-    # Enhance alert_rules table
-    with op.batch_alter_table('alert_rules') as batch_op:
-        batch_op.add_column(sa.Column('name', sa.String(255), nullable=True, comment='User-friendly rule name'))
-        batch_op.add_column(sa.Column('webhook_enabled', sa.Boolean(), nullable=False, server_default='0', comment='Whether to send webhooks for this rule'))
-        batch_op.add_column(sa.Column('severity', sa.String(20), nullable=True, comment='Alert severity level (critical, warning, info)'))
-        batch_op.add_column(sa.Column('filter_conditions', sa.Text(), nullable=True, comment='JSON filter conditions for the rule'))
-        batch_op.add_column(sa.Column('config_file', sa.String(255), nullable=True, comment='Optional: specific config file this rule applies to'))
-        batch_op.add_column(sa.Column('updated_at', sa.DateTime(), nullable=True, comment='Last update timestamp'))
-
-    # Enhance alerts table
-    with op.batch_alter_table('alerts') as batch_op:
-        batch_op.add_column(sa.Column('rule_id', sa.Integer(), nullable=True, comment='Associated alert rule'))
-        batch_op.add_column(sa.Column('webhook_sent', sa.Boolean(), nullable=False, server_default='0', comment='Whether webhook was sent'))
-        batch_op.add_column(sa.Column('webhook_sent_at', sa.DateTime(), nullable=True, comment='When webhook was sent'))
-        batch_op.add_column(sa.Column('acknowledged', sa.Boolean(), nullable=False, server_default='0', comment='Whether alert was acknowledged'))
-        batch_op.add_column(sa.Column('acknowledged_at', sa.DateTime(), nullable=True, comment='When alert was acknowledged'))
-        batch_op.add_column(sa.Column('acknowledged_by', sa.String(255), nullable=True, comment='User who acknowledged the alert'))
-        batch_op.create_foreign_key('fk_alerts_rule_id', 'alert_rules', ['rule_id'], ['id'])
-        batch_op.create_index('idx_alerts_rule_id', ['rule_id'])
-        batch_op.create_index('idx_alerts_acknowledged', ['acknowledged'])
-
-    # Create webhooks table
-    op.create_table('webhooks',
-        sa.Column('id', sa.Integer(), nullable=False),
-        sa.Column('name', sa.String(255), nullable=False, comment='Webhook name'),
-        sa.Column('url', sa.Text(), nullable=False, comment='Webhook URL'),
-        sa.Column('enabled', sa.Boolean(), nullable=False, server_default='1', comment='Whether webhook is enabled'),
-        sa.Column('auth_type', sa.String(20), nullable=True, comment='Authentication type: none, bearer, basic, custom'),
-        sa.Column('auth_token', sa.Text(), nullable=True, comment='Encrypted authentication token'),
-        sa.Column('custom_headers', sa.Text(), nullable=True, comment='JSON custom headers'),
-        sa.Column('alert_types', sa.Text(), nullable=True, comment='JSON array of alert types to trigger on'),
-        sa.Column('severity_filter', sa.Text(), nullable=True, comment='JSON array of severities to trigger on'),
-        sa.Column('timeout', sa.Integer(), nullable=True, server_default='10', comment='Request timeout in seconds'),
-        sa.Column('retry_count', sa.Integer(), nullable=True, server_default='3', comment='Number of retry attempts'),
-        sa.Column('created_at', sa.DateTime(), nullable=False),
-        sa.Column('updated_at', sa.DateTime(), nullable=False),
-        sa.PrimaryKeyConstraint('id')
-    )
-
-    # Create webhook_delivery_log table
-    op.create_table('webhook_delivery_log',
-        sa.Column('id', sa.Integer(), nullable=False),
-        sa.Column('webhook_id', sa.Integer(), nullable=False, comment='Associated webhook'),
-        sa.Column('alert_id', sa.Integer(), nullable=False, comment='Associated alert'),
-        sa.Column('status', sa.String(20), nullable=True, comment='Delivery status: success, failed, retrying'),
-        sa.Column('response_code', sa.Integer(), nullable=True, comment='HTTP response code'),
-        sa.Column('response_body', sa.Text(), nullable=True, comment='Response body from webhook'),
-        sa.Column('error_message', sa.Text(), nullable=True, comment='Error message if failed'),
-        sa.Column('attempt_number', sa.Integer(), nullable=True, comment='Which attempt this was'),
-        sa.Column('delivered_at', sa.DateTime(), nullable=False, comment='Delivery timestamp'),
-        sa.ForeignKeyConstraint(['webhook_id'], ['webhooks.id'], ),
-        sa.ForeignKeyConstraint(['alert_id'], ['alerts.id'], ),
-        sa.PrimaryKeyConstraint('id')
-    )
-
-    # Create indexes for webhook_delivery_log
-    op.create_index('idx_webhook_delivery_alert_id', 'webhook_delivery_log', ['alert_id'])
-    op.create_index('idx_webhook_delivery_webhook_id', 'webhook_delivery_log', ['webhook_id'])
-    op.create_index('idx_webhook_delivery_status', 'webhook_delivery_log', ['status'])
-
-
-def downgrade():
-    """Remove Phase 5 alert enhancements."""
-
-    # Drop webhook_delivery_log table and its indexes
-    op.drop_index('idx_webhook_delivery_status', table_name='webhook_delivery_log')
-    op.drop_index('idx_webhook_delivery_webhook_id', table_name='webhook_delivery_log')
-    op.drop_index('idx_webhook_delivery_alert_id', table_name='webhook_delivery_log')
-    op.drop_table('webhook_delivery_log')
-
-    # Drop webhooks table
-    op.drop_table('webhooks')
-
-    # Remove enhancements from alerts table
-    with op.batch_alter_table('alerts') as batch_op:
-        batch_op.drop_index('idx_alerts_acknowledged')
-        batch_op.drop_index('idx_alerts_rule_id')
-        batch_op.drop_constraint('fk_alerts_rule_id', type_='foreignkey')
-        batch_op.drop_column('acknowledged_by')
-        batch_op.drop_column('acknowledged_at')
-        batch_op.drop_column('acknowledged')
-        batch_op.drop_column('webhook_sent_at')
-        batch_op.drop_column('webhook_sent')
-        batch_op.drop_column('rule_id')
-
-    # Remove enhancements from alert_rules table
-    with op.batch_alter_table('alert_rules') as batch_op:
-        batch_op.drop_column('updated_at')
-        batch_op.drop_column('config_file')
-        batch_op.drop_column('filter_conditions')
-        batch_op.drop_column('severity')
-        batch_op.drop_column('webhook_enabled')
-        batch_op.drop_column('name')

app/migrations/versions/005_add_webhook_templates.py

@@ -1,83 +0,0 @@
-"""Add webhook template support
-
-Revision ID: 005
-Revises: 004
-Create Date: 2025-11-18
-
-"""
-from alembic import op
-import sqlalchemy as sa
-import json
-
-
-# revision identifiers, used by Alembic
-revision = '005'
-down_revision = '004'
-branch_labels = None
-depends_on = None
-
-
-# Default template that matches the current JSON payload structure
-DEFAULT_TEMPLATE = """{
-  "event": "alert.created",
-  "alert": {
-    "id": {{ alert.id }},
-    "type": "{{ alert.type }}",
-    "severity": "{{ alert.severity }}",
-    "message": "{{ alert.message }}",
-    {% if alert.ip_address %}"ip_address": "{{ alert.ip_address }}",{% endif %}
-    {% if alert.port %}"port": {{ alert.port }},{% endif %}
-    "acknowledged": {{ alert.acknowledged|lower }},
-    "created_at": "{{ alert.created_at.isoformat() }}"
-  },
-  "scan": {
-    "id": {{ scan.id }},
-    "title": "{{ scan.title }}",
-    "timestamp": "{{ scan.timestamp.isoformat() }}",
-    "status": "{{ scan.status }}"
-  },
-  "rule": {
-    "id": {{ rule.id }},
-    "name": "{{ rule.name }}",
-    "type": "{{ rule.type }}",
-    "threshold": {{ rule.threshold if rule.threshold else 'null' }}
-  }
-}"""
-
-
-def upgrade():
-    """
-    Add webhook template fields:
-    - template: Jinja2 template for payload
-    - template_format: Output format (json, text)
-    - content_type_override: Optional custom Content-Type
-    """
-
-    # Add new columns to webhooks table
-    with op.batch_alter_table('webhooks') as batch_op:
-        batch_op.add_column(sa.Column('template', sa.Text(), nullable=True, comment='Jinja2 template for webhook payload'))
-        batch_op.add_column(sa.Column('template_format', sa.String(20), nullable=True, server_default='json', comment='Template output format: json, text'))
-        batch_op.add_column(sa.Column('content_type_override', sa.String(100), nullable=True, comment='Optional custom Content-Type header'))
-
-    # Populate existing webhooks with default template
-    # This ensures backward compatibility by converting existing webhooks to use the
-    # same JSON structure they're currently sending
-    connection = op.get_bind()
-    connection.execute(
-        sa.text("""
-            UPDATE webhooks
-            SET template = :template,
-                template_format = 'json'
-            WHERE template IS NULL
-        """),
-        {"template": DEFAULT_TEMPLATE}
-    )
-
-
-def downgrade():
-    """Remove webhook template fields."""
-
-    with op.batch_alter_table('webhooks') as batch_op:
-        batch_op.drop_column('content_type_override')
-        batch_op.drop_column('template_format')
-        batch_op.drop_column('template')

app/migrations/versions/006_add_reusable_sites.py

@@ -1,161 +0,0 @@
-"""Add reusable site definitions
-
-Revision ID: 006
-Revises: 005
-Create Date: 2025-11-19
-
-This migration introduces reusable site definitions that can be shared across
-multiple scans. Sites are defined once with CIDR ranges and can be referenced
-in multiple scan configurations.
-"""
-from alembic import op
-import sqlalchemy as sa
-from sqlalchemy import text
-
-
-# revision identifiers, used by Alembic
-revision = '006'
-down_revision = '005'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    """
-    Create new site tables and migrate existing scan_sites data to the new structure.
-    """
-
-    # Create sites table (master site definitions)
-    op.create_table('sites',
-        sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
-        sa.Column('name', sa.String(length=255), nullable=False, comment='Unique site name'),
-        sa.Column('description', sa.Text(), nullable=True, comment='Site description'),
-        sa.Column('created_at', sa.DateTime(), nullable=False, comment='Site creation time'),
-        sa.Column('updated_at', sa.DateTime(), nullable=False, comment='Last modification time'),
-        sa.PrimaryKeyConstraint('id'),
-        sa.UniqueConstraint('name', name='uix_site_name')
-    )
-    op.create_index(op.f('ix_sites_name'), 'sites', ['name'], unique=True)
-
-    # Create site_cidrs table (CIDR ranges for each site)
-    op.create_table('site_cidrs',
-        sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
-        sa.Column('site_id', sa.Integer(), nullable=False, comment='FK to sites'),
-        sa.Column('cidr', sa.String(length=45), nullable=False, comment='CIDR notation (e.g., 10.0.0.0/24)'),
-        sa.Column('expected_ping', sa.Boolean(), nullable=True, comment='Expected ping response for this CIDR'),
-        sa.Column('expected_tcp_ports', sa.Text(), nullable=True, comment='JSON array of expected TCP ports'),
-        sa.Column('expected_udp_ports', sa.Text(), nullable=True, comment='JSON array of expected UDP ports'),
-        sa.Column('created_at', sa.DateTime(), nullable=False, comment='CIDR creation time'),
-        sa.ForeignKeyConstraint(['site_id'], ['sites.id'], ),
-        sa.PrimaryKeyConstraint('id'),
-        sa.UniqueConstraint('site_id', 'cidr', name='uix_site_cidr')
-    )
-    op.create_index(op.f('ix_site_cidrs_site_id'), 'site_cidrs', ['site_id'], unique=False)
-
-    # Create site_ips table (IP-level overrides within CIDRs)
-    op.create_table('site_ips',
-        sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
-        sa.Column('site_cidr_id', sa.Integer(), nullable=False, comment='FK to site_cidrs'),
-        sa.Column('ip_address', sa.String(length=45), nullable=False, comment='IPv4 or IPv6 address'),
-        sa.Column('expected_ping', sa.Boolean(), nullable=True, comment='Override ping expectation for this IP'),
-        sa.Column('expected_tcp_ports', sa.Text(), nullable=True, comment='JSON array of expected TCP ports (overrides CIDR)'),
-        sa.Column('expected_udp_ports', sa.Text(), nullable=True, comment='JSON array of expected UDP ports (overrides CIDR)'),
-        sa.Column('created_at', sa.DateTime(), nullable=False, comment='IP override creation time'),
-        sa.ForeignKeyConstraint(['site_cidr_id'], ['site_cidrs.id'], ),
-        sa.PrimaryKeyConstraint('id'),
-        sa.UniqueConstraint('site_cidr_id', 'ip_address', name='uix_site_cidr_ip')
-    )
-    op.create_index(op.f('ix_site_ips_site_cidr_id'), 'site_ips', ['site_cidr_id'], unique=False)
-
-    # Create scan_site_associations table (many-to-many between scans and sites)
-    op.create_table('scan_site_associations',
-        sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
-        sa.Column('scan_id', sa.Integer(), nullable=False, comment='FK to scans'),
-        sa.Column('site_id', sa.Integer(), nullable=False, comment='FK to sites'),
-        sa.Column('created_at', sa.DateTime(), nullable=False, comment='Association creation time'),
-        sa.ForeignKeyConstraint(['scan_id'], ['scans.id'], ),
-        sa.ForeignKeyConstraint(['site_id'], ['sites.id'], ),
-        sa.PrimaryKeyConstraint('id'),
-        sa.UniqueConstraint('scan_id', 'site_id', name='uix_scan_site')
-    )
-    op.create_index(op.f('ix_scan_site_associations_scan_id'), 'scan_site_associations', ['scan_id'], unique=False)
-    op.create_index(op.f('ix_scan_site_associations_site_id'), 'scan_site_associations', ['site_id'], unique=False)
-
-    # Migrate existing data
-    connection = op.get_bind()
-
-    # 1. Extract unique site names from existing scan_sites and create master Site records
-    # This groups all historical scan sites by name and creates one master site per unique name
-    connection.execute(text("""
-        INSERT INTO sites (name, description, created_at, updated_at)
-        SELECT DISTINCT
-            site_name,
-            'Migrated from scan_sites' as description,
-            datetime('now') as created_at,
-            datetime('now') as updated_at
-        FROM scan_sites
-        WHERE site_name NOT IN (SELECT name FROM sites)
-    """))
-
-    # 2. Create scan_site_associations linking scans to their sites
-    # This maintains the historical relationship between scans and the sites they used
-    connection.execute(text("""
-        INSERT INTO scan_site_associations (scan_id, site_id, created_at)
-        SELECT DISTINCT
-            ss.scan_id,
-            s.id as site_id,
-            datetime('now') as created_at
-        FROM scan_sites ss
-        INNER JOIN sites s ON s.name = ss.site_name
-        WHERE NOT EXISTS (
-            SELECT 1 FROM scan_site_associations ssa
-            WHERE ssa.scan_id = ss.scan_id AND ssa.site_id = s.id
-        )
-    """))
-
-    # 3. For each migrated site, create a CIDR entry from the IPs in scan_ips
-    # Since historical data has individual IPs, we'll create /32 CIDRs for each unique IP
-    # This preserves the exact IP addresses while fitting them into the new CIDR-based model
-    connection.execute(text("""
-        INSERT INTO site_cidrs (site_id, cidr, expected_ping, expected_tcp_ports, expected_udp_ports, created_at)
-        SELECT DISTINCT
-            s.id as site_id,
-            si.ip_address || '/32' as cidr,
-            si.ping_expected,
-            '[]' as expected_tcp_ports,
-            '[]' as expected_udp_ports,
-            datetime('now') as created_at
-        FROM scan_ips si
-        INNER JOIN scan_sites ss ON ss.id = si.site_id
-        INNER JOIN sites s ON s.name = ss.site_name
-        WHERE NOT EXISTS (
-            SELECT 1 FROM site_cidrs sc
-            WHERE sc.site_id = s.id AND sc.cidr = si.ip_address || '/32'
-        )
-        GROUP BY s.id, si.ip_address, si.ping_expected
-    """))
-
-    print("✓ Migration complete: Reusable sites created from historical scan data")
-    print(f"  - Created {connection.execute(text('SELECT COUNT(*) FROM sites')).scalar()} master site(s)")
-    print(f"  - Created {connection.execute(text('SELECT COUNT(*) FROM site_cidrs')).scalar()} CIDR range(s)")
-    print(f"  - Created {connection.execute(text('SELECT COUNT(*) FROM scan_site_associations')).scalar()} scan-site association(s)")
-
-
-def downgrade():
-    """Remove reusable site tables."""
-
-    # Drop tables in reverse order of creation (respecting foreign keys)
-    op.drop_index(op.f('ix_scan_site_associations_site_id'), table_name='scan_site_associations')
-    op.drop_index(op.f('ix_scan_site_associations_scan_id'), table_name='scan_site_associations')
-    op.drop_table('scan_site_associations')
-
-    op.drop_index(op.f('ix_site_ips_site_cidr_id'), table_name='site_ips')
-    op.drop_table('site_ips')
-
-    op.drop_index(op.f('ix_site_cidrs_site_id'), table_name='site_cidrs')
-    op.drop_table('site_cidrs')
-
-    op.drop_index(op.f('ix_sites_name'), table_name='sites')
-    op.drop_table('sites')
-
-    print("✓ Downgrade complete: Reusable site tables removed")

app/migrations/versions/007_configs_to_database.py

@@ -1,102 +0,0 @@
-"""Add database-stored scan configurations
-
-Revision ID: 007
-Revises: 006
-Create Date: 2025-11-19
-
-This migration introduces database-stored scan configurations to replace YAML
-config files. Configs reference sites from the sites table, enabling visual
-config builder and better data management.
-"""
-from alembic import op
-import sqlalchemy as sa
-from sqlalchemy import text
-
-
-# revision identifiers, used by Alembic
-revision = '007'
-down_revision = '006'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    """
-    Create scan_configs and scan_config_sites tables.
-    Add config_id foreign keys to scans and schedules tables.
-    """
-
-    # Create scan_configs table
-    op.create_table('scan_configs',
-        sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
-        sa.Column('title', sa.String(length=255), nullable=False, comment='Configuration title'),
-        sa.Column('description', sa.Text(), nullable=True, comment='Configuration description'),
-        sa.Column('created_at', sa.DateTime(), nullable=False, comment='Config creation time'),
-        sa.Column('updated_at', sa.DateTime(), nullable=False, comment='Last modification time'),
-        sa.PrimaryKeyConstraint('id')
-    )
-
-    # Create scan_config_sites table (many-to-many between configs and sites)
-    op.create_table('scan_config_sites',
-        sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
-        sa.Column('config_id', sa.Integer(), nullable=False, comment='FK to scan_configs'),
-        sa.Column('site_id', sa.Integer(), nullable=False, comment='FK to sites'),
-        sa.Column('created_at', sa.DateTime(), nullable=False, comment='Association creation time'),
-        sa.ForeignKeyConstraint(['config_id'], ['scan_configs.id'], ),
-        sa.ForeignKeyConstraint(['site_id'], ['sites.id'], ),
-        sa.PrimaryKeyConstraint('id'),
-        sa.UniqueConstraint('config_id', 'site_id', name='uix_config_site')
-    )
-    op.create_index(op.f('ix_scan_config_sites_config_id'), 'scan_config_sites', ['config_id'], unique=False)
-    op.create_index(op.f('ix_scan_config_sites_site_id'), 'scan_config_sites', ['site_id'], unique=False)
-
-    # Add config_id to scans table
-    with op.batch_alter_table('scans', schema=None) as batch_op:
-        batch_op.add_column(sa.Column('config_id', sa.Integer(), nullable=True, comment='FK to scan_configs table'))
-        batch_op.create_index('ix_scans_config_id', ['config_id'], unique=False)
-        batch_op.create_foreign_key('fk_scans_config_id', 'scan_configs', ['config_id'], ['id'])
-        # Mark config_file as deprecated in comment (already has nullable=True)
-
-    # Add config_id to schedules table and make config_file nullable
-    with op.batch_alter_table('schedules', schema=None) as batch_op:
-        batch_op.add_column(sa.Column('config_id', sa.Integer(), nullable=True, comment='FK to scan_configs table'))
-        batch_op.create_index('ix_schedules_config_id', ['config_id'], unique=False)
-        batch_op.create_foreign_key('fk_schedules_config_id', 'scan_configs', ['config_id'], ['id'])
-        # Make config_file nullable (it was required before)
-        batch_op.alter_column('config_file', existing_type=sa.Text(), nullable=True)
-
-    connection = op.get_bind()
-
-    print("✓ Migration complete: Scan configs tables created")
-    print("  - Created scan_configs table for database-stored configurations")
-    print("  - Created scan_config_sites association table")
-    print("  - Added config_id to scans table")
-    print("  - Added config_id to schedules table")
-    print("  - Existing YAML configs remain in config_file column for backward compatibility")
-
-
-def downgrade():
-    """Remove scan config tables and columns."""
-
-    # Remove foreign keys and columns from schedules
-    with op.batch_alter_table('schedules', schema=None) as batch_op:
-        batch_op.drop_constraint('fk_schedules_config_id', type_='foreignkey')
-        batch_op.drop_index('ix_schedules_config_id')
-        batch_op.drop_column('config_id')
-        # Restore config_file as required
-        batch_op.alter_column('config_file', existing_type=sa.Text(), nullable=False)
-
-    # Remove foreign keys and columns from scans
-    with op.batch_alter_table('scans', schema=None) as batch_op:
-        batch_op.drop_constraint('fk_scans_config_id', type_='foreignkey')
-        batch_op.drop_index('ix_scans_config_id')
-        batch_op.drop_column('config_id')
-
-    # Drop tables in reverse order
-    op.drop_index(op.f('ix_scan_config_sites_site_id'), table_name='scan_config_sites')
-    op.drop_index(op.f('ix_scan_config_sites_config_id'), table_name='scan_config_sites')
-    op.drop_table('scan_config_sites')
-
-    op.drop_table('scan_configs')
-
-    print("✓ Downgrade complete: Scan config tables and columns removed")

app/migrations/versions/008_expand_cidrs_to_ips.py

@@ -1,270 +0,0 @@
-"""Expand CIDRs to individual IPs with per-IP settings
-
-Revision ID: 008
-Revises: 007
-Create Date: 2025-11-19
-
-This migration changes the site architecture to automatically expand CIDRs into
-individual IPs in the database. Each IP has its own port and ping settings.
-
-Changes:
-- Add site_id to site_ips (direct link to sites, support standalone IPs)
-- Make site_cidr_id nullable (IPs can exist without a CIDR parent)
-- Remove settings from site_cidrs (settings now only at IP level)
-- Add unique constraint: no duplicate IPs within a site
-- Expand existing CIDRs to individual IPs
-"""
-from alembic import op
-import sqlalchemy as sa
-from sqlalchemy import text
-import ipaddress
-
-
-# revision identifiers, used by Alembic
-revision = '008'
-down_revision = '007'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    """
-    Modify schema to support per-IP settings and auto-expand CIDRs.
-    """
-
-    connection = op.get_bind()
-
-    # Check if site_id column already exists
-    inspector = sa.inspect(connection)
-    site_ips_columns = [col['name'] for col in inspector.get_columns('site_ips')]
-    site_cidrs_columns = [col['name'] for col in inspector.get_columns('site_cidrs')]
-
-    # Step 1: Add site_id column to site_ips (will be populated from site_cidr_id)
-    if 'site_id' not in site_ips_columns:
-        print("Adding site_id column to site_ips...")
-        op.add_column('site_ips', sa.Column('site_id', sa.Integer(), nullable=True, comment='FK to sites (direct link)'))
-    else:
-        print("site_id column already exists in site_ips, skipping...")
-
-    # Step 2: Populate site_id from site_cidr_id (before we make it nullable)
-    print("Populating site_id from existing site_cidr relationships...")
-    connection.execute(text("""
-        UPDATE site_ips
-        SET site_id = (
-            SELECT site_id
-            FROM site_cidrs
-            WHERE site_cidrs.id = site_ips.site_cidr_id
-        )
-        WHERE site_cidr_id IS NOT NULL
-    """))
-
-    # Step 3: Make site_id NOT NULL and add foreign key
-    # Check if foreign key exists before creating
-    try:
-        op.alter_column('site_ips', 'site_id', nullable=False)
-        print("Made site_id NOT NULL")
-    except Exception as e:
-        print(f"site_id already NOT NULL or error: {e}")
-
-    # Check if foreign key exists
-    try:
-        op.create_foreign_key('fk_site_ips_site_id', 'site_ips', 'sites', ['site_id'], ['id'])
-        print("Created foreign key fk_site_ips_site_id")
-    except Exception as e:
-        print(f"Foreign key already exists or error: {e}")
-
-    # Check if index exists
-    try:
-        op.create_index(op.f('ix_site_ips_site_id'), 'site_ips', ['site_id'], unique=False)
-        print("Created index ix_site_ips_site_id")
-    except Exception as e:
-        print(f"Index already exists or error: {e}")
-
-    # Step 4: Make site_cidr_id nullable (for standalone IPs)
-    try:
-        op.alter_column('site_ips', 'site_cidr_id', nullable=True)
-        print("Made site_cidr_id nullable")
-    except Exception as e:
-        print(f"site_cidr_id already nullable or error: {e}")
-
-    # Step 5: Drop old unique constraint and create new one (site_id, ip_address)
-    # This prevents duplicate IPs within a site (across all CIDRs and standalone)
-    try:
-        op.drop_constraint('uix_site_cidr_ip', 'site_ips', type_='unique')
-        print("Dropped old constraint uix_site_cidr_ip")
-    except Exception as e:
-        print(f"Constraint already dropped or doesn't exist: {e}")
-
-    try:
-        op.create_unique_constraint('uix_site_ip_address', 'site_ips', ['site_id', 'ip_address'])
-        print("Created new constraint uix_site_ip_address")
-    except Exception as e:
-        print(f"Constraint already exists or error: {e}")
-
-    # Step 6: Expand existing CIDRs to individual IPs
-    print("Expanding existing CIDRs to individual IPs...")
-
-    # Get all existing CIDRs
-    cidrs = connection.execute(text("""
-        SELECT id, site_id, cidr, expected_ping, expected_tcp_ports, expected_udp_ports
-        FROM site_cidrs
-    """)).fetchall()
-
-    expanded_count = 0
-    skipped_count = 0
-
-    for cidr_row in cidrs:
-        cidr_id, site_id, cidr_str, expected_ping, expected_tcp_ports, expected_udp_ports = cidr_row
-
-        try:
-            # Parse CIDR
-            network = ipaddress.ip_network(cidr_str, strict=False)
-
-            # Check size - skip if too large (> /24 for IPv4, > /64 for IPv6)
-            if isinstance(network, ipaddress.IPv4Network) and network.prefixlen < 24:
-                print(f"  ⚠ Skipping large CIDR {cidr_str} (>{network.num_addresses} IPs)")
-                skipped_count += 1
-                continue
-            elif isinstance(network, ipaddress.IPv6Network) and network.prefixlen < 64:
-                print(f"  ⚠ Skipping large CIDR {cidr_str} (>{network.num_addresses} IPs)")
-                skipped_count += 1
-                continue
-
-            # Expand to individual IPs
-            for ip in network.hosts() if network.num_addresses > 2 else [network.network_address]:
-                ip_str = str(ip)
-
-                # Check if this IP already exists (from previous IP overrides)
-                existing = connection.execute(text("""
-                    SELECT id FROM site_ips
-                    WHERE site_cidr_id = :cidr_id AND ip_address = :ip_address
-                """), {'cidr_id': cidr_id, 'ip_address': ip_str}).fetchone()
-
-                if not existing:
-                    # Insert new IP with settings from CIDR
-                    connection.execute(text("""
-                        INSERT INTO site_ips (
-                            site_id, site_cidr_id, ip_address,
-                            expected_ping, expected_tcp_ports, expected_udp_ports,
-                            created_at
-                        )
-                        VALUES (
-                            :site_id, :cidr_id, :ip_address,
-                            :expected_ping, :expected_tcp_ports, :expected_udp_ports,
-                            datetime('now')
-                        )
-                    """), {
-                        'site_id': site_id,
-                        'cidr_id': cidr_id,
-                        'ip_address': ip_str,
-                        'expected_ping': expected_ping,
-                        'expected_tcp_ports': expected_tcp_ports,
-                        'expected_udp_ports': expected_udp_ports
-                    })
-                    expanded_count += 1
-
-        except Exception as e:
-            print(f"  ✗ Error expanding CIDR {cidr_str}: {e}")
-            skipped_count += 1
-            continue
-
-    print(f"  ✓ Expanded {expanded_count} IPs from CIDRs")
-    if skipped_count > 0:
-        print(f"  ⚠ Skipped {skipped_count} CIDRs (too large or errors)")
-
-    # Step 7: Remove settings columns from site_cidrs (now only at IP level)
-    print("Removing settings columns from site_cidrs...")
-    # Re-inspect to get current columns
-    site_cidrs_columns = [col['name'] for col in inspector.get_columns('site_cidrs')]
-
-    if 'expected_ping' in site_cidrs_columns:
-        try:
-            op.drop_column('site_cidrs', 'expected_ping')
-            print("Dropped expected_ping from site_cidrs")
-        except Exception as e:
-            print(f"Error dropping expected_ping: {e}")
-    else:
-        print("expected_ping already dropped from site_cidrs")
-
-    if 'expected_tcp_ports' in site_cidrs_columns:
-        try:
-            op.drop_column('site_cidrs', 'expected_tcp_ports')
-            print("Dropped expected_tcp_ports from site_cidrs")
-        except Exception as e:
-            print(f"Error dropping expected_tcp_ports: {e}")
-    else:
-        print("expected_tcp_ports already dropped from site_cidrs")
-
-    if 'expected_udp_ports' in site_cidrs_columns:
-        try:
-            op.drop_column('site_cidrs', 'expected_udp_ports')
-            print("Dropped expected_udp_ports from site_cidrs")
-        except Exception as e:
-            print(f"Error dropping expected_udp_ports: {e}")
-    else:
-        print("expected_udp_ports already dropped from site_cidrs")
-
-    # Print summary
-    total_sites = connection.execute(text('SELECT COUNT(*) FROM sites')).scalar()
-    total_cidrs = connection.execute(text('SELECT COUNT(*) FROM site_cidrs')).scalar()
-    total_ips = connection.execute(text('SELECT COUNT(*) FROM site_ips')).scalar()
-
-    print("\n✓ Migration 008 complete: CIDRs expanded to individual IPs")
-    print(f"  - Total sites: {total_sites}")
-    print(f"  - Total CIDRs: {total_cidrs}")
-    print(f"  - Total IPs: {total_ips}")
-
-
-def downgrade():
-    """
-    Revert schema changes (restore CIDR-level settings).
-    Note: This will lose per-IP granularity!
-    """
-
-    connection = op.get_bind()
-
-    print("Rolling back to CIDR-level settings...")
-
-    # Step 1: Add settings columns back to site_cidrs
-    op.add_column('site_cidrs', sa.Column('expected_ping', sa.Boolean(), nullable=True))
-    op.add_column('site_cidrs', sa.Column('expected_tcp_ports', sa.Text(), nullable=True))
-    op.add_column('site_cidrs', sa.Column('expected_udp_ports', sa.Text(), nullable=True))
-
-    # Step 2: Populate CIDR settings from first IP in each CIDR (approximation)
-    connection.execute(text("""
-        UPDATE site_cidrs
-        SET
-            expected_ping = (
-                SELECT expected_ping FROM site_ips
-                WHERE site_ips.site_cidr_id = site_cidrs.id
-                LIMIT 1
-            ),
-            expected_tcp_ports = (
-                SELECT expected_tcp_ports FROM site_ips
-                WHERE site_ips.site_cidr_id = site_cidrs.id
-                LIMIT 1
-            ),
-            expected_udp_ports = (
-                SELECT expected_udp_ports FROM site_ips
-                WHERE site_ips.site_cidr_id = site_cidrs.id
-                LIMIT 1
-            )
-    """))
-
-    # Step 3: Delete auto-expanded IPs (keep only original overrides)
-    # In practice, this is difficult to determine, so we'll keep all IPs
-    # and just remove the schema changes
-
-    # Step 4: Drop new unique constraint and restore old one
-    op.drop_constraint('uix_site_ip_address', 'site_ips', type_='unique')
-    op.create_unique_constraint('uix_site_cidr_ip', 'site_ips', ['site_cidr_id', 'ip_address'])
-
-    # Step 5: Make site_cidr_id NOT NULL again
-    op.alter_column('site_ips', 'site_cidr_id', nullable=False)
-
-    # Step 6: Drop site_id column and related constraints
-    op.drop_index(op.f('ix_site_ips_site_id'), table_name='site_ips')
-    op.drop_constraint('fk_site_ips_site_id', 'site_ips', type_='foreignkey')
-    op.drop_column('site_ips', 'site_id')
-
-    print("✓ Downgrade complete: Reverted to CIDR-level settings")

app/migrations/versions/009_remove_cidrs.py

@@ -1,210 +0,0 @@
-"""Remove CIDR table - make sites IP-only
-
-Revision ID: 009
-Revises: 008
-Create Date: 2025-11-19
-
-This migration removes the SiteCIDR table entirely, making sites purely
-IP-based. CIDRs are now only used as a convenience for bulk IP addition,
-not stored as permanent entities.
-
-Changes:
-- Set all site_ips.site_cidr_id to NULL (preserve all IPs)
-- Drop foreign key from site_ips to site_cidrs
-- Drop site_cidrs table
-- Remove site_cidr_id column from site_ips
-
-All existing IPs are preserved. They become "standalone" IPs without
-a CIDR parent.
-"""
-from alembic import op
-import sqlalchemy as sa
-from sqlalchemy import text
-
-
-# revision identifiers, used by Alembic
-revision = '009'
-down_revision = '008'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    """
-    Remove CIDR table and make all IPs standalone.
-    """
-
-    connection = op.get_bind()
-    inspector = sa.inspect(connection)
-
-    print("\n=== Migration 009: Remove CIDR Table ===\n")
-
-    # Get counts before migration
-    try:
-        total_cidrs = connection.execute(text('SELECT COUNT(*) FROM site_cidrs')).scalar()
-        total_ips = connection.execute(text('SELECT COUNT(*) FROM site_ips')).scalar()
-        ips_with_cidr = connection.execute(text(
-            'SELECT COUNT(*) FROM site_ips WHERE site_cidr_id IS NOT NULL'
-        )).scalar()
-
-        print(f"Before migration:")
-        print(f"  - Total CIDRs: {total_cidrs}")
-        print(f"  - Total IPs: {total_ips}")
-        print(f"  - IPs linked to CIDRs: {ips_with_cidr}")
-        print(f"  - Standalone IPs: {total_ips - ips_with_cidr}\n")
-    except Exception as e:
-        print(f"Could not get pre-migration stats: {e}\n")
-
-    # Step 1: Set all site_cidr_id to NULL (preserve all IPs as standalone)
-    print("Step 1: Converting all IPs to standalone (nulling CIDR associations)...")
-    try:
-        result = connection.execute(text("""
-            UPDATE site_ips
-            SET site_cidr_id = NULL
-            WHERE site_cidr_id IS NOT NULL
-        """))
-        print(f"  ✓ Converted {result.rowcount} IPs to standalone\n")
-    except Exception as e:
-        print(f"  ⚠ Error or already done: {e}\n")
-
-    # Step 2: Drop foreign key constraint from site_ips to site_cidrs
-    print("Step 2: Dropping foreign key constraint from site_ips to site_cidrs...")
-    foreign_keys = inspector.get_foreign_keys('site_ips')
-    fk_to_drop = None
-
-    for fk in foreign_keys:
-        if fk['referred_table'] == 'site_cidrs':
-            fk_to_drop = fk['name']
-            break
-
-    if fk_to_drop:
-        try:
-            op.drop_constraint(fk_to_drop, 'site_ips', type_='foreignkey')
-            print(f"  ✓ Dropped foreign key constraint: {fk_to_drop}\n")
-        except Exception as e:
-            print(f"  ⚠ Could not drop foreign key: {e}\n")
-    else:
-        print("  ⚠ Foreign key constraint not found or already dropped\n")
-
-    # Step 3: Drop index on site_cidr_id (if exists)
-    print("Step 3: Dropping index on site_cidr_id...")
-    indexes = inspector.get_indexes('site_ips')
-    index_to_drop = None
-
-    for idx in indexes:
-        if 'site_cidr_id' in idx['column_names']:
-            index_to_drop = idx['name']
-            break
-
-    if index_to_drop:
-        try:
-            op.drop_index(index_to_drop, table_name='site_ips')
-            print(f"  ✓ Dropped index: {index_to_drop}\n")
-        except Exception as e:
-            print(f"  ⚠ Could not drop index: {e}\n")
-    else:
-        print("  ⚠ Index not found or already dropped\n")
-
-    # Step 4: Drop site_cidrs table
-    print("Step 4: Dropping site_cidrs table...")
-    tables = inspector.get_table_names()
-
-    if 'site_cidrs' in tables:
-        try:
-            op.drop_table('site_cidrs')
-            print("  ✓ Dropped site_cidrs table\n")
-        except Exception as e:
-            print(f"  ⚠ Could not drop table: {e}\n")
-    else:
-        print("  ⚠ Table site_cidrs not found or already dropped\n")
-
-    # Step 5: Drop site_cidr_id column from site_ips
-    print("Step 5: Dropping site_cidr_id column from site_ips...")
-    site_ips_columns = [col['name'] for col in inspector.get_columns('site_ips')]
-
-    if 'site_cidr_id' in site_ips_columns:
-        try:
-            op.drop_column('site_ips', 'site_cidr_id')
-            print("  ✓ Dropped site_cidr_id column from site_ips\n")
-        except Exception as e:
-            print(f"  ⚠ Could not drop column: {e}\n")
-    else:
-        print("  ⚠ Column site_cidr_id not found or already dropped\n")
-
-    # Get counts after migration
-    try:
-        final_ips = connection.execute(text('SELECT COUNT(*) FROM site_ips')).scalar()
-        total_sites = connection.execute(text('SELECT COUNT(*) FROM sites')).scalar()
-
-        print("After migration:")
-        print(f"  - Total sites: {total_sites}")
-        print(f"  - Total IPs (all standalone): {final_ips}")
-        print(f"  - CIDRs: N/A (table removed)")
-    except Exception as e:
-        print(f"Could not get post-migration stats: {e}")
-
-    print("\n✓ Migration 009 complete: Sites are now IP-only")
-    print("  All IPs preserved as standalone. CIDRs can still be used")
-    print("  via the API/UI for bulk IP creation, but are not stored.\n")
-
-
-def downgrade():
-    """
-    Recreate site_cidrs table (CANNOT restore original CIDR associations).
-
-    WARNING: This downgrade creates an empty site_cidrs table structure but
-    cannot restore the original CIDR-to-IP associations since that data was
-    deleted. All IPs will remain standalone.
-    """
-
-    connection = op.get_bind()
-
-    print("\n=== Downgrade 009: Recreate CIDR Table Structure ===\n")
-    print("⚠ WARNING: Cannot restore original CIDR associations!")
-    print("  The site_cidrs table structure will be recreated but will be empty.")
-    print("  All IPs will remain standalone. This is a PARTIAL downgrade.\n")
-
-    # Step 1: Recreate site_cidrs table (empty)
-    print("Step 1: Recreating site_cidrs table structure...")
-    try:
-        op.create_table(
-            'site_cidrs',
-            sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
-            sa.Column('site_id', sa.Integer(), nullable=False),
-            sa.Column('cidr', sa.String(length=45), nullable=False, comment='CIDR notation (e.g., 10.0.0.0/24)'),
-            sa.Column('created_at', sa.DateTime(), nullable=False),
-            sa.PrimaryKeyConstraint('id'),
-            sa.ForeignKeyConstraint(['site_id'], ['sites.id'], ),
-            sa.UniqueConstraint('site_id', 'cidr', name='uix_site_cidr')
-        )
-        print("  ✓ Recreated site_cidrs table (empty)\n")
-    except Exception as e:
-        print(f"  ⚠ Could not create table: {e}\n")
-
-    # Step 2: Add site_cidr_id column back to site_ips (nullable)
-    print("Step 2: Adding site_cidr_id column back to site_ips...")
-    try:
-        op.add_column('site_ips', sa.Column('site_cidr_id', sa.Integer(), nullable=True, comment='FK to site_cidrs (optional, for grouping)'))
-        print("  ✓ Added site_cidr_id column (nullable)\n")
-    except Exception as e:
-        print(f"  ⚠ Could not add column: {e}\n")
-
-    # Step 3: Add foreign key constraint
-    print("Step 3: Adding foreign key constraint...")
-    try:
-        op.create_foreign_key('fk_site_ips_site_cidr_id', 'site_ips', 'site_cidrs', ['site_cidr_id'], ['id'])
-        print("  ✓ Created foreign key constraint\n")
-    except Exception as e:
-        print(f"  ⚠ Could not create foreign key: {e}\n")
-
-    # Step 4: Add index on site_cidr_id
-    print("Step 4: Adding index on site_cidr_id...")
-    try:
-        op.create_index('ix_site_ips_site_cidr_id', 'site_ips', ['site_cidr_id'], unique=False)
-        print("  ✓ Created index on site_cidr_id\n")
-    except Exception as e:
-        print(f"  ⚠ Could not create index: {e}\n")
-
-    print("✓ Downgrade complete: CIDR table structure restored (but empty)")
-    print("  All IPs remain standalone. You would need to manually recreate")
-    print("  CIDR records and associate IPs with them.\n")

app/migrations/versions/010_alert_rules_config_id.py

@@ -1,53 +0,0 @@
-"""Add config_id to alert_rules table
-
-Revision ID: 010
-Revises: 009
-Create Date: 2025-11-19
-
-This migration adds config_id foreign key to alert_rules table to replace
-the config_file column, completing the migration from file-based to
-database-based configurations.
-"""
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic
-revision = '010'
-down_revision = '009'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    """
-    Add config_id to alert_rules table and remove config_file.
-    """
-
-    with op.batch_alter_table('alert_rules', schema=None) as batch_op:
-        # Add config_id column with foreign key
-        batch_op.add_column(sa.Column('config_id', sa.Integer(), nullable=True, comment='FK to scan_configs table'))
-        batch_op.create_index('ix_alert_rules_config_id', ['config_id'], unique=False)
-        batch_op.create_foreign_key('fk_alert_rules_config_id', 'scan_configs', ['config_id'], ['id'])
-
-        # Remove the old config_file column
-        batch_op.drop_column('config_file')
-
-    print("✓ Migration complete: AlertRule now uses config_id")
-    print("  - Added config_id foreign key to alert_rules table")
-    print("  - Removed deprecated config_file column")
-
-
-def downgrade():
-    """Remove config_id and restore config_file on alert_rules."""
-
-    with op.batch_alter_table('alert_rules', schema=None) as batch_op:
-        # Remove foreign key and config_id column
-        batch_op.drop_constraint('fk_alert_rules_config_id', type_='foreignkey')
-        batch_op.drop_index('ix_alert_rules_config_id')
-        batch_op.drop_column('config_id')
-
-        # Restore config_file column
-        batch_op.add_column(sa.Column('config_file', sa.String(255), nullable=True, comment='Optional: specific config file this rule applies to'))
-
-    print("✓ Downgrade complete: AlertRule config_id removed, config_file restored")

app/migrations/versions/011_drop_config_file.py

@@ -1,86 +0,0 @@
-"""Drop deprecated config_file columns
-
-Revision ID: 011
-Revises: 010
-Create Date: 2025-11-19
-
-This migration removes the deprecated config_file columns from scans and schedules
-tables. All functionality now uses config_id to reference database-stored configs.
-"""
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic
-revision = '011'
-down_revision = '010'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    """
-    Drop config_file columns from scans and schedules tables.
-
-    Prerequisites:
-    - All scans must have config_id set
-    - All schedules must have config_id set
-    - Code must be updated to no longer reference config_file
-    """
-
-    connection = op.get_bind()
-
-    # Check for any records missing config_id
-    result = connection.execute(sa.text(
-        "SELECT COUNT(*) FROM scans WHERE config_id IS NULL"
-    ))
-    scans_without_config = result.scalar()
-
-    result = connection.execute(sa.text(
-        "SELECT COUNT(*) FROM schedules WHERE config_id IS NULL"
-    ))
-    schedules_without_config = result.scalar()
-
-    if scans_without_config > 0:
-        print(f"WARNING: {scans_without_config} scans have NULL config_id")
-        print("  These scans will lose their config reference after migration")
-
-    if schedules_without_config > 0:
-        raise Exception(
-            f"Cannot proceed: {schedules_without_config} schedules have NULL config_id. "
-            "Please set config_id for all schedules before running this migration."
-        )
-
-    # Drop config_file from scans table
-    with op.batch_alter_table('scans', schema=None) as batch_op:
-        batch_op.drop_column('config_file')
-
-    # Drop config_file from schedules table
-    with op.batch_alter_table('schedules', schema=None) as batch_op:
-        batch_op.drop_column('config_file')
-
-    print("✓ Migration complete: Dropped config_file columns")
-    print("  - Removed config_file from scans table")
-    print("  - Removed config_file from schedules table")
-    print("  - All references should now use config_id")
-
-
-def downgrade():
-    """Re-add config_file columns (data will be lost)."""
-
-    # Add config_file back to scans
-    with op.batch_alter_table('scans', schema=None) as batch_op:
-        batch_op.add_column(
-            sa.Column('config_file', sa.Text(), nullable=True,
-                     comment='Path to YAML config used (deprecated)')
-        )
-
-    # Add config_file back to schedules
-    with op.batch_alter_table('schedules', schema=None) as batch_op:
-        batch_op.add_column(
-            sa.Column('config_file', sa.Text(), nullable=True,
-                     comment='Path to YAML config (deprecated)')
-        )
-
-    print("✓ Downgrade complete: Re-added config_file columns")
-    print("  WARNING: config_file values are lost and will be NULL")

app/migrations/versions/012_add_scan_progress.py

@@ -1,58 +0,0 @@
-"""Add scan progress tracking
-
-Revision ID: 012
-Revises: 011
-Create Date: 2024-01-01 00:00:00.000000
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = '012'
-down_revision = '011'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    # Add progress tracking columns to scans table
-    op.add_column('scans', sa.Column('current_phase', sa.String(50), nullable=True,
-                  comment='Current scan phase: ping, tcp_scan, udp_scan, service_detection, http_analysis'))
-    op.add_column('scans', sa.Column('total_ips', sa.Integer(), nullable=True,
-                  comment='Total number of IPs to scan'))
-    op.add_column('scans', sa.Column('completed_ips', sa.Integer(), nullable=True, default=0,
-                  comment='Number of IPs completed in current phase'))
-
-    # Create scan_progress table for per-IP progress tracking
-    op.create_table(
-        'scan_progress',
-        sa.Column('id', sa.Integer(), primary_key=True, autoincrement=True),
-        sa.Column('scan_id', sa.Integer(), sa.ForeignKey('scans.id'), nullable=False, index=True),
-        sa.Column('ip_address', sa.String(45), nullable=False, comment='IP address being scanned'),
-        sa.Column('site_name', sa.String(255), nullable=True, comment='Site name this IP belongs to'),
-        sa.Column('phase', sa.String(50), nullable=False,
-                  comment='Phase: ping, tcp_scan, udp_scan, service_detection, http_analysis'),
-        sa.Column('status', sa.String(20), nullable=False, default='pending',
-                  comment='pending, in_progress, completed, failed'),
-        sa.Column('ping_result', sa.Boolean(), nullable=True, comment='Ping response result'),
-        sa.Column('tcp_ports', sa.Text(), nullable=True, comment='JSON array of discovered TCP ports'),
-        sa.Column('udp_ports', sa.Text(), nullable=True, comment='JSON array of discovered UDP ports'),
-        sa.Column('services', sa.Text(), nullable=True, comment='JSON array of detected services'),
-        sa.Column('created_at', sa.DateTime(), nullable=False, server_default=sa.func.now(),
-                  comment='Entry creation time'),
-        sa.Column('updated_at', sa.DateTime(), nullable=False, server_default=sa.func.now(),
-                  onupdate=sa.func.now(), comment='Last update time'),
-        sa.UniqueConstraint('scan_id', 'ip_address', name='uix_scan_progress_ip')
-    )
-
-
-def downgrade():
-    # Drop scan_progress table
-    op.drop_table('scan_progress')
-
-    # Remove progress tracking columns from scans table
-    op.drop_column('scans', 'completed_ips')
-    op.drop_column('scans', 'total_ips')
-    op.drop_column('scans', 'current_phase')

app/web/api/scans.py

@@ -7,6 +7,9 @@ scan results.
 
 import json
 import logging
+from datetime import datetime
+from pathlib import Path
+
 from flask import Blueprint, current_app, jsonify, request
 from sqlalchemy.exc import SQLAlchemyError
 
@@ -20,6 +23,89 @@ bp = Blueprint('scans', __name__)
 logger = logging.getLogger(__name__)
 
 
+def _recover_orphaned_scan(scan: Scan, session) -> dict:
+    """
+    Recover an orphaned scan by checking for output files.
+
+    If output files exist: mark as 'completed' (smart recovery)
+    If no output files: mark as 'cancelled'
+
+    Args:
+        scan: The orphaned Scan object
+        session: Database session
+
+    Returns:
+        Dictionary with recovery result for API response
+    """
+    # Check for existing output files
+    output_exists = False
+    output_files_found = []
+
+    # Check paths stored in database
+    if scan.json_path and Path(scan.json_path).exists():
+        output_exists = True
+        output_files_found.append('json')
+    if scan.html_path and Path(scan.html_path).exists():
+        output_files_found.append('html')
+    if scan.zip_path and Path(scan.zip_path).exists():
+        output_files_found.append('zip')
+
+    # Also check by timestamp pattern if paths not stored yet
+    if not output_exists and scan.started_at:
+        output_dir = Path('/app/output')
+        if output_dir.exists():
+            timestamp_pattern = scan.started_at.strftime('%Y%m%d')
+            for json_file in output_dir.glob(f'scan_report_{timestamp_pattern}*.json'):
+                output_exists = True
+                output_files_found.append('json')
+                # Update scan record with found paths
+                scan.json_path = str(json_file)
+                html_file = json_file.with_suffix('.html')
+                if html_file.exists():
+                    scan.html_path = str(html_file)
+                    output_files_found.append('html')
+                zip_file = json_file.with_suffix('.zip')
+                if zip_file.exists():
+                    scan.zip_path = str(zip_file)
+                    output_files_found.append('zip')
+                break
+
+    if output_exists:
+        # Smart recovery: outputs exist, mark as completed
+        scan.status = 'completed'
+        scan.completed_at = datetime.utcnow()
+        if scan.started_at:
+            scan.duration = (datetime.utcnow() - scan.started_at).total_seconds()
+        scan.error_message = None
+        session.commit()
+
+        logger.info(f"Scan {scan.id}: Recovered as completed (files: {output_files_found})")
+
+        return {
+            'scan_id': scan.id,
+            'status': 'completed',
+            'message': f'Scan recovered as completed (output files found: {", ".join(output_files_found)})',
+            'recovery_type': 'smart_recovery'
+        }
+    else:
+        # No outputs: mark as cancelled
+        scan.status = 'cancelled'
+        scan.completed_at = datetime.utcnow()
+        if scan.started_at:
+            scan.duration = (datetime.utcnow() - scan.started_at).total_seconds()
+        scan.error_message = 'Scan process was interrupted before completion. No output files were generated.'
+        session.commit()
+
+        logger.info(f"Scan {scan.id}: Marked as cancelled (orphaned, no output files)")
+
+        return {
+            'scan_id': scan.id,
+            'status': 'cancelled',
+            'message': 'Orphaned scan cancelled (no output files found)',
+            'recovery_type': 'orphan_cleanup'
+        }
+
+
 @bp.route('', methods=['GET'])
 @api_auth_required
 def list_scans():
@@ -247,18 +333,23 @@ def delete_scan(scan_id):
 @api_auth_required
 def stop_running_scan(scan_id):
     """
-    Stop a running scan.
+    Stop a running scan with smart recovery for orphaned scans.
+
+    If the scan is actively running in the registry, sends a cancel signal.
+    If the scan shows as running/finalizing but is not in the registry (orphaned),
+    performs smart recovery: marks as 'completed' if output files exist,
+    otherwise marks as 'cancelled'.
 
     Args:
         scan_id: Scan ID to stop
 
     Returns:
-        JSON response with stop status
+        JSON response with stop status or recovery result
     """
     try:
         session = current_app.db_session
 
-        # Check if scan exists and is running
+        # Check if scan exists
         scan = session.query(Scan).filter_by(id=scan_id).first()
         if not scan:
             logger.warning(f"Scan not found for stop request: {scan_id}")
@@ -267,7 +358,8 @@ def stop_running_scan(scan_id):
                 'message': f'Scan with ID {scan_id} not found'
             }), 404
 
-        if scan.status != 'running':
+        # Allow stopping scans with status 'running' or 'finalizing'
+        if scan.status not in ('running', 'finalizing'):
             logger.warning(f"Cannot stop scan {scan_id}: status is '{scan.status}'")
             return jsonify({
                 'error': 'Invalid state',
@@ -288,11 +380,11 @@ def stop_running_scan(scan_id):
                 'status': 'stopping'
             }), 200
         else:
-            logger.warning(f"Failed to stop scan {scan_id}: not found in running scanners")
-            return jsonify({
-                'error': 'Stop failed',
-                'message': 'Scan not found in running scanners registry'
-            }), 404
+            # Scanner not in registry - this is an orphaned scan
+            # Attempt smart recovery
+            logger.warning(f"Scan {scan_id} not in registry, attempting smart recovery")
+            recovery_result = _recover_orphaned_scan(scan, session)
+            return jsonify(recovery_result), 200
 
     except SQLAlchemyError as e:
         logger.error(f"Database error stopping scan {scan_id}: {str(e)}")

app/web/app.py

@@ -307,9 +307,12 @@ def init_scheduler(app: Flask) -> None:
     with app.app_context():
         # Clean up any orphaned scans from previous crashes/restarts
         scan_service = ScanService(app.db_session)
-        orphaned_count = scan_service.cleanup_orphaned_scans()
-        if orphaned_count > 0:
-            app.logger.warning(f"Cleaned up {orphaned_count} orphaned scan(s) on startup")
+        cleanup_result = scan_service.cleanup_orphaned_scans()
+        if cleanup_result['total'] > 0:
+            app.logger.warning(
+                f"Cleaned up {cleanup_result['total']} orphaned scan(s) on startup: "
+                f"{cleanup_result['recovered']} recovered, {cleanup_result['failed']} failed"
+            )
 
         # Load all enabled schedules from database
         scheduler.load_schedules_on_startup()

app/web/jobs/scan_job.py

@@ -240,14 +240,47 @@ def execute_scan(scan_id: int, config_id: int, db_url: str = None):
         scan_duration = (end_time - start_time).total_seconds()
         logger.info(f"Scan {scan_id}: Scanner completed in {scan_duration:.2f} seconds")
 
-        # Generate output files (JSON, HTML, ZIP)
-        logger.info(f"Scan {scan_id}: Generating output files...")
-        output_paths = scanner.generate_outputs(report, timestamp)
+        # Transition to 'finalizing' status before output generation
+        try:
+            scan = session.query(Scan).filter_by(id=scan_id).first()
+            if scan:
+                scan.status = 'finalizing'
+                scan.current_phase = 'generating_outputs'
+                session.commit()
+                logger.info(f"Scan {scan_id}: Status changed to 'finalizing'")
+        except Exception as e:
+            logger.error(f"Scan {scan_id}: Failed to update status to finalizing: {e}")
+            session.rollback()
 
-        # Save results to database
-        logger.info(f"Scan {scan_id}: Saving results to database...")
-        scan_service = ScanService(session)
-        scan_service._save_scan_to_db(report, scan_id, status='completed', output_paths=output_paths)
+        # Generate output files (JSON, HTML, ZIP) with error handling
+        output_paths = {}
+        output_generation_failed = False
+        try:
+            logger.info(f"Scan {scan_id}: Generating output files...")
+            output_paths = scanner.generate_outputs(report, timestamp)
+        except Exception as e:
+            output_generation_failed = True
+            logger.error(f"Scan {scan_id}: Output generation failed: {str(e)}")
+            logger.error(f"Scan {scan_id}: Traceback:\n{traceback.format_exc()}")
+            # Still mark scan as completed with warning since scan data is valid
+            try:
+                scan = session.query(Scan).filter_by(id=scan_id).first()
+                if scan:
+                    scan.status = 'completed'
+                    scan.error_message = f"Scan completed but output file generation failed: {str(e)}"
+                    scan.completed_at = datetime.utcnow()
+                    if scan.started_at:
+                        scan.duration = (datetime.utcnow() - scan.started_at).total_seconds()
+                    session.commit()
+                    logger.info(f"Scan {scan_id}: Marked as completed with output generation warning")
+            except Exception as db_error:
+                logger.error(f"Scan {scan_id}: Failed to update status after output error: {db_error}")
+
+        # Save results to database (only if output generation succeeded)
+        if not output_generation_failed:
+            logger.info(f"Scan {scan_id}: Saving results to database...")
+            scan_service = ScanService(session)
+            scan_service._save_scan_to_db(report, scan_id, status='completed', output_paths=output_paths)
 
         # Evaluate alert rules
         logger.info(f"Scan {scan_id}: Evaluating alert rules...")

app/web/models.py

@@ -45,7 +45,7 @@ class Scan(Base):
     id = Column(Integer, primary_key=True, autoincrement=True)
     timestamp = Column(DateTime, nullable=False, index=True, comment="Scan start time (UTC)")
     duration = Column(Float, nullable=True, comment="Total scan duration in seconds")
-    status = Column(String(20), nullable=False, default='running', comment="running, completed, failed")
+    status = Column(String(20), nullable=False, default='running', comment="running, finalizing, completed, failed, cancelled")
     config_id = Column(Integer, ForeignKey('scan_configs.id'), nullable=True, index=True, comment="FK to scan_configs table")
     title = Column(Text, nullable=True, comment="Scan title from config")
     json_path = Column(Text, nullable=True, comment="Path to JSON report")

app/web/services/scan_service.py

@@ -286,52 +286,96 @@ class ScanService:
 
         return [self._scan_to_summary_dict(scan) for scan in scans]
 
-    def cleanup_orphaned_scans(self) -> int:
+    def cleanup_orphaned_scans(self) -> dict:
         """
-        Clean up orphaned scans that are stuck in 'running' status.
+        Clean up orphaned scans with smart recovery.
+
+        For scans stuck in 'running' or 'finalizing' status:
+        - If output files exist: mark as 'completed' (smart recovery)
+        - If no output files: mark as 'failed'
 
         This should be called on application startup to handle scans that
         were running when the system crashed or was restarted.
 
-        Scans in 'running' status are marked as 'failed' with an appropriate
-        error message indicating they were orphaned.
-
         Returns:
-            Number of orphaned scans cleaned up
+            Dictionary with cleanup results: {'recovered': N, 'failed': N, 'total': N}
         """
-        # Find all scans with status='running'
-        orphaned_scans = self.db.query(Scan).filter(Scan.status == 'running').all()
+        # Find all scans with status='running' or 'finalizing'
+        orphaned_scans = self.db.query(Scan).filter(
+            Scan.status.in_(['running', 'finalizing'])
+        ).all()
 
         if not orphaned_scans:
             logger.info("No orphaned scans found")
-            return 0
+            return {'recovered': 0, 'failed': 0, 'total': 0}
 
         count = len(orphaned_scans)
-        logger.warning(f"Found {count} orphaned scan(s) in 'running' status, marking as failed")
+        logger.warning(f"Found {count} orphaned scan(s), attempting smart recovery")
+
+        recovered_count = 0
+        failed_count = 0
+        output_dir = Path('/app/output')
 
-        # Mark each orphaned scan as failed
         for scan in orphaned_scans:
-            scan.status = 'failed'
+            # Check for existing output files
+            output_exists = False
+            output_files_found = []
+
+            # Check paths stored in database
+            if scan.json_path and Path(scan.json_path).exists():
+                output_exists = True
+                output_files_found.append('json')
+            if scan.html_path and Path(scan.html_path).exists():
+                output_files_found.append('html')
+            if scan.zip_path and Path(scan.zip_path).exists():
+                output_files_found.append('zip')
+
+            # Also check by timestamp pattern if paths not stored yet
+            if not output_exists and scan.started_at and output_dir.exists():
+                timestamp_pattern = scan.started_at.strftime('%Y%m%d')
+                for json_file in output_dir.glob(f'scan_report_{timestamp_pattern}*.json'):
+                    output_exists = True
+                    output_files_found.append('json')
+                    # Update scan record with found paths
+                    scan.json_path = str(json_file)
+                    html_file = json_file.with_suffix('.html')
+                    if html_file.exists():
+                        scan.html_path = str(html_file)
+                        output_files_found.append('html')
+                    zip_file = json_file.with_suffix('.zip')
+                    if zip_file.exists():
+                        scan.zip_path = str(zip_file)
+                        output_files_found.append('zip')
+                    break
+
+            if output_exists:
+                # Smart recovery: outputs exist, mark as completed
+                scan.status = 'completed'
+                scan.error_message = f'Recovered from orphaned state (output files found: {", ".join(output_files_found)})'
+                recovered_count += 1
+                logger.info(f"Recovered orphaned scan {scan.id} as completed (files: {output_files_found})")
+            else:
+                # No outputs: mark as failed
+                scan.status = 'failed'
+                scan.error_message = (
+                    "Scan was interrupted by system shutdown or crash. "
+                    "No output files were generated."
+                )
+                failed_count += 1
+                logger.info(f"Marked orphaned scan {scan.id} as failed (no output files)")
+
             scan.completed_at = datetime.utcnow()
-            scan.error_message = (
-                "Scan was interrupted by system shutdown or crash. "
-                "The scan was running but did not complete normally."
-            )
-
-            # Calculate duration if we have a started_at time
             if scan.started_at:
-                duration = (datetime.utcnow() - scan.started_at).total_seconds()
-                scan.duration = duration
-
-            logger.info(
-                f"Marked orphaned scan {scan.id} as failed "
-                f"(started: {scan.started_at.isoformat() if scan.started_at else 'unknown'})"
-            )
+                scan.duration = (datetime.utcnow() - scan.started_at).total_seconds()
 
         self.db.commit()
-        logger.info(f"Cleaned up {count} orphaned scan(s)")
+        logger.info(f"Cleaned up {count} orphaned scan(s): {recovered_count} recovered, {failed_count} failed")
 
-        return count
+        return {
+            'recovered': recovered_count,
+            'failed': failed_count,
+            'total': count
+        }
 
     def _save_scan_to_db(self, report: Dict[str, Any], scan_id: int,
                         status: str = 'completed', output_paths: Dict = None) -> None:

app/web/utils/validators.py

@@ -23,7 +23,7 @@ def validate_scan_status(status: str) -> tuple[bool, Optional[str]]:
         >>> validate_scan_status('invalid')
         (False, 'Invalid status: invalid. Must be one of: running, completed, failed')
     """
-    valid_statuses = ['running', 'completed', 'failed', 'cancelled']
+    valid_statuses = ['running', 'finalizing', 'completed', 'failed', 'cancelled']
 
     if status not in valid_statuses:
         return False, f'Invalid status: {status}. Must be one of: {", ".join(valid_statuses)}'

docker-compose.yml

@@ -2,12 +2,10 @@ version: '3.8'
 
 services:
   web:
-    build: .
-    image: sneakyscanner:latest
+    image: sneakyscan
     container_name: sneakyscanner-web
-    # Use entrypoint script that auto-initializes database on first run
-    entrypoint: ["/docker-entrypoint.sh"]
-    command: ["python3", "-u", "-m", "web.app"]
+    working_dir: /app
+    entrypoint: ["python3", "-u", "-m", "web.app"]
     # Note: Using host network mode for scanner capabilities, so no port mapping needed
     # The Flask app will be accessible at http://localhost:5000
     volumes:
@@ -59,8 +57,7 @@ services:
   # Optional: Initialize database on first run
   # Run with: docker-compose -f docker-compose-web.yml run --rm init-db
   init-db:
-    build: .
-    image: sneakyscanner:latest
+    image: sneakyscan
     container_name: sneakyscanner-init-db
     entrypoint: ["python3"]
     command: ["init_db.py", "--db-url", "sqlite:////app/data/sneakyscanner.db"]
@@ -68,3 +65,4 @@ services:
       - ./data:/app/data
     profiles:
       - tools
+    networks: []

setup.sh

@@ -91,27 +91,40 @@ echo "Creating required directories..."
 mkdir -p data logs output configs
 echo "✓ Directories created"
 
-# Check if Docker is running
+# Check if Podman is running
 echo ""
-echo "Checking Docker..."
-if ! docker info > /dev/null 2>&1; then
-    echo "✗ Docker is not running or not installed"
-    echo "Please install Docker and start the Docker daemon"
+echo "Checking Podman..."
+if ! podman info > /dev/null 2>&1; then
+    echo "✗ Podman is not running or not installed"
+    echo "Please install Podman"
     exit 1
 fi
-echo "✓ Docker is running"
+echo "✓ Podman is available"
 
 # Build and start
 echo ""
-echo "Building and starting SneakyScanner..."
+echo "Starting SneakyScanner..."
 echo "This may take a few minutes on first run..."
 echo ""
 
-docker compose build
+podman build --network=host -t sneakyscan .
 
+# Initialize database if it doesn't exist or is empty
 echo ""
-echo "Starting SneakyScanner..."
-docker compose up -d
+echo "Initializing database..."
+
+# Build init command with optional password
+INIT_CMD="init_db.py --db-url sqlite:////app/data/sneakyscanner.db --force"
+if [ -n "$INITIAL_PASSWORD" ]; then
+    INIT_CMD="$INIT_CMD --password $INITIAL_PASSWORD"
+fi
+
+podman run --rm --entrypoint python3 -w /app \
+    -v "$(pwd)/data:/app/data" \
+    sneakyscan $INIT_CMD
+echo "✓ Database initialized"
+
+podman-compose up -d
 
 # Wait for service to be healthy
 echo ""
@@ -119,7 +132,7 @@ echo "Waiting for application to start..."
 sleep 5
 
 # Check if container is running
-if docker ps | grep -q sneakyscanner-web; then
+if podman ps | grep -q sneakyscanner-web; then
     echo ""
     echo "================================================"
     echo "  ✓ SneakyScanner is Running!"
@@ -140,15 +153,15 @@ if docker ps | grep -q sneakyscanner-web; then
     fi
     echo ""
     echo "Useful commands:"
-    echo "  docker compose logs -f     # View logs"
-    echo "  docker compose stop        # Stop the service"
-    echo "  docker compose restart     # Restart the service"
+    echo "  podman-compose logs -f     # View logs"
+    echo "  podman-compose stop        # Stop the service"
+    echo "  podman-compose restart     # Restart the service"
     echo ""
     echo "⚠  IMPORTANT: Change your password after first login!"
     echo "================================================"
 else
     echo ""
     echo "✗ Container failed to start. Check logs with:"
-    echo "  docker compose logs"
+    echo "  podman-compose logs"
     exit 1
 fi