database fixes / simplification

Reviewed-on: #10

Dockerfile

@@ -39,13 +39,12 @@ COPY app/web/ ./web/
 COPY app/migrations/ ./migrations/
 COPY app/alembic.ini .
 COPY app/init_db.py .
-COPY app/docker-entrypoint.sh /docker-entrypoint.sh
 
 # Create required directories
 RUN mkdir -p /app/output /app/logs
 
 # Make scripts executable
-RUN chmod +x /app/src/scanner.py /app/init_db.py /docker-entrypoint.sh
+RUN chmod +x /app/src/scanner.py /app/init_db.py
 
 # Force Python unbuffered output
 ENV PYTHONUNBUFFERED=1

app/migrations/env.py

@@ -69,8 +69,12 @@ def run_migrations_online() -> None:
     )
 
     with connectable.connect() as connection:
+        # Enable batch mode for SQLite to support ALTER TABLE operations
+        # like DROP COLUMN which SQLite doesn't natively support
         context.configure(
-            connection=connection, target_metadata=target_metadata
+            connection=connection,
+            target_metadata=target_metadata,
+            render_as_batch=True
         )
 
         with context.begin_transaction():

app/migrations/versions/001_initial_schema.py

@@ -1,125 +1,214 @@
-"""Initial database schema for SneakyScanner
+"""Initial schema for SneakyScanner
 
 Revision ID: 001
-Revises:
+Revises: None
-Create Date: 2025-11-13 18:00:00.000000
+Create Date: 2025-12-24
 
+This is the complete initial schema for SneakyScanner. All tables are created
+in the correct order to satisfy foreign key constraints.
 """
 from alembic import op
 import sqlalchemy as sa
 
 
-# revision identifiers, used by Alembic.
+# revision identifiers, used by Alembic
 revision = '001'
 down_revision = None
 branch_labels = None
 depends_on = None
 
 
-def upgrade() -> None:
+def upgrade():
-    """Create all initial tables for SneakyScanner."""
+    """Create all tables for SneakyScanner."""
 
-    # Create schedules table first (referenced by scans)
+    # =========================================================================
-    op.create_table('schedules',
+    # Settings Table (no dependencies)
+    # =========================================================================
+    op.create_table(
+        'settings',
         sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
-        sa.Column('name', sa.String(length=255), nullable=False, comment='Schedule name (e.g., \'Daily prod scan\')'),
+        sa.Column('key', sa.String(length=255), nullable=False, comment='Setting key'),
-        sa.Column('config_file', sa.Text(), nullable=False, comment='Path to YAML config'),
+        sa.Column('value', sa.Text(), nullable=True, comment='Setting value (JSON for complex values)'),
-        sa.Column('cron_expression', sa.String(length=100), nullable=False, comment='Cron-like schedule (e.g., \'0 2 * * *\')'),
+        sa.Column('updated_at', sa.DateTime(), nullable=False, comment='Last modification time'),
-        sa.Column('enabled', sa.Boolean(), nullable=False, comment='Is schedule active?'),
+        sa.PrimaryKeyConstraint('id'),
-        sa.Column('last_run', sa.DateTime(), nullable=True, comment='Last execution time'),
+        sa.UniqueConstraint('key')
-        sa.Column('next_run', sa.DateTime(), nullable=True, comment='Next scheduled execution'),
+    )
-        sa.Column('created_at', sa.DateTime(), nullable=False, comment='Schedule creation time'),
+    op.create_index('ix_settings_key', 'settings', ['key'], unique=True)
+
+    # =========================================================================
+    # Reusable Site Definition Tables
+    # =========================================================================
+    op.create_table(
+        'sites',
+        sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
+        sa.Column('name', sa.String(length=255), nullable=False, comment='Unique site name'),
+        sa.Column('description', sa.Text(), nullable=True, comment='Site description'),
+        sa.Column('created_at', sa.DateTime(), nullable=False, comment='Site creation time'),
+        sa.Column('updated_at', sa.DateTime(), nullable=False, comment='Last modification time'),
+        sa.PrimaryKeyConstraint('id'),
+        sa.UniqueConstraint('name')
+    )
+    op.create_index('ix_sites_name', 'sites', ['name'], unique=True)
+
+    op.create_table(
+        'site_ips',
+        sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
+        sa.Column('site_id', sa.Integer(), nullable=False, comment='FK to sites'),
+        sa.Column('ip_address', sa.String(length=45), nullable=False, comment='IPv4 or IPv6 address'),
+        sa.Column('expected_ping', sa.Boolean(), nullable=True, comment='Expected ping response'),
+        sa.Column('expected_tcp_ports', sa.Text(), nullable=True, comment='JSON array of expected TCP ports'),
+        sa.Column('expected_udp_ports', sa.Text(), nullable=True, comment='JSON array of expected UDP ports'),
+        sa.Column('created_at', sa.DateTime(), nullable=False, comment='IP creation time'),
+        sa.ForeignKeyConstraint(['site_id'], ['sites.id']),
+        sa.PrimaryKeyConstraint('id'),
+        sa.UniqueConstraint('site_id', 'ip_address', name='uix_site_ip_address')
+    )
+    op.create_index('ix_site_ips_site_id', 'site_ips', ['site_id'])
+
+    # =========================================================================
+    # Scan Configuration Tables
+    # =========================================================================
+    op.create_table(
+        'scan_configs',
+        sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
+        sa.Column('title', sa.String(length=255), nullable=False, comment='Configuration title'),
+        sa.Column('description', sa.Text(), nullable=True, comment='Configuration description'),
+        sa.Column('created_at', sa.DateTime(), nullable=False, comment='Config creation time'),
         sa.Column('updated_at', sa.DateTime(), nullable=False, comment='Last modification time'),
         sa.PrimaryKeyConstraint('id')
     )
 
-    # Create scans table
+    op.create_table(
-    op.create_table('scans',
+        'scan_config_sites',
+        sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
+        sa.Column('config_id', sa.Integer(), nullable=False),
+        sa.Column('site_id', sa.Integer(), nullable=False),
+        sa.Column('created_at', sa.DateTime(), nullable=False, comment='Association creation time'),
+        sa.ForeignKeyConstraint(['config_id'], ['scan_configs.id']),
+        sa.ForeignKeyConstraint(['site_id'], ['sites.id']),
+        sa.PrimaryKeyConstraint('id'),
+        sa.UniqueConstraint('config_id', 'site_id', name='uix_config_site')
+    )
+    op.create_index('ix_scan_config_sites_config_id', 'scan_config_sites', ['config_id'])
+    op.create_index('ix_scan_config_sites_site_id', 'scan_config_sites', ['site_id'])
+
+    # =========================================================================
+    # Scheduling Tables
+    # =========================================================================
+    op.create_table(
+        'schedules',
+        sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
+        sa.Column('name', sa.String(length=255), nullable=False, comment='Schedule name'),
+        sa.Column('config_id', sa.Integer(), nullable=True, comment='FK to scan_configs table'),
+        sa.Column('cron_expression', sa.String(length=100), nullable=False, comment='Cron-like schedule'),
+        sa.Column('enabled', sa.Boolean(), nullable=False, default=True, comment='Is schedule active?'),
+        sa.Column('last_run', sa.DateTime(), nullable=True, comment='Last execution time'),
+        sa.Column('next_run', sa.DateTime(), nullable=True, comment='Next scheduled execution'),
+        sa.Column('created_at', sa.DateTime(), nullable=False, comment='Schedule creation time'),
+        sa.Column('updated_at', sa.DateTime(), nullable=False, comment='Last modification time'),
+        sa.ForeignKeyConstraint(['config_id'], ['scan_configs.id']),
+        sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index('ix_schedules_config_id', 'schedules', ['config_id'])
+
+    # =========================================================================
+    # Core Scan Tables
+    # =========================================================================
+    op.create_table(
+        'scans',
         sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
         sa.Column('timestamp', sa.DateTime(), nullable=False, comment='Scan start time (UTC)'),
         sa.Column('duration', sa.Float(), nullable=True, comment='Total scan duration in seconds'),
-        sa.Column('status', sa.String(length=20), nullable=False, comment='running, completed, failed'),
+        sa.Column('status', sa.String(length=20), nullable=False, default='running', comment='running, finalizing, completed, failed, cancelled'),
-        sa.Column('config_file', sa.Text(), nullable=True, comment='Path to YAML config used'),
+        sa.Column('config_id', sa.Integer(), nullable=True, comment='FK to scan_configs table'),
         sa.Column('title', sa.Text(), nullable=True, comment='Scan title from config'),
         sa.Column('json_path', sa.Text(), nullable=True, comment='Path to JSON report'),
         sa.Column('html_path', sa.Text(), nullable=True, comment='Path to HTML report'),
         sa.Column('zip_path', sa.Text(), nullable=True, comment='Path to ZIP archive'),
         sa.Column('screenshot_dir', sa.Text(), nullable=True, comment='Path to screenshot directory'),
         sa.Column('created_at', sa.DateTime(), nullable=False, comment='Record creation time'),
-        sa.Column('triggered_by', sa.String(length=50), nullable=False, comment='manual, scheduled, api'),
+        sa.Column('triggered_by', sa.String(length=50), nullable=False, default='manual', comment='manual, scheduled, api'),
         sa.Column('schedule_id', sa.Integer(), nullable=True, comment='FK to schedules if triggered by schedule'),
-        sa.ForeignKeyConstraint(['schedule_id'], ['schedules.id'], ),
+        sa.Column('started_at', sa.DateTime(), nullable=True, comment='Scan execution start time'),
+        sa.Column('completed_at', sa.DateTime(), nullable=True, comment='Scan execution completion time'),
+        sa.Column('error_message', sa.Text(), nullable=True, comment='Error message if scan failed'),
+        sa.Column('current_phase', sa.String(length=50), nullable=True, comment='Current scan phase'),
+        sa.Column('total_ips', sa.Integer(), nullable=True, comment='Total number of IPs to scan'),
+        sa.Column('completed_ips', sa.Integer(), nullable=True, default=0, comment='Number of IPs completed'),
+        sa.ForeignKeyConstraint(['config_id'], ['scan_configs.id']),
+        sa.ForeignKeyConstraint(['schedule_id'], ['schedules.id']),
         sa.PrimaryKeyConstraint('id')
     )
-    op.create_index(op.f('ix_scans_timestamp'), 'scans', ['timestamp'], unique=False)
+    op.create_index('ix_scans_timestamp', 'scans', ['timestamp'])
+    op.create_index('ix_scans_config_id', 'scans', ['config_id'])
 
-    # Create scan_sites table
+    op.create_table(
-    op.create_table('scan_sites',
+        'scan_sites',
         sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
-        sa.Column('scan_id', sa.Integer(), nullable=False, comment='FK to scans'),
+        sa.Column('scan_id', sa.Integer(), nullable=False),
         sa.Column('site_name', sa.String(length=255), nullable=False, comment='Site name from config'),
-        sa.ForeignKeyConstraint(['scan_id'], ['scans.id'], ),
+        sa.ForeignKeyConstraint(['scan_id'], ['scans.id']),
         sa.PrimaryKeyConstraint('id')
     )
-    op.create_index(op.f('ix_scan_sites_scan_id'), 'scan_sites', ['scan_id'], unique=False)
+    op.create_index('ix_scan_sites_scan_id', 'scan_sites', ['scan_id'])
 
-    # Create scan_ips table
+    op.create_table(
-    op.create_table('scan_ips',
+        'scan_ips',
         sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
-        sa.Column('scan_id', sa.Integer(), nullable=False, comment='FK to scans'),
+        sa.Column('scan_id', sa.Integer(), nullable=False),
-        sa.Column('site_id', sa.Integer(), nullable=False, comment='FK to scan_sites'),
+        sa.Column('site_id', sa.Integer(), nullable=False),
         sa.Column('ip_address', sa.String(length=45), nullable=False, comment='IPv4 or IPv6 address'),
         sa.Column('ping_expected', sa.Boolean(), nullable=True, comment='Expected ping response'),
         sa.Column('ping_actual', sa.Boolean(), nullable=True, comment='Actual ping response'),
-        sa.ForeignKeyConstraint(['scan_id'], ['scans.id'], ),
+        sa.ForeignKeyConstraint(['scan_id'], ['scans.id']),
-        sa.ForeignKeyConstraint(['site_id'], ['scan_sites.id'], ),
+        sa.ForeignKeyConstraint(['site_id'], ['scan_sites.id']),
         sa.PrimaryKeyConstraint('id'),
         sa.UniqueConstraint('scan_id', 'ip_address', name='uix_scan_ip')
     )
-    op.create_index(op.f('ix_scan_ips_scan_id'), 'scan_ips', ['scan_id'], unique=False)
+    op.create_index('ix_scan_ips_scan_id', 'scan_ips', ['scan_id'])
-    op.create_index(op.f('ix_scan_ips_site_id'), 'scan_ips', ['site_id'], unique=False)
+    op.create_index('ix_scan_ips_site_id', 'scan_ips', ['site_id'])
 
-    # Create scan_ports table
+    op.create_table(
-    op.create_table('scan_ports',
+        'scan_ports',
         sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
-        sa.Column('scan_id', sa.Integer(), nullable=False, comment='FK to scans'),
+        sa.Column('scan_id', sa.Integer(), nullable=False),
-        sa.Column('ip_id', sa.Integer(), nullable=False, comment='FK to scan_ips'),
+        sa.Column('ip_id', sa.Integer(), nullable=False),
         sa.Column('port', sa.Integer(), nullable=False, comment='Port number (1-65535)'),
         sa.Column('protocol', sa.String(length=10), nullable=False, comment='tcp or udp'),
         sa.Column('expected', sa.Boolean(), nullable=True, comment='Was this port expected?'),
-        sa.Column('state', sa.String(length=20), nullable=False, comment='open, closed, filtered'),
+        sa.Column('state', sa.String(length=20), nullable=False, default='open', comment='open, closed, filtered'),
-        sa.ForeignKeyConstraint(['ip_id'], ['scan_ips.id'], ),
+        sa.ForeignKeyConstraint(['scan_id'], ['scans.id']),
-        sa.ForeignKeyConstraint(['scan_id'], ['scans.id'], ),
+        sa.ForeignKeyConstraint(['ip_id'], ['scan_ips.id']),
         sa.PrimaryKeyConstraint('id'),
         sa.UniqueConstraint('scan_id', 'ip_id', 'port', 'protocol', name='uix_scan_ip_port')
     )
-    op.create_index(op.f('ix_scan_ports_ip_id'), 'scan_ports', ['ip_id'], unique=False)
+    op.create_index('ix_scan_ports_scan_id', 'scan_ports', ['scan_id'])
-    op.create_index(op.f('ix_scan_ports_scan_id'), 'scan_ports', ['scan_id'], unique=False)
+    op.create_index('ix_scan_ports_ip_id', 'scan_ports', ['ip_id'])
 
-    # Create scan_services table
+    op.create_table(
-    op.create_table('scan_services',
+        'scan_services',
         sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
-        sa.Column('scan_id', sa.Integer(), nullable=False, comment='FK to scans'),
+        sa.Column('scan_id', sa.Integer(), nullable=False),
-        sa.Column('port_id', sa.Integer(), nullable=False, comment='FK to scan_ports'),
+        sa.Column('port_id', sa.Integer(), nullable=False),
-        sa.Column('service_name', sa.String(length=100), nullable=True, comment='Service name (e.g., ssh, http)'),
+        sa.Column('service_name', sa.String(length=100), nullable=True, comment='Service name'),
-        sa.Column('product', sa.String(length=255), nullable=True, comment='Product name (e.g., OpenSSH)'),
+        sa.Column('product', sa.String(length=255), nullable=True, comment='Product name'),
         sa.Column('version', sa.String(length=100), nullable=True, comment='Version string'),
         sa.Column('extrainfo', sa.Text(), nullable=True, comment='Additional nmap info'),
         sa.Column('ostype', sa.String(length=100), nullable=True, comment='OS type if detected'),
-        sa.Column('http_protocol', sa.String(length=10), nullable=True, comment='http or https (if web service)'),
+        sa.Column('http_protocol', sa.String(length=10), nullable=True, comment='http or https'),
         sa.Column('screenshot_path', sa.Text(), nullable=True, comment='Relative path to screenshot'),
-        sa.ForeignKeyConstraint(['port_id'], ['scan_ports.id'], ),
+        sa.ForeignKeyConstraint(['scan_id'], ['scans.id']),
-        sa.ForeignKeyConstraint(['scan_id'], ['scans.id'], ),
+        sa.ForeignKeyConstraint(['port_id'], ['scan_ports.id']),
         sa.PrimaryKeyConstraint('id')
     )
-    op.create_index(op.f('ix_scan_services_port_id'), 'scan_services', ['port_id'], unique=False)
+    op.create_index('ix_scan_services_scan_id', 'scan_services', ['scan_id'])
-    op.create_index(op.f('ix_scan_services_scan_id'), 'scan_services', ['scan_id'], unique=False)
+    op.create_index('ix_scan_services_port_id', 'scan_services', ['port_id'])
 
-    # Create scan_certificates table
+    op.create_table(
-    op.create_table('scan_certificates',
+        'scan_certificates',
         sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
-        sa.Column('scan_id', sa.Integer(), nullable=False, comment='FK to scans'),
+        sa.Column('scan_id', sa.Integer(), nullable=False),
-        sa.Column('service_id', sa.Integer(), nullable=False, comment='FK to scan_services'),
+        sa.Column('service_id', sa.Integer(), nullable=False),
         sa.Column('subject', sa.Text(), nullable=True, comment='Certificate subject (CN)'),
         sa.Column('issuer', sa.Text(), nullable=True, comment='Certificate issuer'),
         sa.Column('serial_number', sa.Text(), nullable=True, comment='Serial number'),
@@ -127,95 +216,177 @@ def upgrade() -> None:
         sa.Column('not_valid_after', sa.DateTime(), nullable=True, comment='Validity end date'),
         sa.Column('days_until_expiry', sa.Integer(), nullable=True, comment='Days until expiration'),
         sa.Column('sans', sa.Text(), nullable=True, comment='JSON array of SANs'),
-        sa.Column('is_self_signed', sa.Boolean(), nullable=True, comment='Self-signed certificate flag'),
+        sa.Column('is_self_signed', sa.Boolean(), nullable=True, default=False, comment='Self-signed flag'),
-        sa.ForeignKeyConstraint(['scan_id'], ['scans.id'], ),
+        sa.ForeignKeyConstraint(['scan_id'], ['scans.id']),
-        sa.ForeignKeyConstraint(['service_id'], ['scan_services.id'], ),
+        sa.ForeignKeyConstraint(['service_id'], ['scan_services.id']),
-        sa.PrimaryKeyConstraint('id'),
+        sa.PrimaryKeyConstraint('id')
-        comment='Index on expiration date for alert queries'
     )
-    op.create_index(op.f('ix_scan_certificates_scan_id'), 'scan_certificates', ['scan_id'], unique=False)
+    op.create_index('ix_scan_certificates_scan_id', 'scan_certificates', ['scan_id'])
-    op.create_index(op.f('ix_scan_certificates_service_id'), 'scan_certificates', ['service_id'], unique=False)
+    op.create_index('ix_scan_certificates_service_id', 'scan_certificates', ['service_id'])
 
-    # Create scan_tls_versions table
+    op.create_table(
-    op.create_table('scan_tls_versions',
+        'scan_tls_versions',
         sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
-        sa.Column('scan_id', sa.Integer(), nullable=False, comment='FK to scans'),
+        sa.Column('scan_id', sa.Integer(), nullable=False),
-        sa.Column('certificate_id', sa.Integer(), nullable=False, comment='FK to scan_certificates'),
+        sa.Column('certificate_id', sa.Integer(), nullable=False),
-        sa.Column('tls_version', sa.String(length=20), nullable=False, comment='TLS 1.0, TLS 1.1, TLS 1.2, TLS 1.3'),
+        sa.Column('tls_version', sa.String(length=20), nullable=False, comment='TLS 1.0, 1.1, 1.2, 1.3'),
         sa.Column('supported', sa.Boolean(), nullable=False, comment='Is this version supported?'),
         sa.Column('cipher_suites', sa.Text(), nullable=True, comment='JSON array of cipher suites'),
-        sa.ForeignKeyConstraint(['certificate_id'], ['scan_certificates.id'], ),
+        sa.ForeignKeyConstraint(['scan_id'], ['scans.id']),
-        sa.ForeignKeyConstraint(['scan_id'], ['scans.id'], ),
+        sa.ForeignKeyConstraint(['certificate_id'], ['scan_certificates.id']),
         sa.PrimaryKeyConstraint('id')
     )
-    op.create_index(op.f('ix_scan_tls_versions_certificate_id'), 'scan_tls_versions', ['certificate_id'], unique=False)
+    op.create_index('ix_scan_tls_versions_scan_id', 'scan_tls_versions', ['scan_id'])
-    op.create_index(op.f('ix_scan_tls_versions_scan_id'), 'scan_tls_versions', ['scan_id'], unique=False)
+    op.create_index('ix_scan_tls_versions_certificate_id', 'scan_tls_versions', ['certificate_id'])
 
-    # Create alerts table
+    op.create_table(
-    op.create_table('alerts',
+        'scan_progress',
         sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
-        sa.Column('scan_id', sa.Integer(), nullable=False, comment='FK to scans'),
+        sa.Column('scan_id', sa.Integer(), nullable=False),
-        sa.Column('alert_type', sa.String(length=50), nullable=False, comment='new_port, cert_expiry, service_change, ping_failed'),
+        sa.Column('ip_address', sa.String(length=45), nullable=False, comment='IP address being scanned'),
-        sa.Column('severity', sa.String(length=20), nullable=False, comment='info, warning, critical'),
+        sa.Column('site_name', sa.String(length=255), nullable=True, comment='Site name'),
-        sa.Column('message', sa.Text(), nullable=False, comment='Human-readable alert message'),
+        sa.Column('phase', sa.String(length=50), nullable=False, comment='Phase: ping, tcp_scan, etc.'),
-        sa.Column('ip_address', sa.String(length=45), nullable=True, comment='Related IP (optional)'),
+        sa.Column('status', sa.String(length=20), nullable=False, default='pending', comment='pending, in_progress, completed, failed'),
-        sa.Column('port', sa.Integer(), nullable=True, comment='Related port (optional)'),
+        sa.Column('ping_result', sa.Boolean(), nullable=True, comment='Ping response result'),
-        sa.Column('email_sent', sa.Boolean(), nullable=False, comment='Was email notification sent?'),
+        sa.Column('tcp_ports', sa.Text(), nullable=True, comment='JSON array of TCP ports'),
-        sa.Column('email_sent_at', sa.DateTime(), nullable=True, comment='Email send timestamp'),
+        sa.Column('udp_ports', sa.Text(), nullable=True, comment='JSON array of UDP ports'),
-        sa.Column('created_at', sa.DateTime(), nullable=False, comment='Alert creation time'),
+        sa.Column('services', sa.Text(), nullable=True, comment='JSON array of services'),
-        sa.ForeignKeyConstraint(['scan_id'], ['scans.id'], ),
+        sa.Column('created_at', sa.DateTime(), nullable=False, comment='Entry creation time'),
+        sa.Column('updated_at', sa.DateTime(), nullable=False, comment='Last update time'),
+        sa.ForeignKeyConstraint(['scan_id'], ['scans.id']),
         sa.PrimaryKeyConstraint('id'),
-        comment='Indexes for alert filtering'
+        sa.UniqueConstraint('scan_id', 'ip_address', name='uix_scan_progress_ip')
     )
-    op.create_index(op.f('ix_alerts_scan_id'), 'alerts', ['scan_id'], unique=False)
+    op.create_index('ix_scan_progress_scan_id', 'scan_progress', ['scan_id'])
 
-    # Create alert_rules table
+    op.create_table(
-    op.create_table('alert_rules',
+        'scan_site_associations',
         sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
-        sa.Column('rule_type', sa.String(length=50), nullable=False, comment='unexpected_port, cert_expiry, service_down, etc.'),
+        sa.Column('scan_id', sa.Integer(), nullable=False),
-        sa.Column('enabled', sa.Boolean(), nullable=False, comment='Is rule active?'),
+        sa.Column('site_id', sa.Integer(), nullable=False),
-        sa.Column('threshold', sa.Integer(), nullable=True, comment='Threshold value (e.g., days for cert expiry)'),
+        sa.Column('created_at', sa.DateTime(), nullable=False, comment='Association creation time'),
-        sa.Column('email_enabled', sa.Boolean(), nullable=False, comment='Send email for this rule?'),
+        sa.ForeignKeyConstraint(['scan_id'], ['scans.id']),
+        sa.ForeignKeyConstraint(['site_id'], ['sites.id']),
+        sa.PrimaryKeyConstraint('id'),
+        sa.UniqueConstraint('scan_id', 'site_id', name='uix_scan_site')
+    )
+    op.create_index('ix_scan_site_associations_scan_id', 'scan_site_associations', ['scan_id'])
+    op.create_index('ix_scan_site_associations_site_id', 'scan_site_associations', ['site_id'])
+
+    # =========================================================================
+    # Alert Tables
+    # =========================================================================
+    op.create_table(
+        'alert_rules',
+        sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
+        sa.Column('name', sa.String(length=255), nullable=True, comment='User-friendly rule name'),
+        sa.Column('rule_type', sa.String(length=50), nullable=False, comment='unexpected_port, cert_expiry, etc.'),
+        sa.Column('enabled', sa.Boolean(), nullable=False, default=True, comment='Is rule active?'),
+        sa.Column('threshold', sa.Integer(), nullable=True, comment='Threshold value'),
+        sa.Column('email_enabled', sa.Boolean(), nullable=False, default=False, comment='Send email?'),
+        sa.Column('webhook_enabled', sa.Boolean(), nullable=False, default=False, comment='Send webhook?'),
+        sa.Column('severity', sa.String(length=20), nullable=True, comment='critical, warning, info'),
+        sa.Column('filter_conditions', sa.Text(), nullable=True, comment='JSON filter conditions'),
+        sa.Column('config_id', sa.Integer(), nullable=True, comment='Optional: specific config this rule applies to'),
         sa.Column('created_at', sa.DateTime(), nullable=False, comment='Rule creation time'),
+        sa.Column('updated_at', sa.DateTime(), nullable=True, comment='Last update time'),
+        sa.ForeignKeyConstraint(['config_id'], ['scan_configs.id']),
+        sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index('ix_alert_rules_config_id', 'alert_rules', ['config_id'])
+
+    op.create_table(
+        'alerts',
+        sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
+        sa.Column('scan_id', sa.Integer(), nullable=False),
+        sa.Column('rule_id', sa.Integer(), nullable=True, comment='Associated alert rule'),
+        sa.Column('alert_type', sa.String(length=50), nullable=False, comment='Alert type'),
+        sa.Column('severity', sa.String(length=20), nullable=False, comment='info, warning, critical'),
+        sa.Column('message', sa.Text(), nullable=False, comment='Human-readable message'),
+        sa.Column('ip_address', sa.String(length=45), nullable=True, comment='Related IP'),
+        sa.Column('port', sa.Integer(), nullable=True, comment='Related port'),
+        sa.Column('email_sent', sa.Boolean(), nullable=False, default=False, comment='Was email sent?'),
+        sa.Column('email_sent_at', sa.DateTime(), nullable=True, comment='Email send timestamp'),
+        sa.Column('webhook_sent', sa.Boolean(), nullable=False, default=False, comment='Was webhook sent?'),
+        sa.Column('webhook_sent_at', sa.DateTime(), nullable=True, comment='Webhook send timestamp'),
+        sa.Column('acknowledged', sa.Boolean(), nullable=False, default=False, comment='Was alert acknowledged?'),
+        sa.Column('acknowledged_at', sa.DateTime(), nullable=True, comment='Acknowledgment timestamp'),
+        sa.Column('acknowledged_by', sa.String(length=255), nullable=True, comment='User who acknowledged'),
+        sa.Column('created_at', sa.DateTime(), nullable=False, comment='Alert creation time'),
+        sa.ForeignKeyConstraint(['scan_id'], ['scans.id']),
+        sa.ForeignKeyConstraint(['rule_id'], ['alert_rules.id']),
+        sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index('ix_alerts_scan_id', 'alerts', ['scan_id'])
+    op.create_index('ix_alerts_rule_id', 'alerts', ['rule_id'])
+    op.create_index('ix_alerts_acknowledged', 'alerts', ['acknowledged'])
+
+    # =========================================================================
+    # Webhook Tables
+    # =========================================================================
+    op.create_table(
+        'webhooks',
+        sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
+        sa.Column('name', sa.String(length=255), nullable=False, comment='Webhook name'),
+        sa.Column('url', sa.Text(), nullable=False, comment='Webhook URL'),
+        sa.Column('enabled', sa.Boolean(), nullable=False, default=True, comment='Is webhook enabled?'),
+        sa.Column('auth_type', sa.String(length=20), nullable=True, comment='none, bearer, basic, custom'),
+        sa.Column('auth_token', sa.Text(), nullable=True, comment='Encrypted auth token'),
+        sa.Column('custom_headers', sa.Text(), nullable=True, comment='JSON custom headers'),
+        sa.Column('alert_types', sa.Text(), nullable=True, comment='JSON array of alert types'),
+        sa.Column('severity_filter', sa.Text(), nullable=True, comment='JSON array of severities'),
+        sa.Column('timeout', sa.Integer(), nullable=True, default=10, comment='Request timeout'),
+        sa.Column('retry_count', sa.Integer(), nullable=True, default=3, comment='Retry attempts'),
+        sa.Column('template', sa.Text(), nullable=True, comment='Jinja2 template for payload'),
+        sa.Column('template_format', sa.String(length=20), nullable=True, default='json', comment='json, text'),
+        sa.Column('content_type_override', sa.String(length=100), nullable=True, comment='Custom Content-Type'),
+        sa.Column('created_at', sa.DateTime(), nullable=False, comment='Creation time'),
+        sa.Column('updated_at', sa.DateTime(), nullable=False, comment='Last update time'),
         sa.PrimaryKeyConstraint('id')
     )
 
-    # Create settings table
+    op.create_table(
-    op.create_table('settings',
+        'webhook_delivery_log',
         sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
-        sa.Column('key', sa.String(length=255), nullable=False, comment='Setting key (e.g., smtp_server)'),
+        sa.Column('webhook_id', sa.Integer(), nullable=False, comment='Associated webhook'),
-        sa.Column('value', sa.Text(), nullable=True, comment='Setting value (JSON for complex values)'),
+        sa.Column('alert_id', sa.Integer(), nullable=False, comment='Associated alert'),
-        sa.Column('updated_at', sa.DateTime(), nullable=False, comment='Last modification time'),
+        sa.Column('status', sa.String(length=20), nullable=True, comment='success, failed, retrying'),
-        sa.PrimaryKeyConstraint('id'),
+        sa.Column('response_code', sa.Integer(), nullable=True, comment='HTTP response code'),
-        sa.UniqueConstraint('key')
+        sa.Column('response_body', sa.Text(), nullable=True, comment='Response body'),
+        sa.Column('error_message', sa.Text(), nullable=True, comment='Error message if failed'),
+        sa.Column('attempt_number', sa.Integer(), nullable=True, comment='Which attempt'),
+        sa.Column('delivered_at', sa.DateTime(), nullable=False, comment='Delivery timestamp'),
+        sa.ForeignKeyConstraint(['webhook_id'], ['webhooks.id']),
+        sa.ForeignKeyConstraint(['alert_id'], ['alerts.id']),
+        sa.PrimaryKeyConstraint('id')
     )
-    op.create_index(op.f('ix_settings_key'), 'settings', ['key'], unique=True)
+    op.create_index('ix_webhook_delivery_log_webhook_id', 'webhook_delivery_log', ['webhook_id'])
+    op.create_index('ix_webhook_delivery_log_alert_id', 'webhook_delivery_log', ['alert_id'])
+    op.create_index('ix_webhook_delivery_log_status', 'webhook_delivery_log', ['status'])
+
+    print("\n✓ Initial schema created successfully")
 
 
-def downgrade() -> None:
+def downgrade():
-    """Drop all tables."""
+    """Drop all tables in reverse order."""
-    op.drop_index(op.f('ix_settings_key'), table_name='settings')
+    op.drop_table('webhook_delivery_log')
-    op.drop_table('settings')
+    op.drop_table('webhooks')
-    op.drop_table('alert_rules')
-    op.drop_index(op.f('ix_alerts_scan_id'), table_name='alerts')
     op.drop_table('alerts')
-    op.drop_index(op.f('ix_scan_tls_versions_scan_id'), table_name='scan_tls_versions')
+    op.drop_table('alert_rules')
-    op.drop_index(op.f('ix_scan_tls_versions_certificate_id'), table_name='scan_tls_versions')
+    op.drop_table('scan_site_associations')
+    op.drop_table('scan_progress')
     op.drop_table('scan_tls_versions')
-    op.drop_index(op.f('ix_scan_certificates_service_id'), table_name='scan_certificates')
-    op.drop_index(op.f('ix_scan_certificates_scan_id'), table_name='scan_certificates')
     op.drop_table('scan_certificates')
-    op.drop_index(op.f('ix_scan_services_scan_id'), table_name='scan_services')
-    op.drop_index(op.f('ix_scan_services_port_id'), table_name='scan_services')
     op.drop_table('scan_services')
-    op.drop_index(op.f('ix_scan_ports_scan_id'), table_name='scan_ports')
-    op.drop_index(op.f('ix_scan_ports_ip_id'), table_name='scan_ports')
     op.drop_table('scan_ports')
-    op.drop_index(op.f('ix_scan_ips_site_id'), table_name='scan_ips')
-    op.drop_index(op.f('ix_scan_ips_scan_id'), table_name='scan_ips')
     op.drop_table('scan_ips')
-    op.drop_index(op.f('ix_scan_sites_scan_id'), table_name='scan_sites')
     op.drop_table('scan_sites')
-    op.drop_index(op.f('ix_scans_timestamp'), table_name='scans')
     op.drop_table('scans')
     op.drop_table('schedules')
+    op.drop_table('scan_config_sites')
+    op.drop_table('scan_configs')
+    op.drop_table('site_ips')
+    op.drop_table('sites')
+    op.drop_table('settings')
+
+    print("\n✓ All tables dropped")

app/migrations/versions/002_add_scan_indexes.py

@@ -1,28 +0,0 @@
-"""Add indexes for scan queries
-
-Revision ID: 002
-Revises: 001
-Create Date: 2025-11-14 00:30:00.000000
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = '002'
-down_revision = '001'
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    """Add database indexes for better query performance."""
-    # Add index on scans.status for filtering
-    # Note: index on scans.timestamp already exists from migration 001
-    op.create_index('ix_scans_status', 'scans', ['status'], unique=False)
-
-
-def downgrade() -> None:
-    """Remove indexes."""
-    op.drop_index('ix_scans_status', table_name='scans')

app/migrations/versions/003_add_scan_timing_fields.py

@@ -1,39 +0,0 @@
-"""Add timing and error fields to scans table
-
-Revision ID: 003
-Revises: 002
-Create Date: 2025-11-14
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic
-revision = '003'
-down_revision = '002'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    """
-    Add fields for tracking scan execution timing and errors.
-
-    New fields:
-    - started_at: When scan execution actually started
-    - completed_at: When scan execution finished (success or failure)
-    - error_message: Error message if scan failed
-    """
-    with op.batch_alter_table('scans') as batch_op:
-        batch_op.add_column(sa.Column('started_at', sa.DateTime(), nullable=True, comment='Scan execution start time'))
-        batch_op.add_column(sa.Column('completed_at', sa.DateTime(), nullable=True, comment='Scan execution completion time'))
-        batch_op.add_column(sa.Column('error_message', sa.Text(), nullable=True, comment='Error message if scan failed'))
-
-
-def downgrade():
-    """Remove the timing and error fields."""
-    with op.batch_alter_table('scans') as batch_op:
-        batch_op.drop_column('error_message')
-        batch_op.drop_column('completed_at')
-        batch_op.drop_column('started_at')

app/migrations/versions/004_add_alert_rule_enhancements.py

@@ -1,120 +0,0 @@
-"""Add enhanced alert features for Phase 5
-
-Revision ID: 004
-Revises: 003
-Create Date: 2025-11-18
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic
-revision = '004'
-down_revision = '003'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    """
-    Add enhancements for Phase 5 Alert Rule Engine:
-    - Enhanced alert_rules fields
-    - Enhanced alerts fields
-    - New webhooks table
-    - New webhook_delivery_log table
-    """
-
-    # Enhance alert_rules table
-    with op.batch_alter_table('alert_rules') as batch_op:
-        batch_op.add_column(sa.Column('name', sa.String(255), nullable=True, comment='User-friendly rule name'))
-        batch_op.add_column(sa.Column('webhook_enabled', sa.Boolean(), nullable=False, server_default='0', comment='Whether to send webhooks for this rule'))
-        batch_op.add_column(sa.Column('severity', sa.String(20), nullable=True, comment='Alert severity level (critical, warning, info)'))
-        batch_op.add_column(sa.Column('filter_conditions', sa.Text(), nullable=True, comment='JSON filter conditions for the rule'))
-        batch_op.add_column(sa.Column('config_file', sa.String(255), nullable=True, comment='Optional: specific config file this rule applies to'))
-        batch_op.add_column(sa.Column('updated_at', sa.DateTime(), nullable=True, comment='Last update timestamp'))
-
-    # Enhance alerts table
-    with op.batch_alter_table('alerts') as batch_op:
-        batch_op.add_column(sa.Column('rule_id', sa.Integer(), nullable=True, comment='Associated alert rule'))
-        batch_op.add_column(sa.Column('webhook_sent', sa.Boolean(), nullable=False, server_default='0', comment='Whether webhook was sent'))
-        batch_op.add_column(sa.Column('webhook_sent_at', sa.DateTime(), nullable=True, comment='When webhook was sent'))
-        batch_op.add_column(sa.Column('acknowledged', sa.Boolean(), nullable=False, server_default='0', comment='Whether alert was acknowledged'))
-        batch_op.add_column(sa.Column('acknowledged_at', sa.DateTime(), nullable=True, comment='When alert was acknowledged'))
-        batch_op.add_column(sa.Column('acknowledged_by', sa.String(255), nullable=True, comment='User who acknowledged the alert'))
-        batch_op.create_foreign_key('fk_alerts_rule_id', 'alert_rules', ['rule_id'], ['id'])
-        batch_op.create_index('idx_alerts_rule_id', ['rule_id'])
-        batch_op.create_index('idx_alerts_acknowledged', ['acknowledged'])
-
-    # Create webhooks table
-    op.create_table('webhooks',
-        sa.Column('id', sa.Integer(), nullable=False),
-        sa.Column('name', sa.String(255), nullable=False, comment='Webhook name'),
-        sa.Column('url', sa.Text(), nullable=False, comment='Webhook URL'),
-        sa.Column('enabled', sa.Boolean(), nullable=False, server_default='1', comment='Whether webhook is enabled'),
-        sa.Column('auth_type', sa.String(20), nullable=True, comment='Authentication type: none, bearer, basic, custom'),
-        sa.Column('auth_token', sa.Text(), nullable=True, comment='Encrypted authentication token'),
-        sa.Column('custom_headers', sa.Text(), nullable=True, comment='JSON custom headers'),
-        sa.Column('alert_types', sa.Text(), nullable=True, comment='JSON array of alert types to trigger on'),
-        sa.Column('severity_filter', sa.Text(), nullable=True, comment='JSON array of severities to trigger on'),
-        sa.Column('timeout', sa.Integer(), nullable=True, server_default='10', comment='Request timeout in seconds'),
-        sa.Column('retry_count', sa.Integer(), nullable=True, server_default='3', comment='Number of retry attempts'),
-        sa.Column('created_at', sa.DateTime(), nullable=False),
-        sa.Column('updated_at', sa.DateTime(), nullable=False),
-        sa.PrimaryKeyConstraint('id')
-    )
-
-    # Create webhook_delivery_log table
-    op.create_table('webhook_delivery_log',
-        sa.Column('id', sa.Integer(), nullable=False),
-        sa.Column('webhook_id', sa.Integer(), nullable=False, comment='Associated webhook'),
-        sa.Column('alert_id', sa.Integer(), nullable=False, comment='Associated alert'),
-        sa.Column('status', sa.String(20), nullable=True, comment='Delivery status: success, failed, retrying'),
-        sa.Column('response_code', sa.Integer(), nullable=True, comment='HTTP response code'),
-        sa.Column('response_body', sa.Text(), nullable=True, comment='Response body from webhook'),
-        sa.Column('error_message', sa.Text(), nullable=True, comment='Error message if failed'),
-        sa.Column('attempt_number', sa.Integer(), nullable=True, comment='Which attempt this was'),
-        sa.Column('delivered_at', sa.DateTime(), nullable=False, comment='Delivery timestamp'),
-        sa.ForeignKeyConstraint(['webhook_id'], ['webhooks.id'], ),
-        sa.ForeignKeyConstraint(['alert_id'], ['alerts.id'], ),
-        sa.PrimaryKeyConstraint('id')
-    )
-
-    # Create indexes for webhook_delivery_log
-    op.create_index('idx_webhook_delivery_alert_id', 'webhook_delivery_log', ['alert_id'])
-    op.create_index('idx_webhook_delivery_webhook_id', 'webhook_delivery_log', ['webhook_id'])
-    op.create_index('idx_webhook_delivery_status', 'webhook_delivery_log', ['status'])
-
-
-def downgrade():
-    """Remove Phase 5 alert enhancements."""
-
-    # Drop webhook_delivery_log table and its indexes
-    op.drop_index('idx_webhook_delivery_status', table_name='webhook_delivery_log')
-    op.drop_index('idx_webhook_delivery_webhook_id', table_name='webhook_delivery_log')
-    op.drop_index('idx_webhook_delivery_alert_id', table_name='webhook_delivery_log')
-    op.drop_table('webhook_delivery_log')
-
-    # Drop webhooks table
-    op.drop_table('webhooks')
-
-    # Remove enhancements from alerts table
-    with op.batch_alter_table('alerts') as batch_op:
-        batch_op.drop_index('idx_alerts_acknowledged')
-        batch_op.drop_index('idx_alerts_rule_id')
-        batch_op.drop_constraint('fk_alerts_rule_id', type_='foreignkey')
-        batch_op.drop_column('acknowledged_by')
-        batch_op.drop_column('acknowledged_at')
-        batch_op.drop_column('acknowledged')
-        batch_op.drop_column('webhook_sent_at')
-        batch_op.drop_column('webhook_sent')
-        batch_op.drop_column('rule_id')
-
-    # Remove enhancements from alert_rules table
-    with op.batch_alter_table('alert_rules') as batch_op:
-        batch_op.drop_column('updated_at')
-        batch_op.drop_column('config_file')
-        batch_op.drop_column('filter_conditions')
-        batch_op.drop_column('severity')
-        batch_op.drop_column('webhook_enabled')
-        batch_op.drop_column('name')

app/migrations/versions/005_add_webhook_templates.py

@@ -1,83 +0,0 @@
-"""Add webhook template support
-
-Revision ID: 005
-Revises: 004
-Create Date: 2025-11-18
-
-"""
-from alembic import op
-import sqlalchemy as sa
-import json
-
-
-# revision identifiers, used by Alembic
-revision = '005'
-down_revision = '004'
-branch_labels = None
-depends_on = None
-
-
-# Default template that matches the current JSON payload structure
-DEFAULT_TEMPLATE = """{
-  "event": "alert.created",
-  "alert": {
-    "id": {{ alert.id }},
-    "type": "{{ alert.type }}",
-    "severity": "{{ alert.severity }}",
-    "message": "{{ alert.message }}",
-    {% if alert.ip_address %}"ip_address": "{{ alert.ip_address }}",{% endif %}
-    {% if alert.port %}"port": {{ alert.port }},{% endif %}
-    "acknowledged": {{ alert.acknowledged|lower }},
-    "created_at": "{{ alert.created_at.isoformat() }}"
-  },
-  "scan": {
-    "id": {{ scan.id }},
-    "title": "{{ scan.title }}",
-    "timestamp": "{{ scan.timestamp.isoformat() }}",
-    "status": "{{ scan.status }}"
-  },
-  "rule": {
-    "id": {{ rule.id }},
-    "name": "{{ rule.name }}",
-    "type": "{{ rule.type }}",
-    "threshold": {{ rule.threshold if rule.threshold else 'null' }}
-  }
-}"""
-
-
-def upgrade():
-    """
-    Add webhook template fields:
-    - template: Jinja2 template for payload
-    - template_format: Output format (json, text)
-    - content_type_override: Optional custom Content-Type
-    """
-
-    # Add new columns to webhooks table
-    with op.batch_alter_table('webhooks') as batch_op:
-        batch_op.add_column(sa.Column('template', sa.Text(), nullable=True, comment='Jinja2 template for webhook payload'))
-        batch_op.add_column(sa.Column('template_format', sa.String(20), nullable=True, server_default='json', comment='Template output format: json, text'))
-        batch_op.add_column(sa.Column('content_type_override', sa.String(100), nullable=True, comment='Optional custom Content-Type header'))
-
-    # Populate existing webhooks with default template
-    # This ensures backward compatibility by converting existing webhooks to use the
-    # same JSON structure they're currently sending
-    connection = op.get_bind()
-    connection.execute(
-        sa.text("""
-            UPDATE webhooks
-            SET template = :template,
-                template_format = 'json'
-            WHERE template IS NULL
-        """),
-        {"template": DEFAULT_TEMPLATE}
-    )
-
-
-def downgrade():
-    """Remove webhook template fields."""
-
-    with op.batch_alter_table('webhooks') as batch_op:
-        batch_op.drop_column('content_type_override')
-        batch_op.drop_column('template_format')
-        batch_op.drop_column('template')

app/migrations/versions/006_add_reusable_sites.py

@@ -1,161 +0,0 @@
-"""Add reusable site definitions
-
-Revision ID: 006
-Revises: 005
-Create Date: 2025-11-19
-
-This migration introduces reusable site definitions that can be shared across
-multiple scans. Sites are defined once with CIDR ranges and can be referenced
-in multiple scan configurations.
-"""
-from alembic import op
-import sqlalchemy as sa
-from sqlalchemy import text
-
-
-# revision identifiers, used by Alembic
-revision = '006'
-down_revision = '005'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    """
-    Create new site tables and migrate existing scan_sites data to the new structure.
-    """
-
-    # Create sites table (master site definitions)
-    op.create_table('sites',
-        sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
-        sa.Column('name', sa.String(length=255), nullable=False, comment='Unique site name'),
-        sa.Column('description', sa.Text(), nullable=True, comment='Site description'),
-        sa.Column('created_at', sa.DateTime(), nullable=False, comment='Site creation time'),
-        sa.Column('updated_at', sa.DateTime(), nullable=False, comment='Last modification time'),
-        sa.PrimaryKeyConstraint('id'),
-        sa.UniqueConstraint('name', name='uix_site_name')
-    )
-    op.create_index(op.f('ix_sites_name'), 'sites', ['name'], unique=True)
-
-    # Create site_cidrs table (CIDR ranges for each site)
-    op.create_table('site_cidrs',
-        sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
-        sa.Column('site_id', sa.Integer(), nullable=False, comment='FK to sites'),
-        sa.Column('cidr', sa.String(length=45), nullable=False, comment='CIDR notation (e.g., 10.0.0.0/24)'),
-        sa.Column('expected_ping', sa.Boolean(), nullable=True, comment='Expected ping response for this CIDR'),
-        sa.Column('expected_tcp_ports', sa.Text(), nullable=True, comment='JSON array of expected TCP ports'),
-        sa.Column('expected_udp_ports', sa.Text(), nullable=True, comment='JSON array of expected UDP ports'),
-        sa.Column('created_at', sa.DateTime(), nullable=False, comment='CIDR creation time'),
-        sa.ForeignKeyConstraint(['site_id'], ['sites.id'], ),
-        sa.PrimaryKeyConstraint('id'),
-        sa.UniqueConstraint('site_id', 'cidr', name='uix_site_cidr')
-    )
-    op.create_index(op.f('ix_site_cidrs_site_id'), 'site_cidrs', ['site_id'], unique=False)
-
-    # Create site_ips table (IP-level overrides within CIDRs)
-    op.create_table('site_ips',
-        sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
-        sa.Column('site_cidr_id', sa.Integer(), nullable=False, comment='FK to site_cidrs'),
-        sa.Column('ip_address', sa.String(length=45), nullable=False, comment='IPv4 or IPv6 address'),
-        sa.Column('expected_ping', sa.Boolean(), nullable=True, comment='Override ping expectation for this IP'),
-        sa.Column('expected_tcp_ports', sa.Text(), nullable=True, comment='JSON array of expected TCP ports (overrides CIDR)'),
-        sa.Column('expected_udp_ports', sa.Text(), nullable=True, comment='JSON array of expected UDP ports (overrides CIDR)'),
-        sa.Column('created_at', sa.DateTime(), nullable=False, comment='IP override creation time'),
-        sa.ForeignKeyConstraint(['site_cidr_id'], ['site_cidrs.id'], ),
-        sa.PrimaryKeyConstraint('id'),
-        sa.UniqueConstraint('site_cidr_id', 'ip_address', name='uix_site_cidr_ip')
-    )
-    op.create_index(op.f('ix_site_ips_site_cidr_id'), 'site_ips', ['site_cidr_id'], unique=False)
-
-    # Create scan_site_associations table (many-to-many between scans and sites)
-    op.create_table('scan_site_associations',
-        sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
-        sa.Column('scan_id', sa.Integer(), nullable=False, comment='FK to scans'),
-        sa.Column('site_id', sa.Integer(), nullable=False, comment='FK to sites'),
-        sa.Column('created_at', sa.DateTime(), nullable=False, comment='Association creation time'),
-        sa.ForeignKeyConstraint(['scan_id'], ['scans.id'], ),
-        sa.ForeignKeyConstraint(['site_id'], ['sites.id'], ),
-        sa.PrimaryKeyConstraint('id'),
-        sa.UniqueConstraint('scan_id', 'site_id', name='uix_scan_site')
-    )
-    op.create_index(op.f('ix_scan_site_associations_scan_id'), 'scan_site_associations', ['scan_id'], unique=False)
-    op.create_index(op.f('ix_scan_site_associations_site_id'), 'scan_site_associations', ['site_id'], unique=False)
-
-    # Migrate existing data
-    connection = op.get_bind()
-
-    # 1. Extract unique site names from existing scan_sites and create master Site records
-    # This groups all historical scan sites by name and creates one master site per unique name
-    connection.execute(text("""
-        INSERT INTO sites (name, description, created_at, updated_at)
-        SELECT DISTINCT
-            site_name,
-            'Migrated from scan_sites' as description,
-            datetime('now') as created_at,
-            datetime('now') as updated_at
-        FROM scan_sites
-        WHERE site_name NOT IN (SELECT name FROM sites)
-    """))
-
-    # 2. Create scan_site_associations linking scans to their sites
-    # This maintains the historical relationship between scans and the sites they used
-    connection.execute(text("""
-        INSERT INTO scan_site_associations (scan_id, site_id, created_at)
-        SELECT DISTINCT
-            ss.scan_id,
-            s.id as site_id,
-            datetime('now') as created_at
-        FROM scan_sites ss
-        INNER JOIN sites s ON s.name = ss.site_name
-        WHERE NOT EXISTS (
-            SELECT 1 FROM scan_site_associations ssa
-            WHERE ssa.scan_id = ss.scan_id AND ssa.site_id = s.id
-        )
-    """))
-
-    # 3. For each migrated site, create a CIDR entry from the IPs in scan_ips
-    # Since historical data has individual IPs, we'll create /32 CIDRs for each unique IP
-    # This preserves the exact IP addresses while fitting them into the new CIDR-based model
-    connection.execute(text("""
-        INSERT INTO site_cidrs (site_id, cidr, expected_ping, expected_tcp_ports, expected_udp_ports, created_at)
-        SELECT DISTINCT
-            s.id as site_id,
-            si.ip_address || '/32' as cidr,
-            si.ping_expected,
-            '[]' as expected_tcp_ports,
-            '[]' as expected_udp_ports,
-            datetime('now') as created_at
-        FROM scan_ips si
-        INNER JOIN scan_sites ss ON ss.id = si.site_id
-        INNER JOIN sites s ON s.name = ss.site_name
-        WHERE NOT EXISTS (
-            SELECT 1 FROM site_cidrs sc
-            WHERE sc.site_id = s.id AND sc.cidr = si.ip_address || '/32'
-        )
-        GROUP BY s.id, si.ip_address, si.ping_expected
-    """))
-
-    print("✓ Migration complete: Reusable sites created from historical scan data")
-    print(f"  - Created {connection.execute(text('SELECT COUNT(*) FROM sites')).scalar()} master site(s)")
-    print(f"  - Created {connection.execute(text('SELECT COUNT(*) FROM site_cidrs')).scalar()} CIDR range(s)")
-    print(f"  - Created {connection.execute(text('SELECT COUNT(*) FROM scan_site_associations')).scalar()} scan-site association(s)")
-
-
-def downgrade():
-    """Remove reusable site tables."""
-
-    # Drop tables in reverse order of creation (respecting foreign keys)
-    op.drop_index(op.f('ix_scan_site_associations_site_id'), table_name='scan_site_associations')
-    op.drop_index(op.f('ix_scan_site_associations_scan_id'), table_name='scan_site_associations')
-    op.drop_table('scan_site_associations')
-
-    op.drop_index(op.f('ix_site_ips_site_cidr_id'), table_name='site_ips')
-    op.drop_table('site_ips')
-
-    op.drop_index(op.f('ix_site_cidrs_site_id'), table_name='site_cidrs')
-    op.drop_table('site_cidrs')
-
-    op.drop_index(op.f('ix_sites_name'), table_name='sites')
-    op.drop_table('sites')
-
-    print("✓ Downgrade complete: Reusable site tables removed")

app/migrations/versions/007_configs_to_database.py

@@ -1,102 +0,0 @@
-"""Add database-stored scan configurations
-
-Revision ID: 007
-Revises: 006
-Create Date: 2025-11-19
-
-This migration introduces database-stored scan configurations to replace YAML
-config files. Configs reference sites from the sites table, enabling visual
-config builder and better data management.
-"""
-from alembic import op
-import sqlalchemy as sa
-from sqlalchemy import text
-
-
-# revision identifiers, used by Alembic
-revision = '007'
-down_revision = '006'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    """
-    Create scan_configs and scan_config_sites tables.
-    Add config_id foreign keys to scans and schedules tables.
-    """
-
-    # Create scan_configs table
-    op.create_table('scan_configs',
-        sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
-        sa.Column('title', sa.String(length=255), nullable=False, comment='Configuration title'),
-        sa.Column('description', sa.Text(), nullable=True, comment='Configuration description'),
-        sa.Column('created_at', sa.DateTime(), nullable=False, comment='Config creation time'),
-        sa.Column('updated_at', sa.DateTime(), nullable=False, comment='Last modification time'),
-        sa.PrimaryKeyConstraint('id')
-    )
-
-    # Create scan_config_sites table (many-to-many between configs and sites)
-    op.create_table('scan_config_sites',
-        sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
-        sa.Column('config_id', sa.Integer(), nullable=False, comment='FK to scan_configs'),
-        sa.Column('site_id', sa.Integer(), nullable=False, comment='FK to sites'),
-        sa.Column('created_at', sa.DateTime(), nullable=False, comment='Association creation time'),
-        sa.ForeignKeyConstraint(['config_id'], ['scan_configs.id'], ),
-        sa.ForeignKeyConstraint(['site_id'], ['sites.id'], ),
-        sa.PrimaryKeyConstraint('id'),
-        sa.UniqueConstraint('config_id', 'site_id', name='uix_config_site')
-    )
-    op.create_index(op.f('ix_scan_config_sites_config_id'), 'scan_config_sites', ['config_id'], unique=False)
-    op.create_index(op.f('ix_scan_config_sites_site_id'), 'scan_config_sites', ['site_id'], unique=False)
-
-    # Add config_id to scans table
-    with op.batch_alter_table('scans', schema=None) as batch_op:
-        batch_op.add_column(sa.Column('config_id', sa.Integer(), nullable=True, comment='FK to scan_configs table'))
-        batch_op.create_index('ix_scans_config_id', ['config_id'], unique=False)
-        batch_op.create_foreign_key('fk_scans_config_id', 'scan_configs', ['config_id'], ['id'])
-        # Mark config_file as deprecated in comment (already has nullable=True)
-
-    # Add config_id to schedules table and make config_file nullable
-    with op.batch_alter_table('schedules', schema=None) as batch_op:
-        batch_op.add_column(sa.Column('config_id', sa.Integer(), nullable=True, comment='FK to scan_configs table'))
-        batch_op.create_index('ix_schedules_config_id', ['config_id'], unique=False)
-        batch_op.create_foreign_key('fk_schedules_config_id', 'scan_configs', ['config_id'], ['id'])
-        # Make config_file nullable (it was required before)
-        batch_op.alter_column('config_file', existing_type=sa.Text(), nullable=True)
-
-    connection = op.get_bind()
-
-    print("✓ Migration complete: Scan configs tables created")
-    print("  - Created scan_configs table for database-stored configurations")
-    print("  - Created scan_config_sites association table")
-    print("  - Added config_id to scans table")
-    print("  - Added config_id to schedules table")
-    print("  - Existing YAML configs remain in config_file column for backward compatibility")
-
-
-def downgrade():
-    """Remove scan config tables and columns."""
-
-    # Remove foreign keys and columns from schedules
-    with op.batch_alter_table('schedules', schema=None) as batch_op:
-        batch_op.drop_constraint('fk_schedules_config_id', type_='foreignkey')
-        batch_op.drop_index('ix_schedules_config_id')
-        batch_op.drop_column('config_id')
-        # Restore config_file as required
-        batch_op.alter_column('config_file', existing_type=sa.Text(), nullable=False)
-
-    # Remove foreign keys and columns from scans
-    with op.batch_alter_table('scans', schema=None) as batch_op:
-        batch_op.drop_constraint('fk_scans_config_id', type_='foreignkey')
-        batch_op.drop_index('ix_scans_config_id')
-        batch_op.drop_column('config_id')
-
-    # Drop tables in reverse order
-    op.drop_index(op.f('ix_scan_config_sites_site_id'), table_name='scan_config_sites')
-    op.drop_index(op.f('ix_scan_config_sites_config_id'), table_name='scan_config_sites')
-    op.drop_table('scan_config_sites')
-
-    op.drop_table('scan_configs')
-
-    print("✓ Downgrade complete: Scan config tables and columns removed")

app/migrations/versions/008_expand_cidrs_to_ips.py

@@ -1,270 +0,0 @@
-"""Expand CIDRs to individual IPs with per-IP settings
-
-Revision ID: 008
-Revises: 007
-Create Date: 2025-11-19
-
-This migration changes the site architecture to automatically expand CIDRs into
-individual IPs in the database. Each IP has its own port and ping settings.
-
-Changes:
-- Add site_id to site_ips (direct link to sites, support standalone IPs)
-- Make site_cidr_id nullable (IPs can exist without a CIDR parent)
-- Remove settings from site_cidrs (settings now only at IP level)
-- Add unique constraint: no duplicate IPs within a site
-- Expand existing CIDRs to individual IPs
-"""
-from alembic import op
-import sqlalchemy as sa
-from sqlalchemy import text
-import ipaddress
-
-
-# revision identifiers, used by Alembic
-revision = '008'
-down_revision = '007'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    """
-    Modify schema to support per-IP settings and auto-expand CIDRs.
-    """
-
-    connection = op.get_bind()
-
-    # Check if site_id column already exists
-    inspector = sa.inspect(connection)
-    site_ips_columns = [col['name'] for col in inspector.get_columns('site_ips')]
-    site_cidrs_columns = [col['name'] for col in inspector.get_columns('site_cidrs')]
-
-    # Step 1: Add site_id column to site_ips (will be populated from site_cidr_id)
-    if 'site_id' not in site_ips_columns:
-        print("Adding site_id column to site_ips...")
-        op.add_column('site_ips', sa.Column('site_id', sa.Integer(), nullable=True, comment='FK to sites (direct link)'))
-    else:
-        print("site_id column already exists in site_ips, skipping...")
-
-    # Step 2: Populate site_id from site_cidr_id (before we make it nullable)
-    print("Populating site_id from existing site_cidr relationships...")
-    connection.execute(text("""
-        UPDATE site_ips
-        SET site_id = (
-            SELECT site_id
-            FROM site_cidrs
-            WHERE site_cidrs.id = site_ips.site_cidr_id
-        )
-        WHERE site_cidr_id IS NOT NULL
-    """))
-
-    # Step 3: Make site_id NOT NULL and add foreign key
-    # Check if foreign key exists before creating
-    try:
-        op.alter_column('site_ips', 'site_id', nullable=False)
-        print("Made site_id NOT NULL")
-    except Exception as e:
-        print(f"site_id already NOT NULL or error: {e}")
-
-    # Check if foreign key exists
-    try:
-        op.create_foreign_key('fk_site_ips_site_id', 'site_ips', 'sites', ['site_id'], ['id'])
-        print("Created foreign key fk_site_ips_site_id")
-    except Exception as e:
-        print(f"Foreign key already exists or error: {e}")
-
-    # Check if index exists
-    try:
-        op.create_index(op.f('ix_site_ips_site_id'), 'site_ips', ['site_id'], unique=False)
-        print("Created index ix_site_ips_site_id")
-    except Exception as e:
-        print(f"Index already exists or error: {e}")
-
-    # Step 4: Make site_cidr_id nullable (for standalone IPs)
-    try:
-        op.alter_column('site_ips', 'site_cidr_id', nullable=True)
-        print("Made site_cidr_id nullable")
-    except Exception as e:
-        print(f"site_cidr_id already nullable or error: {e}")
-
-    # Step 5: Drop old unique constraint and create new one (site_id, ip_address)
-    # This prevents duplicate IPs within a site (across all CIDRs and standalone)
-    try:
-        op.drop_constraint('uix_site_cidr_ip', 'site_ips', type_='unique')
-        print("Dropped old constraint uix_site_cidr_ip")
-    except Exception as e:
-        print(f"Constraint already dropped or doesn't exist: {e}")
-
-    try:
-        op.create_unique_constraint('uix_site_ip_address', 'site_ips', ['site_id', 'ip_address'])
-        print("Created new constraint uix_site_ip_address")
-    except Exception as e:
-        print(f"Constraint already exists or error: {e}")
-
-    # Step 6: Expand existing CIDRs to individual IPs
-    print("Expanding existing CIDRs to individual IPs...")
-
-    # Get all existing CIDRs
-    cidrs = connection.execute(text("""
-        SELECT id, site_id, cidr, expected_ping, expected_tcp_ports, expected_udp_ports
-        FROM site_cidrs
-    """)).fetchall()
-
-    expanded_count = 0
-    skipped_count = 0
-
-    for cidr_row in cidrs:
-        cidr_id, site_id, cidr_str, expected_ping, expected_tcp_ports, expected_udp_ports = cidr_row
-
-        try:
-            # Parse CIDR
-            network = ipaddress.ip_network(cidr_str, strict=False)
-
-            # Check size - skip if too large (> /24 for IPv4, > /64 for IPv6)
-            if isinstance(network, ipaddress.IPv4Network) and network.prefixlen < 24:
-                print(f"  ⚠ Skipping large CIDR {cidr_str} (>{network.num_addresses} IPs)")
-                skipped_count += 1
-                continue
-            elif isinstance(network, ipaddress.IPv6Network) and network.prefixlen < 64:
-                print(f"  ⚠ Skipping large CIDR {cidr_str} (>{network.num_addresses} IPs)")
-                skipped_count += 1
-                continue
-
-            # Expand to individual IPs
-            for ip in network.hosts() if network.num_addresses > 2 else [network.network_address]:
-                ip_str = str(ip)
-
-                # Check if this IP already exists (from previous IP overrides)
-                existing = connection.execute(text("""
-                    SELECT id FROM site_ips
-                    WHERE site_cidr_id = :cidr_id AND ip_address = :ip_address
-                """), {'cidr_id': cidr_id, 'ip_address': ip_str}).fetchone()
-
-                if not existing:
-                    # Insert new IP with settings from CIDR
-                    connection.execute(text("""
-                        INSERT INTO site_ips (
-                            site_id, site_cidr_id, ip_address,
-                            expected_ping, expected_tcp_ports, expected_udp_ports,
-                            created_at
-                        )
-                        VALUES (
-                            :site_id, :cidr_id, :ip_address,
-                            :expected_ping, :expected_tcp_ports, :expected_udp_ports,
-                            datetime('now')
-                        )
-                    """), {
-                        'site_id': site_id,
-                        'cidr_id': cidr_id,
-                        'ip_address': ip_str,
-                        'expected_ping': expected_ping,
-                        'expected_tcp_ports': expected_tcp_ports,
-                        'expected_udp_ports': expected_udp_ports
-                    })
-                    expanded_count += 1
-
-        except Exception as e:
-            print(f"  ✗ Error expanding CIDR {cidr_str}: {e}")
-            skipped_count += 1
-            continue
-
-    print(f"  ✓ Expanded {expanded_count} IPs from CIDRs")
-    if skipped_count > 0:
-        print(f"  ⚠ Skipped {skipped_count} CIDRs (too large or errors)")
-
-    # Step 7: Remove settings columns from site_cidrs (now only at IP level)
-    print("Removing settings columns from site_cidrs...")
-    # Re-inspect to get current columns
-    site_cidrs_columns = [col['name'] for col in inspector.get_columns('site_cidrs')]
-
-    if 'expected_ping' in site_cidrs_columns:
-        try:
-            op.drop_column('site_cidrs', 'expected_ping')
-            print("Dropped expected_ping from site_cidrs")
-        except Exception as e:
-            print(f"Error dropping expected_ping: {e}")
-    else:
-        print("expected_ping already dropped from site_cidrs")
-
-    if 'expected_tcp_ports' in site_cidrs_columns:
-        try:
-            op.drop_column('site_cidrs', 'expected_tcp_ports')
-            print("Dropped expected_tcp_ports from site_cidrs")
-        except Exception as e:
-            print(f"Error dropping expected_tcp_ports: {e}")
-    else:
-        print("expected_tcp_ports already dropped from site_cidrs")
-
-    if 'expected_udp_ports' in site_cidrs_columns:
-        try:
-            op.drop_column('site_cidrs', 'expected_udp_ports')
-            print("Dropped expected_udp_ports from site_cidrs")
-        except Exception as e:
-            print(f"Error dropping expected_udp_ports: {e}")
-    else:
-        print("expected_udp_ports already dropped from site_cidrs")
-
-    # Print summary
-    total_sites = connection.execute(text('SELECT COUNT(*) FROM sites')).scalar()
-    total_cidrs = connection.execute(text('SELECT COUNT(*) FROM site_cidrs')).scalar()
-    total_ips = connection.execute(text('SELECT COUNT(*) FROM site_ips')).scalar()
-
-    print("\n✓ Migration 008 complete: CIDRs expanded to individual IPs")
-    print(f"  - Total sites: {total_sites}")
-    print(f"  - Total CIDRs: {total_cidrs}")
-    print(f"  - Total IPs: {total_ips}")
-
-
-def downgrade():
-    """
-    Revert schema changes (restore CIDR-level settings).
-    Note: This will lose per-IP granularity!
-    """
-
-    connection = op.get_bind()
-
-    print("Rolling back to CIDR-level settings...")
-
-    # Step 1: Add settings columns back to site_cidrs
-    op.add_column('site_cidrs', sa.Column('expected_ping', sa.Boolean(), nullable=True))
-    op.add_column('site_cidrs', sa.Column('expected_tcp_ports', sa.Text(), nullable=True))
-    op.add_column('site_cidrs', sa.Column('expected_udp_ports', sa.Text(), nullable=True))
-
-    # Step 2: Populate CIDR settings from first IP in each CIDR (approximation)
-    connection.execute(text("""
-        UPDATE site_cidrs
-        SET
-            expected_ping = (
-                SELECT expected_ping FROM site_ips
-                WHERE site_ips.site_cidr_id = site_cidrs.id
-                LIMIT 1
-            ),
-            expected_tcp_ports = (
-                SELECT expected_tcp_ports FROM site_ips
-                WHERE site_ips.site_cidr_id = site_cidrs.id
-                LIMIT 1
-            ),
-            expected_udp_ports = (
-                SELECT expected_udp_ports FROM site_ips
-                WHERE site_ips.site_cidr_id = site_cidrs.id
-                LIMIT 1
-            )
-    """))
-
-    # Step 3: Delete auto-expanded IPs (keep only original overrides)
-    # In practice, this is difficult to determine, so we'll keep all IPs
-    # and just remove the schema changes
-
-    # Step 4: Drop new unique constraint and restore old one
-    op.drop_constraint('uix_site_ip_address', 'site_ips', type_='unique')
-    op.create_unique_constraint('uix_site_cidr_ip', 'site_ips', ['site_cidr_id', 'ip_address'])
-
-    # Step 5: Make site_cidr_id NOT NULL again
-    op.alter_column('site_ips', 'site_cidr_id', nullable=False)
-
-    # Step 6: Drop site_id column and related constraints
-    op.drop_index(op.f('ix_site_ips_site_id'), table_name='site_ips')
-    op.drop_constraint('fk_site_ips_site_id', 'site_ips', type_='foreignkey')
-    op.drop_column('site_ips', 'site_id')
-
-    print("✓ Downgrade complete: Reverted to CIDR-level settings")

app/migrations/versions/009_remove_cidrs.py

@@ -1,210 +0,0 @@
-"""Remove CIDR table - make sites IP-only
-
-Revision ID: 009
-Revises: 008
-Create Date: 2025-11-19
-
-This migration removes the SiteCIDR table entirely, making sites purely
-IP-based. CIDRs are now only used as a convenience for bulk IP addition,
-not stored as permanent entities.
-
-Changes:
-- Set all site_ips.site_cidr_id to NULL (preserve all IPs)
-- Drop foreign key from site_ips to site_cidrs
-- Drop site_cidrs table
-- Remove site_cidr_id column from site_ips
-
-All existing IPs are preserved. They become "standalone" IPs without
-a CIDR parent.
-"""
-from alembic import op
-import sqlalchemy as sa
-from sqlalchemy import text
-
-
-# revision identifiers, used by Alembic
-revision = '009'
-down_revision = '008'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    """
-    Remove CIDR table and make all IPs standalone.
-    """
-
-    connection = op.get_bind()
-    inspector = sa.inspect(connection)
-
-    print("\n=== Migration 009: Remove CIDR Table ===\n")
-
-    # Get counts before migration
-    try:
-        total_cidrs = connection.execute(text('SELECT COUNT(*) FROM site_cidrs')).scalar()
-        total_ips = connection.execute(text('SELECT COUNT(*) FROM site_ips')).scalar()
-        ips_with_cidr = connection.execute(text(
-            'SELECT COUNT(*) FROM site_ips WHERE site_cidr_id IS NOT NULL'
-        )).scalar()
-
-        print(f"Before migration:")
-        print(f"  - Total CIDRs: {total_cidrs}")
-        print(f"  - Total IPs: {total_ips}")
-        print(f"  - IPs linked to CIDRs: {ips_with_cidr}")
-        print(f"  - Standalone IPs: {total_ips - ips_with_cidr}\n")
-    except Exception as e:
-        print(f"Could not get pre-migration stats: {e}\n")
-
-    # Step 1: Set all site_cidr_id to NULL (preserve all IPs as standalone)
-    print("Step 1: Converting all IPs to standalone (nulling CIDR associations)...")
-    try:
-        result = connection.execute(text("""
-            UPDATE site_ips
-            SET site_cidr_id = NULL
-            WHERE site_cidr_id IS NOT NULL
-        """))
-        print(f"  ✓ Converted {result.rowcount} IPs to standalone\n")
-    except Exception as e:
-        print(f"  ⚠ Error or already done: {e}\n")
-
-    # Step 2: Drop foreign key constraint from site_ips to site_cidrs
-    print("Step 2: Dropping foreign key constraint from site_ips to site_cidrs...")
-    foreign_keys = inspector.get_foreign_keys('site_ips')
-    fk_to_drop = None
-
-    for fk in foreign_keys:
-        if fk['referred_table'] == 'site_cidrs':
-            fk_to_drop = fk['name']
-            break
-
-    if fk_to_drop:
-        try:
-            op.drop_constraint(fk_to_drop, 'site_ips', type_='foreignkey')
-            print(f"  ✓ Dropped foreign key constraint: {fk_to_drop}\n")
-        except Exception as e:
-            print(f"  ⚠ Could not drop foreign key: {e}\n")
-    else:
-        print("  ⚠ Foreign key constraint not found or already dropped\n")
-
-    # Step 3: Drop index on site_cidr_id (if exists)
-    print("Step 3: Dropping index on site_cidr_id...")
-    indexes = inspector.get_indexes('site_ips')
-    index_to_drop = None
-
-    for idx in indexes:
-        if 'site_cidr_id' in idx['column_names']:
-            index_to_drop = idx['name']
-            break
-
-    if index_to_drop:
-        try:
-            op.drop_index(index_to_drop, table_name='site_ips')
-            print(f"  ✓ Dropped index: {index_to_drop}\n")
-        except Exception as e:
-            print(f"  ⚠ Could not drop index: {e}\n")
-    else:
-        print("  ⚠ Index not found or already dropped\n")
-
-    # Step 4: Drop site_cidrs table
-    print("Step 4: Dropping site_cidrs table...")
-    tables = inspector.get_table_names()
-
-    if 'site_cidrs' in tables:
-        try:
-            op.drop_table('site_cidrs')
-            print("  ✓ Dropped site_cidrs table\n")
-        except Exception as e:
-            print(f"  ⚠ Could not drop table: {e}\n")
-    else:
-        print("  ⚠ Table site_cidrs not found or already dropped\n")
-
-    # Step 5: Drop site_cidr_id column from site_ips
-    print("Step 5: Dropping site_cidr_id column from site_ips...")
-    site_ips_columns = [col['name'] for col in inspector.get_columns('site_ips')]
-
-    if 'site_cidr_id' in site_ips_columns:
-        try:
-            op.drop_column('site_ips', 'site_cidr_id')
-            print("  ✓ Dropped site_cidr_id column from site_ips\n")
-        except Exception as e:
-            print(f"  ⚠ Could not drop column: {e}\n")
-    else:
-        print("  ⚠ Column site_cidr_id not found or already dropped\n")
-
-    # Get counts after migration
-    try:
-        final_ips = connection.execute(text('SELECT COUNT(*) FROM site_ips')).scalar()
-        total_sites = connection.execute(text('SELECT COUNT(*) FROM sites')).scalar()
-
-        print("After migration:")
-        print(f"  - Total sites: {total_sites}")
-        print(f"  - Total IPs (all standalone): {final_ips}")
-        print(f"  - CIDRs: N/A (table removed)")
-    except Exception as e:
-        print(f"Could not get post-migration stats: {e}")
-
-    print("\n✓ Migration 009 complete: Sites are now IP-only")
-    print("  All IPs preserved as standalone. CIDRs can still be used")
-    print("  via the API/UI for bulk IP creation, but are not stored.\n")
-
-
-def downgrade():
-    """
-    Recreate site_cidrs table (CANNOT restore original CIDR associations).
-
-    WARNING: This downgrade creates an empty site_cidrs table structure but
-    cannot restore the original CIDR-to-IP associations since that data was
-    deleted. All IPs will remain standalone.
-    """
-
-    connection = op.get_bind()
-
-    print("\n=== Downgrade 009: Recreate CIDR Table Structure ===\n")
-    print("⚠ WARNING: Cannot restore original CIDR associations!")
-    print("  The site_cidrs table structure will be recreated but will be empty.")
-    print("  All IPs will remain standalone. This is a PARTIAL downgrade.\n")
-
-    # Step 1: Recreate site_cidrs table (empty)
-    print("Step 1: Recreating site_cidrs table structure...")
-    try:
-        op.create_table(
-            'site_cidrs',
-            sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
-            sa.Column('site_id', sa.Integer(), nullable=False),
-            sa.Column('cidr', sa.String(length=45), nullable=False, comment='CIDR notation (e.g., 10.0.0.0/24)'),
-            sa.Column('created_at', sa.DateTime(), nullable=False),
-            sa.PrimaryKeyConstraint('id'),
-            sa.ForeignKeyConstraint(['site_id'], ['sites.id'], ),
-            sa.UniqueConstraint('site_id', 'cidr', name='uix_site_cidr')
-        )
-        print("  ✓ Recreated site_cidrs table (empty)\n")
-    except Exception as e:
-        print(f"  ⚠ Could not create table: {e}\n")
-
-    # Step 2: Add site_cidr_id column back to site_ips (nullable)
-    print("Step 2: Adding site_cidr_id column back to site_ips...")
-    try:
-        op.add_column('site_ips', sa.Column('site_cidr_id', sa.Integer(), nullable=True, comment='FK to site_cidrs (optional, for grouping)'))
-        print("  ✓ Added site_cidr_id column (nullable)\n")
-    except Exception as e:
-        print(f"  ⚠ Could not add column: {e}\n")
-
-    # Step 3: Add foreign key constraint
-    print("Step 3: Adding foreign key constraint...")
-    try:
-        op.create_foreign_key('fk_site_ips_site_cidr_id', 'site_ips', 'site_cidrs', ['site_cidr_id'], ['id'])
-        print("  ✓ Created foreign key constraint\n")
-    except Exception as e:
-        print(f"  ⚠ Could not create foreign key: {e}\n")
-
-    # Step 4: Add index on site_cidr_id
-    print("Step 4: Adding index on site_cidr_id...")
-    try:
-        op.create_index('ix_site_ips_site_cidr_id', 'site_ips', ['site_cidr_id'], unique=False)
-        print("  ✓ Created index on site_cidr_id\n")
-    except Exception as e:
-        print(f"  ⚠ Could not create index: {e}\n")
-
-    print("✓ Downgrade complete: CIDR table structure restored (but empty)")
-    print("  All IPs remain standalone. You would need to manually recreate")
-    print("  CIDR records and associate IPs with them.\n")

app/migrations/versions/010_alert_rules_config_id.py

@@ -1,53 +0,0 @@
-"""Add config_id to alert_rules table
-
-Revision ID: 010
-Revises: 009
-Create Date: 2025-11-19
-
-This migration adds config_id foreign key to alert_rules table to replace
-the config_file column, completing the migration from file-based to
-database-based configurations.
-"""
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic
-revision = '010'
-down_revision = '009'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    """
-    Add config_id to alert_rules table and remove config_file.
-    """
-
-    with op.batch_alter_table('alert_rules', schema=None) as batch_op:
-        # Add config_id column with foreign key
-        batch_op.add_column(sa.Column('config_id', sa.Integer(), nullable=True, comment='FK to scan_configs table'))
-        batch_op.create_index('ix_alert_rules_config_id', ['config_id'], unique=False)
-        batch_op.create_foreign_key('fk_alert_rules_config_id', 'scan_configs', ['config_id'], ['id'])
-
-        # Remove the old config_file column
-        batch_op.drop_column('config_file')
-
-    print("✓ Migration complete: AlertRule now uses config_id")
-    print("  - Added config_id foreign key to alert_rules table")
-    print("  - Removed deprecated config_file column")
-
-
-def downgrade():
-    """Remove config_id and restore config_file on alert_rules."""
-
-    with op.batch_alter_table('alert_rules', schema=None) as batch_op:
-        # Remove foreign key and config_id column
-        batch_op.drop_constraint('fk_alert_rules_config_id', type_='foreignkey')
-        batch_op.drop_index('ix_alert_rules_config_id')
-        batch_op.drop_column('config_id')
-
-        # Restore config_file column
-        batch_op.add_column(sa.Column('config_file', sa.String(255), nullable=True, comment='Optional: specific config file this rule applies to'))
-
-    print("✓ Downgrade complete: AlertRule config_id removed, config_file restored")

app/migrations/versions/011_drop_config_file.py

@@ -1,86 +0,0 @@
-"""Drop deprecated config_file columns
-
-Revision ID: 011
-Revises: 010
-Create Date: 2025-11-19
-
-This migration removes the deprecated config_file columns from scans and schedules
-tables. All functionality now uses config_id to reference database-stored configs.
-"""
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic
-revision = '011'
-down_revision = '010'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    """
-    Drop config_file columns from scans and schedules tables.
-
-    Prerequisites:
-    - All scans must have config_id set
-    - All schedules must have config_id set
-    - Code must be updated to no longer reference config_file
-    """
-
-    connection = op.get_bind()
-
-    # Check for any records missing config_id
-    result = connection.execute(sa.text(
-        "SELECT COUNT(*) FROM scans WHERE config_id IS NULL"
-    ))
-    scans_without_config = result.scalar()
-
-    result = connection.execute(sa.text(
-        "SELECT COUNT(*) FROM schedules WHERE config_id IS NULL"
-    ))
-    schedules_without_config = result.scalar()
-
-    if scans_without_config > 0:
-        print(f"WARNING: {scans_without_config} scans have NULL config_id")
-        print("  These scans will lose their config reference after migration")
-
-    if schedules_without_config > 0:
-        raise Exception(
-            f"Cannot proceed: {schedules_without_config} schedules have NULL config_id. "
-            "Please set config_id for all schedules before running this migration."
-        )
-
-    # Drop config_file from scans table
-    with op.batch_alter_table('scans', schema=None) as batch_op:
-        batch_op.drop_column('config_file')
-
-    # Drop config_file from schedules table
-    with op.batch_alter_table('schedules', schema=None) as batch_op:
-        batch_op.drop_column('config_file')
-
-    print("✓ Migration complete: Dropped config_file columns")
-    print("  - Removed config_file from scans table")
-    print("  - Removed config_file from schedules table")
-    print("  - All references should now use config_id")
-
-
-def downgrade():
-    """Re-add config_file columns (data will be lost)."""
-
-    # Add config_file back to scans
-    with op.batch_alter_table('scans', schema=None) as batch_op:
-        batch_op.add_column(
-            sa.Column('config_file', sa.Text(), nullable=True,
-                     comment='Path to YAML config used (deprecated)')
-        )
-
-    # Add config_file back to schedules
-    with op.batch_alter_table('schedules', schema=None) as batch_op:
-        batch_op.add_column(
-            sa.Column('config_file', sa.Text(), nullable=True,
-                     comment='Path to YAML config (deprecated)')
-        )
-
-    print("✓ Downgrade complete: Re-added config_file columns")
-    print("  WARNING: config_file values are lost and will be NULL")

app/migrations/versions/012_add_scan_progress.py

@@ -1,58 +0,0 @@
-"""Add scan progress tracking
-
-Revision ID: 012
-Revises: 011
-Create Date: 2024-01-01 00:00:00.000000
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = '012'
-down_revision = '011'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    # Add progress tracking columns to scans table
-    op.add_column('scans', sa.Column('current_phase', sa.String(50), nullable=True,
-                  comment='Current scan phase: ping, tcp_scan, udp_scan, service_detection, http_analysis'))
-    op.add_column('scans', sa.Column('total_ips', sa.Integer(), nullable=True,
-                  comment='Total number of IPs to scan'))
-    op.add_column('scans', sa.Column('completed_ips', sa.Integer(), nullable=True, default=0,
-                  comment='Number of IPs completed in current phase'))
-
-    # Create scan_progress table for per-IP progress tracking
-    op.create_table(
-        'scan_progress',
-        sa.Column('id', sa.Integer(), primary_key=True, autoincrement=True),
-        sa.Column('scan_id', sa.Integer(), sa.ForeignKey('scans.id'), nullable=False, index=True),
-        sa.Column('ip_address', sa.String(45), nullable=False, comment='IP address being scanned'),
-        sa.Column('site_name', sa.String(255), nullable=True, comment='Site name this IP belongs to'),
-        sa.Column('phase', sa.String(50), nullable=False,
-                  comment='Phase: ping, tcp_scan, udp_scan, service_detection, http_analysis'),
-        sa.Column('status', sa.String(20), nullable=False, default='pending',
-                  comment='pending, in_progress, completed, failed'),
-        sa.Column('ping_result', sa.Boolean(), nullable=True, comment='Ping response result'),
-        sa.Column('tcp_ports', sa.Text(), nullable=True, comment='JSON array of discovered TCP ports'),
-        sa.Column('udp_ports', sa.Text(), nullable=True, comment='JSON array of discovered UDP ports'),
-        sa.Column('services', sa.Text(), nullable=True, comment='JSON array of detected services'),
-        sa.Column('created_at', sa.DateTime(), nullable=False, server_default=sa.func.now(),
-                  comment='Entry creation time'),
-        sa.Column('updated_at', sa.DateTime(), nullable=False, server_default=sa.func.now(),
-                  onupdate=sa.func.now(), comment='Last update time'),
-        sa.UniqueConstraint('scan_id', 'ip_address', name='uix_scan_progress_ip')
-    )
-
-
-def downgrade():
-    # Drop scan_progress table
-    op.drop_table('scan_progress')
-
-    # Remove progress tracking columns from scans table
-    op.drop_column('scans', 'completed_ips')
-    op.drop_column('scans', 'total_ips')
-    op.drop_column('scans', 'current_phase')

app/src/scanner.py

@@ -676,29 +676,57 @@ class SneakyScanner:
 
         return services
 
-    def _is_likely_web_service(self, service: Dict) -> bool:
+    def _is_likely_web_service(self, service: Dict, ip: str = None) -> bool:
         """
-        Check if a service is likely HTTP/HTTPS based on nmap detection or common web ports
+        Check if a service is a web server by actually making an HTTP request
 
         Args:
             service: Service dictionary from nmap results
+            ip: IP address to test (required for HTTP probe)
 
         Returns:
-            True if service appears to be web-related
+            True if service responds to HTTP/HTTPS requests
         """
-        # Check service name
+        import requests
+        import urllib3
+        urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+        # Quick check for known web service names first
         web_services = ['http', 'https', 'ssl', 'http-proxy', 'https-alt',
                        'http-alt', 'ssl/http', 'ssl/https']
         service_name = service.get('service', '').lower()
 
+        # If no IP provided, can't do HTTP probe
+        port = service.get('port')
+        if not ip or not port:
+            # check just the service if no IP - honestly shouldn't get here, but just incase...
             if service_name in web_services:
                 return True
+            return False
 
-        # Check common non-standard web ports
+        # Actually try to connect - this is the definitive test
-        web_ports = [80, 443, 8000, 8006, 8008, 8080, 8081, 8443, 8888, 9443]
+        # Try HTTPS first, then HTTP
-        port = service.get('port')
+        for protocol in ['https', 'http']:
+            url = f"{protocol}://{ip}:{port}/"
+            try:
+                response = requests.get(
+                    url,
+                    timeout=3,
+                    verify=False,
+                    allow_redirects=False
+                )
+                # Any status code means it's a web server
+                # (including 404, 500, etc. - still a web server)
+                return True
+            except requests.exceptions.SSLError:
+                # SSL error on HTTPS, try HTTP next
+                continue
+            except (requests.exceptions.ConnectionError,
+                    requests.exceptions.Timeout,
+                    requests.exceptions.RequestException):
+                continue
 
-        return port in web_ports
+        return False
 
     def _detect_http_https(self, ip: str, port: int, timeout: int = 5) -> str:
         """
@@ -886,7 +914,7 @@ class SneakyScanner:
             ip_results = {}
 
             for service in services:
-                if not self._is_likely_web_service(service):
+                if not self._is_likely_web_service(service, ip):
                     continue
 
                 port = service['port']

app/web/api/scans.py

@@ -7,6 +7,9 @@ scan results.
 
 import json
 import logging
+from datetime import datetime
+from pathlib import Path
+
 from flask import Blueprint, current_app, jsonify, request
 from sqlalchemy.exc import SQLAlchemyError
 
@@ -20,6 +23,89 @@ bp = Blueprint('scans', __name__)
 logger = logging.getLogger(__name__)
 
 
+def _recover_orphaned_scan(scan: Scan, session) -> dict:
+    """
+    Recover an orphaned scan by checking for output files.
+
+    If output files exist: mark as 'completed' (smart recovery)
+    If no output files: mark as 'cancelled'
+
+    Args:
+        scan: The orphaned Scan object
+        session: Database session
+
+    Returns:
+        Dictionary with recovery result for API response
+    """
+    # Check for existing output files
+    output_exists = False
+    output_files_found = []
+
+    # Check paths stored in database
+    if scan.json_path and Path(scan.json_path).exists():
+        output_exists = True
+        output_files_found.append('json')
+    if scan.html_path and Path(scan.html_path).exists():
+        output_files_found.append('html')
+    if scan.zip_path and Path(scan.zip_path).exists():
+        output_files_found.append('zip')
+
+    # Also check by timestamp pattern if paths not stored yet
+    if not output_exists and scan.started_at:
+        output_dir = Path('/app/output')
+        if output_dir.exists():
+            timestamp_pattern = scan.started_at.strftime('%Y%m%d')
+            for json_file in output_dir.glob(f'scan_report_{timestamp_pattern}*.json'):
+                output_exists = True
+                output_files_found.append('json')
+                # Update scan record with found paths
+                scan.json_path = str(json_file)
+                html_file = json_file.with_suffix('.html')
+                if html_file.exists():
+                    scan.html_path = str(html_file)
+                    output_files_found.append('html')
+                zip_file = json_file.with_suffix('.zip')
+                if zip_file.exists():
+                    scan.zip_path = str(zip_file)
+                    output_files_found.append('zip')
+                break
+
+    if output_exists:
+        # Smart recovery: outputs exist, mark as completed
+        scan.status = 'completed'
+        scan.completed_at = datetime.utcnow()
+        if scan.started_at:
+            scan.duration = (datetime.utcnow() - scan.started_at).total_seconds()
+        scan.error_message = None
+        session.commit()
+
+        logger.info(f"Scan {scan.id}: Recovered as completed (files: {output_files_found})")
+
+        return {
+            'scan_id': scan.id,
+            'status': 'completed',
+            'message': f'Scan recovered as completed (output files found: {", ".join(output_files_found)})',
+            'recovery_type': 'smart_recovery'
+        }
+    else:
+        # No outputs: mark as cancelled
+        scan.status = 'cancelled'
+        scan.completed_at = datetime.utcnow()
+        if scan.started_at:
+            scan.duration = (datetime.utcnow() - scan.started_at).total_seconds()
+        scan.error_message = 'Scan process was interrupted before completion. No output files were generated.'
+        session.commit()
+
+        logger.info(f"Scan {scan.id}: Marked as cancelled (orphaned, no output files)")
+
+        return {
+            'scan_id': scan.id,
+            'status': 'cancelled',
+            'message': 'Orphaned scan cancelled (no output files found)',
+            'recovery_type': 'orphan_cleanup'
+        }
+
+
 @bp.route('', methods=['GET'])
 @api_auth_required
 def list_scans():
@@ -247,18 +333,23 @@ def delete_scan(scan_id):
 @api_auth_required
 def stop_running_scan(scan_id):
     """
-    Stop a running scan.
+    Stop a running scan with smart recovery for orphaned scans.
+
+    If the scan is actively running in the registry, sends a cancel signal.
+    If the scan shows as running/finalizing but is not in the registry (orphaned),
+    performs smart recovery: marks as 'completed' if output files exist,
+    otherwise marks as 'cancelled'.
 
     Args:
         scan_id: Scan ID to stop
 
     Returns:
-        JSON response with stop status
+        JSON response with stop status or recovery result
     """
     try:
         session = current_app.db_session
 
-        # Check if scan exists and is running
+        # Check if scan exists
         scan = session.query(Scan).filter_by(id=scan_id).first()
         if not scan:
             logger.warning(f"Scan not found for stop request: {scan_id}")
@@ -267,7 +358,8 @@ def stop_running_scan(scan_id):
                 'message': f'Scan with ID {scan_id} not found'
             }), 404
 
-        if scan.status != 'running':
+        # Allow stopping scans with status 'running' or 'finalizing'
+        if scan.status not in ('running', 'finalizing'):
             logger.warning(f"Cannot stop scan {scan_id}: status is '{scan.status}'")
             return jsonify({
                 'error': 'Invalid state',
@@ -288,11 +380,11 @@ def stop_running_scan(scan_id):
                 'status': 'stopping'
             }), 200
         else:
-            logger.warning(f"Failed to stop scan {scan_id}: not found in running scanners")
+            # Scanner not in registry - this is an orphaned scan
-            return jsonify({
+            # Attempt smart recovery
-                'error': 'Stop failed',
+            logger.warning(f"Scan {scan_id} not in registry, attempting smart recovery")
-                'message': 'Scan not found in running scanners registry'
+            recovery_result = _recover_orphaned_scan(scan, session)
-            }), 404
+            return jsonify(recovery_result), 200
 
     except SQLAlchemyError as e:
         logger.error(f"Database error stopping scan {scan_id}: {str(e)}")

app/web/app.py

@@ -307,9 +307,12 @@ def init_scheduler(app: Flask) -> None:
     with app.app_context():
         # Clean up any orphaned scans from previous crashes/restarts
         scan_service = ScanService(app.db_session)
-        orphaned_count = scan_service.cleanup_orphaned_scans()
+        cleanup_result = scan_service.cleanup_orphaned_scans()
-        if orphaned_count > 0:
+        if cleanup_result['total'] > 0:
-            app.logger.warning(f"Cleaned up {orphaned_count} orphaned scan(s) on startup")
+            app.logger.warning(
+                f"Cleaned up {cleanup_result['total']} orphaned scan(s) on startup: "
+                f"{cleanup_result['recovered']} recovered, {cleanup_result['failed']} failed"
+            )
 
         # Load all enabled schedules from database
         scheduler.load_schedules_on_startup()

app/web/jobs/scan_job.py

@@ -240,11 +240,44 @@ def execute_scan(scan_id: int, config_id: int, db_url: str = None):
         scan_duration = (end_time - start_time).total_seconds()
         logger.info(f"Scan {scan_id}: Scanner completed in {scan_duration:.2f} seconds")
 
-        # Generate output files (JSON, HTML, ZIP)
+        # Transition to 'finalizing' status before output generation
+        try:
+            scan = session.query(Scan).filter_by(id=scan_id).first()
+            if scan:
+                scan.status = 'finalizing'
+                scan.current_phase = 'generating_outputs'
+                session.commit()
+                logger.info(f"Scan {scan_id}: Status changed to 'finalizing'")
+        except Exception as e:
+            logger.error(f"Scan {scan_id}: Failed to update status to finalizing: {e}")
+            session.rollback()
+
+        # Generate output files (JSON, HTML, ZIP) with error handling
+        output_paths = {}
+        output_generation_failed = False
+        try:
             logger.info(f"Scan {scan_id}: Generating output files...")
             output_paths = scanner.generate_outputs(report, timestamp)
+        except Exception as e:
+            output_generation_failed = True
+            logger.error(f"Scan {scan_id}: Output generation failed: {str(e)}")
+            logger.error(f"Scan {scan_id}: Traceback:\n{traceback.format_exc()}")
+            # Still mark scan as completed with warning since scan data is valid
+            try:
+                scan = session.query(Scan).filter_by(id=scan_id).first()
+                if scan:
+                    scan.status = 'completed'
+                    scan.error_message = f"Scan completed but output file generation failed: {str(e)}"
+                    scan.completed_at = datetime.utcnow()
+                    if scan.started_at:
+                        scan.duration = (datetime.utcnow() - scan.started_at).total_seconds()
+                    session.commit()
+                    logger.info(f"Scan {scan_id}: Marked as completed with output generation warning")
+            except Exception as db_error:
+                logger.error(f"Scan {scan_id}: Failed to update status after output error: {db_error}")
 
-        # Save results to database
+        # Save results to database (only if output generation succeeded)
+        if not output_generation_failed:
             logger.info(f"Scan {scan_id}: Saving results to database...")
             scan_service = ScanService(session)
             scan_service._save_scan_to_db(report, scan_id, status='completed', output_paths=output_paths)

app/web/models.py

@@ -45,7 +45,7 @@ class Scan(Base):
     id = Column(Integer, primary_key=True, autoincrement=True)
     timestamp = Column(DateTime, nullable=False, index=True, comment="Scan start time (UTC)")
     duration = Column(Float, nullable=True, comment="Total scan duration in seconds")
-    status = Column(String(20), nullable=False, default='running', comment="running, completed, failed")
+    status = Column(String(20), nullable=False, default='running', comment="running, finalizing, completed, failed, cancelled")
     config_id = Column(Integer, ForeignKey('scan_configs.id'), nullable=True, index=True, comment="FK to scan_configs table")
     title = Column(Text, nullable=True, comment="Scan title from config")
     json_path = Column(Text, nullable=True, comment="Path to JSON report")

app/web/services/scan_service.py

@@ -286,52 +286,96 @@ class ScanService:
 
         return [self._scan_to_summary_dict(scan) for scan in scans]
 
-    def cleanup_orphaned_scans(self) -> int:
+    def cleanup_orphaned_scans(self) -> dict:
         """
-        Clean up orphaned scans that are stuck in 'running' status.
+        Clean up orphaned scans with smart recovery.
+
+        For scans stuck in 'running' or 'finalizing' status:
+        - If output files exist: mark as 'completed' (smart recovery)
+        - If no output files: mark as 'failed'
 
         This should be called on application startup to handle scans that
         were running when the system crashed or was restarted.
 
-        Scans in 'running' status are marked as 'failed' with an appropriate
-        error message indicating they were orphaned.
-
         Returns:
-            Number of orphaned scans cleaned up
+            Dictionary with cleanup results: {'recovered': N, 'failed': N, 'total': N}
         """
-        # Find all scans with status='running'
+        # Find all scans with status='running' or 'finalizing'
-        orphaned_scans = self.db.query(Scan).filter(Scan.status == 'running').all()
+        orphaned_scans = self.db.query(Scan).filter(
+            Scan.status.in_(['running', 'finalizing'])
+        ).all()
 
         if not orphaned_scans:
             logger.info("No orphaned scans found")
-            return 0
+            return {'recovered': 0, 'failed': 0, 'total': 0}
 
         count = len(orphaned_scans)
-        logger.warning(f"Found {count} orphaned scan(s) in 'running' status, marking as failed")
+        logger.warning(f"Found {count} orphaned scan(s), attempting smart recovery")
+
+        recovered_count = 0
+        failed_count = 0
+        output_dir = Path('/app/output')
 
-        # Mark each orphaned scan as failed
         for scan in orphaned_scans:
+            # Check for existing output files
+            output_exists = False
+            output_files_found = []
+
+            # Check paths stored in database
+            if scan.json_path and Path(scan.json_path).exists():
+                output_exists = True
+                output_files_found.append('json')
+            if scan.html_path and Path(scan.html_path).exists():
+                output_files_found.append('html')
+            if scan.zip_path and Path(scan.zip_path).exists():
+                output_files_found.append('zip')
+
+            # Also check by timestamp pattern if paths not stored yet
+            if not output_exists and scan.started_at and output_dir.exists():
+                timestamp_pattern = scan.started_at.strftime('%Y%m%d')
+                for json_file in output_dir.glob(f'scan_report_{timestamp_pattern}*.json'):
+                    output_exists = True
+                    output_files_found.append('json')
+                    # Update scan record with found paths
+                    scan.json_path = str(json_file)
+                    html_file = json_file.with_suffix('.html')
+                    if html_file.exists():
+                        scan.html_path = str(html_file)
+                        output_files_found.append('html')
+                    zip_file = json_file.with_suffix('.zip')
+                    if zip_file.exists():
+                        scan.zip_path = str(zip_file)
+                        output_files_found.append('zip')
+                    break
+
+            if output_exists:
+                # Smart recovery: outputs exist, mark as completed
+                scan.status = 'completed'
+                scan.error_message = f'Recovered from orphaned state (output files found: {", ".join(output_files_found)})'
+                recovered_count += 1
+                logger.info(f"Recovered orphaned scan {scan.id} as completed (files: {output_files_found})")
+            else:
+                # No outputs: mark as failed
                 scan.status = 'failed'
-            scan.completed_at = datetime.utcnow()
                 scan.error_message = (
                     "Scan was interrupted by system shutdown or crash. "
-                "The scan was running but did not complete normally."
+                    "No output files were generated."
                 )
+                failed_count += 1
+                logger.info(f"Marked orphaned scan {scan.id} as failed (no output files)")
 
-            # Calculate duration if we have a started_at time
+            scan.completed_at = datetime.utcnow()
             if scan.started_at:
-                duration = (datetime.utcnow() - scan.started_at).total_seconds()
+                scan.duration = (datetime.utcnow() - scan.started_at).total_seconds()
-                scan.duration = duration
-
-            logger.info(
-                f"Marked orphaned scan {scan.id} as failed "
-                f"(started: {scan.started_at.isoformat() if scan.started_at else 'unknown'})"
-            )
 
         self.db.commit()
-        logger.info(f"Cleaned up {count} orphaned scan(s)")
+        logger.info(f"Cleaned up {count} orphaned scan(s): {recovered_count} recovered, {failed_count} failed")
 
-        return count
+        return {
+            'recovered': recovered_count,
+            'failed': failed_count,
+            'total': count
+        }
 
     def _save_scan_to_db(self, report: Dict[str, Any], scan_id: int,
                         status: str = 'completed', output_paths: Dict = None) -> None:

app/web/services/schedule_service.py

@@ -6,7 +6,7 @@ scheduled scans with cron expressions.
 """
 
 import logging
-from datetime import datetime
+from datetime import datetime, timezone
 from typing import Any, Dict, List, Optional, Tuple
 
 from croniter import croniter
@@ -71,6 +71,7 @@ class ScheduleService:
         next_run = self.calculate_next_run(cron_expression) if enabled else None
 
         # Create schedule record
+        now_utc = datetime.now(timezone.utc)
         schedule = Schedule(
             name=name,
             config_id=config_id,
@@ -78,8 +79,8 @@ class ScheduleService:
             enabled=enabled,
             last_run=None,
             next_run=next_run,
-            created_at=datetime.utcnow(),
+            created_at=now_utc,
-            updated_at=datetime.utcnow()
+            updated_at=now_utc
         )
 
         self.db.add(schedule)
@@ -103,7 +104,14 @@ class ScheduleService:
         Raises:
             ValueError: If schedule not found
         """
-        schedule = self.db.query(Schedule).filter(Schedule.id == schedule_id).first()
+        from sqlalchemy.orm import joinedload
+
+        schedule = (
+            self.db.query(Schedule)
+            .options(joinedload(Schedule.config))
+            .filter(Schedule.id == schedule_id)
+            .first()
+        )
 
         if not schedule:
             raise ValueError(f"Schedule {schedule_id} not found")
@@ -138,8 +146,10 @@ class ScheduleService:
                 'pages': int
             }
         """
-        # Build query
+        from sqlalchemy.orm import joinedload
-        query = self.db.query(Schedule)
+
+        # Build query and eagerly load config relationship
+        query = self.db.query(Schedule).options(joinedload(Schedule.config))
 
         # Apply filter
         if enabled_filter is not None:
@@ -215,7 +225,7 @@ class ScheduleService:
             if hasattr(schedule, key):
                 setattr(schedule, key, value)
 
-        schedule.updated_at = datetime.utcnow()
+        schedule.updated_at = datetime.now(timezone.utc)
 
         self.db.commit()
         self.db.refresh(schedule)
@@ -298,7 +308,7 @@ class ScheduleService:
 
         schedule.last_run = last_run
         schedule.next_run = next_run
-        schedule.updated_at = datetime.utcnow()
+        schedule.updated_at = datetime.now(timezone.utc)
 
         self.db.commit()
 
@@ -311,23 +321,43 @@ class ScheduleService:
         Validate a cron expression.
 
         Args:
-            cron_expr: Cron expression to validate
+            cron_expr: Cron expression to validate in standard crontab format
+                      Format: minute hour day month day_of_week
+                      Day of week: 0=Sunday, 1=Monday, ..., 6=Saturday
+                      (APScheduler will convert this to its internal format automatically)
 
         Returns:
             Tuple of (is_valid, error_message)
             - (True, None) if valid
             - (False, error_message) if invalid
+
+        Note:
+            This validates using croniter which uses standard crontab format.
+            APScheduler's from_crontab() will handle the conversion when the
+            schedule is registered with the scheduler.
         """
         try:
             # Try to create a croniter instance
-            base_time = datetime.utcnow()
+            # croniter uses standard crontab format (Sunday=0)
+            from datetime import timezone
+            base_time = datetime.now(timezone.utc)
             cron = croniter(cron_expr, base_time)
 
             # Try to get the next run time (validates the expression)
             cron.get_next(datetime)
 
+            # Validate basic format (5 fields)
+            fields = cron_expr.split()
+            if len(fields) != 5:
+                return (False, f"Cron expression must have 5 fields (minute hour day month day_of_week), got {len(fields)}")
+
             return (True, None)
         except (ValueError, KeyError) as e:
+            error_msg = str(e)
+            # Add helpful hint for day_of_week errors
+            if "day" in error_msg.lower() and len(cron_expr.split()) >= 5:
+                hint = "\nNote: Use standard crontab format where 0=Sunday, 1=Monday, ..., 6=Saturday"
+                return (False, f"{error_msg}{hint}")
             return (False, str(e))
         except Exception as e:
             return (False, f"Unexpected error: {str(e)}")
@@ -345,17 +375,24 @@ class ScheduleService:
             from_time: Base time (defaults to now UTC)
 
         Returns:
-            Next run datetime (UTC)
+            Next run datetime (UTC, timezone-aware)
 
         Raises:
             ValueError: If cron expression is invalid
         """
         if from_time is None:
-            from_time = datetime.utcnow()
+            from_time = datetime.now(timezone.utc)
 
         try:
             cron = croniter(cron_expr, from_time)
-            return cron.get_next(datetime)
+            next_run = cron.get_next(datetime)
+
+            # croniter returns naive datetime, so we need to add timezone info
+            # Since we're using UTC for all calculations, add UTC timezone
+            if next_run.tzinfo is None:
+                next_run = next_run.replace(tzinfo=timezone.utc)
+
+            return next_run
         except Exception as e:
             raise ValueError(f"Invalid cron expression '{cron_expr}': {str(e)}")
 
@@ -403,10 +440,16 @@ class ScheduleService:
         Returns:
             Dictionary representation
         """
+        # Get config title if relationship is loaded
+        config_name = None
+        if schedule.config:
+            config_name = schedule.config.title
+
         return {
             'id': schedule.id,
             'name': schedule.name,
             'config_id': schedule.config_id,
+            'config_name': config_name,
             'cron_expression': schedule.cron_expression,
             'enabled': schedule.enabled,
             'last_run': schedule.last_run.isoformat() if schedule.last_run else None,
@@ -421,7 +464,7 @@ class ScheduleService:
         Format datetime as relative time.
 
         Args:
-            dt: Datetime to format (UTC)
+            dt: Datetime to format (UTC, can be naive or aware)
 
         Returns:
             Human-readable relative time (e.g., "in 2 hours", "yesterday")
@@ -429,7 +472,13 @@ class ScheduleService:
         if dt is None:
             return None
 
-        now = datetime.utcnow()
+        # Ensure both datetimes are timezone-aware for comparison
+        now = datetime.now(timezone.utc)
+
+        # If dt is naive, assume it's UTC and add timezone info
+        if dt.tzinfo is None:
+            dt = dt.replace(tzinfo=timezone.utc)
+
         diff = dt - now
 
         # Future times

app/web/services/scheduler_service.py

@@ -149,6 +149,51 @@ class SchedulerService:
         except Exception as e:
             logger.error(f"Error loading schedules on startup: {str(e)}", exc_info=True)
 
+    @staticmethod
+    def validate_cron_expression(cron_expression: str) -> tuple[bool, str]:
+        """
+        Validate a cron expression and provide helpful feedback.
+
+        Args:
+            cron_expression: Cron expression to validate
+
+        Returns:
+            Tuple of (is_valid: bool, message: str)
+            - If valid: (True, "Valid cron expression")
+            - If invalid: (False, "Error message with details")
+
+        Note:
+            Standard crontab format: minute hour day month day_of_week
+            Day of week: 0=Sunday, 1=Monday, ..., 6=Saturday (or 7=Sunday)
+        """
+        from apscheduler.triggers.cron import CronTrigger
+
+        try:
+            # Try to parse the expression
+            trigger = CronTrigger.from_crontab(cron_expression)
+
+            # Validate basic format (5 fields)
+            fields = cron_expression.split()
+            if len(fields) != 5:
+                return False, f"Cron expression must have 5 fields (minute hour day month day_of_week), got {len(fields)}"
+
+            return True, "Valid cron expression"
+
+        except (ValueError, KeyError) as e:
+            error_msg = str(e)
+
+            # Provide helpful hints for common errors
+            if "day_of_week" in error_msg.lower() or (len(cron_expression.split()) >= 5):
+                # Check if day_of_week field might be using APScheduler format by mistake
+                fields = cron_expression.split()
+                if len(fields) == 5:
+                    dow_field = fields[4]
+                    if dow_field.isdigit() and int(dow_field) >= 0:
+                        hint = "\nNote: Use standard crontab format where 0=Sunday, 1=Monday, ..., 6=Saturday"
+                        return False, f"Invalid cron expression: {error_msg}{hint}"
+
+            return False, f"Invalid cron expression: {error_msg}"
+
     def queue_scan(self, scan_id: int, config_id: int) -> str:
         """
         Queue a scan for immediate background execution.
@@ -188,6 +233,10 @@ class SchedulerService:
             schedule_id: Database ID of the schedule
             config_id: Database config ID
             cron_expression: Cron expression (e.g., "0 2 * * *" for 2am daily)
+                           IMPORTANT: Use standard crontab format where:
+                           - Day of week: 0 = Sunday, 1 = Monday, ..., 6 = Saturday
+                           - APScheduler automatically converts to its internal format
+                           - from_crontab() handles the conversion properly
 
         Returns:
             Job ID from APScheduler
@@ -195,18 +244,29 @@ class SchedulerService:
         Raises:
             RuntimeError: If scheduler not initialized
             ValueError: If cron expression is invalid
+
+        Note:
+            APScheduler internally uses Monday=0, but from_crontab() accepts
+            standard crontab format (Sunday=0) and converts it automatically.
         """
         if not self.scheduler:
             raise RuntimeError("Scheduler not initialized. Call init_scheduler() first.")
 
         from apscheduler.triggers.cron import CronTrigger
 
+        # Validate cron expression first to provide helpful error messages
+        is_valid, message = self.validate_cron_expression(cron_expression)
+        if not is_valid:
+            raise ValueError(message)
+
         # Create cron trigger from expression using local timezone
-        # This allows users to specify times in their local timezone
+        # from_crontab() parses standard crontab format (Sunday=0)
+        # and converts to APScheduler's internal format (Monday=0) automatically
         try:
             trigger = CronTrigger.from_crontab(cron_expression)
             # timezone defaults to local system timezone
         except (ValueError, KeyError) as e:
+            # This should not happen due to validation above, but catch anyway
             raise ValueError(f"Invalid cron expression '{cron_expression}': {str(e)}")
 
         # Add cron job
@@ -294,11 +354,16 @@ class SchedulerService:
 
                 # Update schedule's last_run and next_run
                 from croniter import croniter
-                next_run = croniter(schedule['cron_expression'], datetime.utcnow()).get_next(datetime)
+                now_utc = datetime.now(timezone.utc)
+                next_run = croniter(schedule['cron_expression'], now_utc).get_next(datetime)
+
+                # croniter returns naive datetime, add UTC timezone
+                if next_run.tzinfo is None:
+                    next_run = next_run.replace(tzinfo=timezone.utc)
 
                 schedule_service.update_run_times(
                     schedule_id=schedule_id,
-                    last_run=datetime.utcnow(),
+                    last_run=now_utc,
                     next_run=next_run
                 )
 

app/web/templates/schedule_edit.html

@@ -298,7 +298,11 @@ async function loadSchedule() {
 function populateForm(schedule) {
     document.getElementById('schedule-id').value = schedule.id;
     document.getElementById('schedule-name').value = schedule.name;
-    document.getElementById('config-id').value = schedule.config_id;
+    // Display config name and ID in the readonly config-file field
+    const configDisplay = schedule.config_name
+        ? `${schedule.config_name} (ID: ${schedule.config_id})`
+        : `Config ID: ${schedule.config_id}`;
+    document.getElementById('config-file').value = configDisplay;
     document.getElementById('cron-expression').value = schedule.cron_expression;
     document.getElementById('schedule-enabled').checked = schedule.enabled;
 

app/web/utils/validators.py

@@ -23,7 +23,7 @@ def validate_scan_status(status: str) -> tuple[bool, Optional[str]]:
         >>> validate_scan_status('invalid')
         (False, 'Invalid status: invalid. Must be one of: running, completed, failed')
     """
-    valid_statuses = ['running', 'completed', 'failed', 'cancelled']
+    valid_statuses = ['running', 'finalizing', 'completed', 'failed', 'cancelled']
 
     if status not in valid_statuses:
         return False, f'Invalid status: {status}. Must be one of: {", ".join(valid_statuses)}'

docker-compose.yml

@@ -2,12 +2,10 @@ version: '3.8'
 
 services:
   web:
-    build: .
+    image: sneakyscan
-    image: sneakyscanner:latest
     container_name: sneakyscanner-web
-    # Use entrypoint script that auto-initializes database on first run
+    working_dir: /app
-    entrypoint: ["/docker-entrypoint.sh"]
+    entrypoint: ["python3", "-u", "-m", "web.app"]
-    command: ["python3", "-u", "-m", "web.app"]
     # Note: Using host network mode for scanner capabilities, so no port mapping needed
     # The Flask app will be accessible at http://localhost:5000
     volumes:
@@ -59,8 +57,7 @@ services:
   # Optional: Initialize database on first run
   # Run with: docker-compose -f docker-compose-web.yml run --rm init-db
   init-db:
-    build: .
+    image: sneakyscan
-    image: sneakyscanner:latest
     container_name: sneakyscanner-init-db
     entrypoint: ["python3"]
     command: ["init_db.py", "--db-url", "sqlite:////app/data/sneakyscanner.db"]
@@ -68,3 +65,4 @@ services:
       - ./data:/app/data
     profiles:
       - tools
+    networks: []

docs/DEPLOYMENT.md

@@ -24,10 +24,10 @@ SneakyScanner is deployed as a Docker container running a Flask web application
 
 **Architecture:**
 - **Web Application**: Flask app on port 5000 with modern web UI
-- **Database**: SQLite (persisted to volume)
+- **Database**: SQLite (persisted to volume) - stores all configurations, scan results, and settings
 - **Background Jobs**: APScheduler for async scan execution
 - **Scanner**: masscan, nmap, sslyze, Playwright
-- **Config Creator**: Web-based CIDR-to-YAML configuration builder
+- **Config Management**: Database-backed configuration system managed entirely via web UI
 - **Scheduling**: Cron-based scheduled scans with dashboard management
 
 ---
@@ -143,6 +143,13 @@ docker compose -f docker-compose-standalone.yml up
 
 SneakyScanner is configured via environment variables. The recommended approach is to use a `.env` file.
 
+
+**UDP Port Scanning**
+
+- UDP Port scanning is disabled by default.
+- You can turn it on via the .env variable.
+- By Default, UDP port scanning only scans the top 20 ports, for convenience I have included the NMAP top 100 UDP ports as well.
+
 #### Creating Your .env File
 
 ```bash
@@ -160,6 +167,7 @@ python3 -c "from cryptography.fernet import Fernet; print('SNEAKYSCANNER_ENCRYPT
 nano .env
 ```
 
+
 #### Key Configuration Options
 
 | Variable | Description | Default | Required |
@@ -190,54 +198,30 @@ The application needs these directories (created automatically by Docker):
 
 ```bash
 # Verify directories exist
-ls -la configs/ data/ output/ logs/
+ls -la data/ output/ logs/
 
 # If missing, create them
-mkdir -p configs data output logs
+mkdir -p data output logs
 ```
 
 ### Step 2: Configure Scan Targets
 
-You can create scan configurations in two ways:
+After starting the application, create scan configurations using the web UI:
 
-**Option A: Using the Web UI (Recommended - Phase 4 Feature)**
+**Creating Configurations via Web UI**
 
 1. Navigate to **Configs** in the web interface
 2. Click **"Create New Config"**
-3. Use the CIDR-based config creator for quick setup:
+3. Use the form-based config creator:
    - Enter site name
    - Enter CIDR range (e.g., `192.168.1.0/24`)
-   - Select expected ports from dropdowns
+   - Select expected TCP/UDP ports from dropdowns
-   - Click **"Generate Config"**
+   - Optionally enable ping checks
-4. Or use the **YAML Editor** for advanced configurations
+4. Click **"Save Configuration"**
-5. Save and use immediately in scans or schedules
+5. Configuration is saved to database and immediately available for scans and schedules
 
-**Option B: Manual YAML Files**
+**Note**: All configurations are stored in the database, not as files. This provides better reliability, easier backup, and seamless management through the web interface.
 
-Create YAML configuration files manually in the `configs/` directory:
-
-```bash
-# Example configuration
-cat > configs/my-network.yaml <<EOF
-title: "My Network Infrastructure"
-sites:
-  - name: "Web Servers"
-    cidr: "192.168.1.0/24"  # Scan entire subnet
-    expected_ports:
-      - port: 80
-        protocol: tcp
-        service: "http"
-      - port: 443
-        protocol: tcp
-        service: "https"
-      - port: 22
-        protocol: tcp
-        service: "ssh"
-    ping_expected: true
-EOF
-```
-
-**Note**: Phase 4 introduced a powerful config creator in the web UI that makes it easy to generate configs from CIDR ranges without manually editing YAML.
 
 ### Step 3: Build Docker Image
 
@@ -389,38 +373,37 @@ The dashboard provides a central view of your scanning activity:
 - **Trend Charts**: Port count trends over time using Chart.js
 - **Quick Actions**: Buttons to run scans, create configs, manage schedules
 
-### Managing Scan Configurations (Phase 4)
+### Managing Scan Configurations
+
+All scan configurations are stored in the database and managed entirely through the web interface.
 
 **Creating Configs:**
 1. Navigate to **Configs** → **Create New Config**
-2. **CIDR Creator Mode**:
+2. Fill in the configuration form:
    - Enter site name (e.g., "Production Servers")
    - Enter CIDR range (e.g., `192.168.1.0/24`)
    - Select expected TCP/UDP ports from dropdowns
-   - Click **"Generate Config"** to create YAML
+   - Enable/disable ping checks
-3. **YAML Editor Mode**:
+3. Click **"Save Configuration"**
-   - Switch to editor tab for advanced configurations
+4. Configuration is immediately stored in database and available for use
-   - Syntax highlighting with line numbers
-   - Validate YAML before saving
 
 **Editing Configs:**
-1. Navigate to **Configs** → Select config
+1. Navigate to **Configs** → Select config from list
 2. Click **"Edit"** button
-3. Make changes in YAML editor
+3. Modify any fields in the configuration form
-4. Save changes (validates YAML automatically)
+4. Click **"Save Changes"** to update database
 
-**Uploading Configs:**
+**Viewing Configs:**
-1. Navigate to **Configs** → **Upload**
+- Navigate to **Configs** page to see all saved configurations
-2. Select YAML file from your computer
+- Each config shows site name, CIDR range, and expected ports
-3. File is validated and saved to `configs/` directory
+- Click on any config to view full details
-
-**Downloading Configs:**
-- Click **"Download"** button next to any config
-- Saves YAML file to your local machine
 
 **Deleting Configs:**
-- Click **"Delete"** button
+- Click **"Delete"** button next to any config
 - **Warning**: Cannot delete configs used by active schedules
+- Deletion removes the configuration from the database permanently
+
+**Note**: All configurations are database-backed, providing automatic backups when you backup the database file.
 
 ### Running Scans
 
@@ -477,12 +460,11 @@ SneakyScanner uses several mounted volumes for data persistence:
 
 | Volume | Container Path | Purpose | Important? |
 |--------|----------------|---------|------------|
-| `./configs` | `/app/configs` | Scan configuration files (managed via web UI) | Yes |
+| `./data` | `/app/data` | SQLite database (contains configurations, scan history, settings) | **Critical** |
-| `./data` | `/app/data` | SQLite database (contains all scan history) | **Critical** |
 | `./output` | `/app/output` | Scan results (JSON, HTML, ZIP, screenshots) | Yes |
 | `./logs` | `/app/logs` | Application logs (rotating file handler) | No |
 
-**Note**: As of Phase 4, the `./configs` volume is read-write to support the web-based config creator and editor. The web UI can now create, edit, and delete configuration files directly.
+**Note**: All scan configurations are stored in the SQLite database (`./data/sneakyscanner.db`). There is no separate configs directory or YAML files. Backing up the database file ensures all your configurations are preserved.
 
 ### Backing Up Data
 
@@ -490,23 +472,22 @@ SneakyScanner uses several mounted volumes for data persistence:
 # Create backup directory
 mkdir -p backups/$(date +%Y%m%d)
 
-# Backup database
+# Backup database (includes all configurations)
 cp data/sneakyscanner.db backups/$(date +%Y%m%d)/
 
 # Backup scan outputs
 tar -czf backups/$(date +%Y%m%d)/output.tar.gz output/
-
-# Backup configurations
-tar -czf backups/$(date +%Y%m%d)/configs.tar.gz configs/
 ```
 
+**Important**: The database backup includes all scan configurations, settings, schedules, and scan history. No separate configuration file backup is needed.
+
 ### Restoring Data
 
 ```bash
 # Stop application
 docker compose -f docker-compose.yml down
 
-# Restore database
+# Restore database (includes all configurations)
 cp backups/YYYYMMDD/sneakyscanner.db data/
 
 # Restore outputs
@@ -516,6 +497,8 @@ tar -xzf backups/YYYYMMDD/output.tar.gz
 docker compose -f docker-compose.yml up -d
 ```
 
+**Note**: Restoring the database file restores all configurations, settings, schedules, and scan history.
+
 ### Cleaning Up Old Scan Results
 
 **Option A: Using the Web UI (Recommended)**
@@ -564,50 +547,52 @@ curl -X POST http://localhost:5000/api/auth/logout \
   -b cookies.txt
 ```
 
-### Config Management (Phase 4)
+### Config Management
 
 ```bash
 # List all configs
 curl http://localhost:5000/api/configs \
   -b cookies.txt
 
-# Get specific config
+# Get specific config by ID
-curl http://localhost:5000/api/configs/prod-network.yaml \
+curl http://localhost:5000/api/configs/1 \
   -b cookies.txt
 
 # Create new config
 curl -X POST http://localhost:5000/api/configs \
   -H "Content-Type: application/json" \
   -d '{
-    "filename": "test-network.yaml",
+    "name": "Test Network",
-    "content": "title: Test Network\nsites:\n  - name: Test\n    cidr: 10.0.0.0/24"
+    "cidr": "10.0.0.0/24",
+    "expected_ports": [
+      {"port": 80, "protocol": "tcp", "service": "http"},
+      {"port": 443, "protocol": "tcp", "service": "https"}
+    ],
+    "ping_expected": true
   }' \
   -b cookies.txt
 
 # Update config
-curl -X PUT http://localhost:5000/api/configs/test-network.yaml \
+curl -X PUT http://localhost:5000/api/configs/1 \
   -H "Content-Type: application/json" \
   -d '{
-    "content": "title: Updated Test Network\nsites:\n  - name: Test Site\n    cidr: 10.0.0.0/24"
+    "name": "Updated Test Network",
+    "cidr": "10.0.1.0/24"
   }' \
   -b cookies.txt
 
-# Download config
-curl http://localhost:5000/api/configs/test-network.yaml/download \
-  -b cookies.txt -o test-network.yaml
-
 # Delete config
-curl -X DELETE http://localhost:5000/api/configs/test-network.yaml \
+curl -X DELETE http://localhost:5000/api/configs/1 \
   -b cookies.txt
 ```
 
 ### Scan Management
 
 ```bash
-# Trigger a scan
+# Trigger a scan (using config ID from database)
 curl -X POST http://localhost:5000/api/scans \
   -H "Content-Type: application/json" \
-  -d '{"config_id": "/app/configs/prod-network.yaml"}' \
+  -d '{"config_id": 1}' \
   -b cookies.txt
 
 # List all scans
@@ -634,12 +619,12 @@ curl -X DELETE http://localhost:5000/api/scans/123 \
 curl http://localhost:5000/api/schedules \
   -b cookies.txt
 
-# Create schedule
+# Create schedule (using config ID from database)
 curl -X POST http://localhost:5000/api/schedules \
   -H "Content-Type: application/json" \
   -d '{
     "name": "Daily Production Scan",
-    "config_id": "/app/configs/prod-network.yaml",
+    "config_id": 1,
     "cron_expression": "0 2 * * *",
     "enabled": true
   }' \
@@ -875,24 +860,25 @@ docker compose -f docker-compose.yml logs web | grep -E "(ERROR|Exception|Traceb
 docker compose -f docker-compose.yml exec web which masscan nmap
 ```
 
-### Config Files Not Appearing in Web UI
+### Configs Not Appearing in Web UI
 
-**Problem**: Manually created configs don't show up in web interface
+**Problem**: Created configs don't show up in web interface
 
 ```bash
-# Check file permissions (must be readable by web container)
+# Check database connectivity
-ls -la configs/
+docker compose -f docker-compose.yml logs web | grep -i "database"
 
-# Fix permissions if needed
+# Verify database file exists and is readable
-sudo chown -R 1000:1000 configs/
+ls -lh data/sneakyscanner.db
-chmod 644 configs/*.yaml
 
-# Verify YAML syntax is valid
+# Check for errors when creating configs
-docker compose -f docker-compose.yml exec web python3 -c \
-  "import yaml; yaml.safe_load(open('/app/configs/your-config.yaml'))"
-
-# Check web logs for parsing errors
 docker compose -f docker-compose.yml logs web | grep -i "config"
+
+# Try accessing configs via API
+curl http://localhost:5000/api/configs -b cookies.txt
+
+# If database is corrupted, check integrity
+docker compose -f docker-compose.yml exec web sqlite3 /app/data/sneakyscanner.db "PRAGMA integrity_check;"
 ```
 
 ### Health Check Failing
@@ -979,11 +965,11 @@ server {
 # Ensure proper ownership of data directories
 sudo chown -R $USER:$USER data/ output/ logs/
 
-# Restrict database file permissions
+# Restrict database file permissions (contains configurations and sensitive data)
 chmod 600 data/sneakyscanner.db
 
-# Configs should be read-only
+# Ensure database directory is writable
-chmod 444 configs/*.yaml
+chmod 700 data/
 ```
 
 ---
@@ -1051,19 +1037,17 @@ mkdir -p "$BACKUP_DIR"
 # Stop application for consistent backup
 docker compose -f docker-compose.yml stop web
 
-# Backup database
+# Backup database (includes all configurations)
 cp data/sneakyscanner.db "$BACKUP_DIR/"
 
 # Backup outputs (last 30 days only)
 find output/ -type f -mtime -30 -exec cp --parents {} "$BACKUP_DIR/" \;
 
-# Backup configs
-cp -r configs/ "$BACKUP_DIR/"
-
 # Restart application
 docker compose -f docker-compose.yml start web
 
 echo "Backup complete: $BACKUP_DIR"
+echo "Database backup includes all configurations, settings, and scan history"
 ```
 
 Make executable and schedule with cron:
@@ -1083,15 +1067,18 @@ crontab -e
 # Stop application
 docker compose -f docker-compose.yml down
 
-# Restore files
+# Restore database (includes all configurations)
 cp backups/YYYYMMDD_HHMMSS/sneakyscanner.db data/
-cp -r backups/YYYYMMDD_HHMMSS/configs/* configs/
+
+# Restore output files
 cp -r backups/YYYYMMDD_HHMMSS/output/* output/
 
 # Start application
 docker compose -f docker-compose.yml up -d
 ```
 
+**Note**: Restoring the database file will restore all configurations, settings, schedules, and scan history from the backup.
+
 ---
 
 ## Support and Further Reading
@@ -1105,13 +1092,13 @@ docker compose -f docker-compose.yml up -d
 
 ## What's New
 
-### Phase 4 (2025-11-17) - Config Creator ✅
+### Phase 4+ (2025-11-17) - Database-Backed Configuration System ✅
-- **CIDR-based Config Creator**: Web UI for generating scan configs from CIDR ranges
+- **Database-Backed Configs**: All configurations stored in SQLite database (no YAML files)
-- **YAML Editor**: Built-in editor with syntax highlighting (CodeMirror)
+- **Web-Based Config Creator**: Form-based UI for creating scan configs from CIDR ranges
-- **Config Management UI**: List, view, edit, download, and delete configs via web interface
+- **Config Management UI**: List, view, edit, and delete configs via web interface
-- **Config Upload**: Direct YAML file upload for advanced users
+- **REST API**: Full config management via RESTful API with database storage
-- **REST API**: 7 new config management endpoints
 - **Schedule Protection**: Prevents deleting configs used by active schedules
+- **Automatic Backups**: Configurations included in database backups
 
 ### Phase 3 (2025-11-14) - Dashboard & Scheduling ✅
 - **Dashboard**: Summary stats, recent scans, trend charts
@@ -1133,5 +1120,5 @@ docker compose -f docker-compose.yml up -d
 
 ---
 
-**Last Updated**: 2025-11-17
+**Last Updated**: 2025-11-24
-**Version**: Phase 4 - Config Creator Complete
+**Version**: Phase 4+ - Database-Backed Configuration System

setup.sh

@@ -91,27 +91,40 @@ echo "Creating required directories..."
 mkdir -p data logs output configs
 echo "✓ Directories created"
 
-# Check if Docker is running
+# Check if Podman is running
 echo ""
-echo "Checking Docker..."
+echo "Checking Podman..."
-if ! docker info > /dev/null 2>&1; then
+if ! podman info > /dev/null 2>&1; then
-    echo "✗ Docker is not running or not installed"
+    echo "✗ Podman is not running or not installed"
-    echo "Please install Docker and start the Docker daemon"
+    echo "Please install Podman"
     exit 1
 fi
-echo "✓ Docker is running"
+echo "✓ Podman is available"
 
 # Build and start
 echo ""
-echo "Building and starting SneakyScanner..."
+echo "Starting SneakyScanner..."
 echo "This may take a few minutes on first run..."
 echo ""
 
-docker compose build
+podman build --network=host -t sneakyscan .
 
+# Initialize database if it doesn't exist or is empty
 echo ""
-echo "Starting SneakyScanner..."
+echo "Initializing database..."
-docker compose up -d
+
+# Build init command with optional password
+INIT_CMD="init_db.py --db-url sqlite:////app/data/sneakyscanner.db --force"
+if [ -n "$INITIAL_PASSWORD" ]; then
+    INIT_CMD="$INIT_CMD --password $INITIAL_PASSWORD"
+fi
+
+podman run --rm --entrypoint python3 -w /app \
+    -v "$(pwd)/data:/app/data" \
+    sneakyscan $INIT_CMD
+echo "✓ Database initialized"
+
+podman-compose up -d
 
 # Wait for service to be healthy
 echo ""
@@ -119,7 +132,7 @@ echo "Waiting for application to start..."
 sleep 5
 
 # Check if container is running
-if docker ps | grep -q sneakyscanner-web; then
+if podman ps | grep -q sneakyscanner-web; then
     echo ""
     echo "================================================"
     echo "  ✓ SneakyScanner is Running!"
@@ -140,15 +153,15 @@ if docker ps | grep -q sneakyscanner-web; then
     fi
     echo ""
     echo "Useful commands:"
-    echo "  docker compose logs -f     # View logs"
+    echo "  podman-compose logs -f     # View logs"
-    echo "  docker compose stop        # Stop the service"
+    echo "  podman-compose stop        # Stop the service"
-    echo "  docker compose restart     # Restart the service"
+    echo "  podman-compose restart     # Restart the service"
     echo ""
     echo "⚠  IMPORTANT: Change your password after first login!"
     echo "================================================"
 else
     echo ""
     echo "✗ Container failed to start. Check logs with:"
-    echo "  docker compose logs"
+    echo "  podman-compose logs"
     exit 1
 fi