Migrate from file-based configs to database with per-IP site configuration

Major architectural changes:
   - Replace YAML config files with database-stored ScanConfig model
   - Remove CIDR block support in favor of individual IP addresses per site
   - Each IP now has its own expected_ping, expected_tcp_ports, expected_udp_ports
   - AlertRule now uses config_id FK instead of config_file string

   API changes:
   - POST /api/scans now requires config_id instead of config_file
   - Alert rules API uses config_id with validation
   - All config dropdowns fetch from /api/configs dynamically

   Template updates:
   - scans.html, dashboard.html, alert_rules.html load configs via API
   - Display format: Config Title (X sites) in dropdowns
   - Removed Jinja2 config_files loops

   Migrations:
   - 008: Expand CIDRs to individual IPs with per-IP port configs
   - 009: Remove CIDR-related columns
   - 010: Add config_id to alert_rules, remove config_file
This commit is contained in:
2025-11-19 19:40:34 -06:00
parent 034f146fa1
commit 0ec338e252
21 changed files with 2004 additions and 686 deletions

View File

@@ -0,0 +1,270 @@
"""Expand CIDRs to individual IPs with per-IP settings
Revision ID: 008
Revises: 007
Create Date: 2025-11-19
This migration changes the site architecture to automatically expand CIDRs into
individual IPs in the database. Each IP has its own port and ping settings.
Changes:
- Add site_id to site_ips (direct link to sites, support standalone IPs)
- Make site_cidr_id nullable (IPs can exist without a CIDR parent)
- Remove settings from site_cidrs (settings now only at IP level)
- Add unique constraint: no duplicate IPs within a site
- Expand existing CIDRs to individual IPs
"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy import text
import ipaddress
# revision identifiers, used by Alembic
revision = '008'
down_revision = '007'
branch_labels = None
depends_on = None
def upgrade():
"""
Modify schema to support per-IP settings and auto-expand CIDRs.
"""
connection = op.get_bind()
# Check if site_id column already exists
inspector = sa.inspect(connection)
site_ips_columns = [col['name'] for col in inspector.get_columns('site_ips')]
site_cidrs_columns = [col['name'] for col in inspector.get_columns('site_cidrs')]
# Step 1: Add site_id column to site_ips (will be populated from site_cidr_id)
if 'site_id' not in site_ips_columns:
print("Adding site_id column to site_ips...")
op.add_column('site_ips', sa.Column('site_id', sa.Integer(), nullable=True, comment='FK to sites (direct link)'))
else:
print("site_id column already exists in site_ips, skipping...")
# Step 2: Populate site_id from site_cidr_id (before we make it nullable)
print("Populating site_id from existing site_cidr relationships...")
connection.execute(text("""
UPDATE site_ips
SET site_id = (
SELECT site_id
FROM site_cidrs
WHERE site_cidrs.id = site_ips.site_cidr_id
)
WHERE site_cidr_id IS NOT NULL
"""))
# Step 3: Make site_id NOT NULL and add foreign key
# Check if foreign key exists before creating
try:
op.alter_column('site_ips', 'site_id', nullable=False)
print("Made site_id NOT NULL")
except Exception as e:
print(f"site_id already NOT NULL or error: {e}")
# Check if foreign key exists
try:
op.create_foreign_key('fk_site_ips_site_id', 'site_ips', 'sites', ['site_id'], ['id'])
print("Created foreign key fk_site_ips_site_id")
except Exception as e:
print(f"Foreign key already exists or error: {e}")
# Check if index exists
try:
op.create_index(op.f('ix_site_ips_site_id'), 'site_ips', ['site_id'], unique=False)
print("Created index ix_site_ips_site_id")
except Exception as e:
print(f"Index already exists or error: {e}")
# Step 4: Make site_cidr_id nullable (for standalone IPs)
try:
op.alter_column('site_ips', 'site_cidr_id', nullable=True)
print("Made site_cidr_id nullable")
except Exception as e:
print(f"site_cidr_id already nullable or error: {e}")
# Step 5: Drop old unique constraint and create new one (site_id, ip_address)
# This prevents duplicate IPs within a site (across all CIDRs and standalone)
try:
op.drop_constraint('uix_site_cidr_ip', 'site_ips', type_='unique')
print("Dropped old constraint uix_site_cidr_ip")
except Exception as e:
print(f"Constraint already dropped or doesn't exist: {e}")
try:
op.create_unique_constraint('uix_site_ip_address', 'site_ips', ['site_id', 'ip_address'])
print("Created new constraint uix_site_ip_address")
except Exception as e:
print(f"Constraint already exists or error: {e}")
# Step 6: Expand existing CIDRs to individual IPs
print("Expanding existing CIDRs to individual IPs...")
# Get all existing CIDRs
cidrs = connection.execute(text("""
SELECT id, site_id, cidr, expected_ping, expected_tcp_ports, expected_udp_ports
FROM site_cidrs
""")).fetchall()
expanded_count = 0
skipped_count = 0
for cidr_row in cidrs:
cidr_id, site_id, cidr_str, expected_ping, expected_tcp_ports, expected_udp_ports = cidr_row
try:
# Parse CIDR
network = ipaddress.ip_network(cidr_str, strict=False)
# Check size - skip if too large (> /24 for IPv4, > /64 for IPv6)
if isinstance(network, ipaddress.IPv4Network) and network.prefixlen < 24:
print(f" ⚠ Skipping large CIDR {cidr_str} (>{network.num_addresses} IPs)")
skipped_count += 1
continue
elif isinstance(network, ipaddress.IPv6Network) and network.prefixlen < 64:
print(f" ⚠ Skipping large CIDR {cidr_str} (>{network.num_addresses} IPs)")
skipped_count += 1
continue
# Expand to individual IPs
for ip in network.hosts() if network.num_addresses > 2 else [network.network_address]:
ip_str = str(ip)
# Check if this IP already exists (from previous IP overrides)
existing = connection.execute(text("""
SELECT id FROM site_ips
WHERE site_cidr_id = :cidr_id AND ip_address = :ip_address
"""), {'cidr_id': cidr_id, 'ip_address': ip_str}).fetchone()
if not existing:
# Insert new IP with settings from CIDR
connection.execute(text("""
INSERT INTO site_ips (
site_id, site_cidr_id, ip_address,
expected_ping, expected_tcp_ports, expected_udp_ports,
created_at
)
VALUES (
:site_id, :cidr_id, :ip_address,
:expected_ping, :expected_tcp_ports, :expected_udp_ports,
datetime('now')
)
"""), {
'site_id': site_id,
'cidr_id': cidr_id,
'ip_address': ip_str,
'expected_ping': expected_ping,
'expected_tcp_ports': expected_tcp_ports,
'expected_udp_ports': expected_udp_ports
})
expanded_count += 1
except Exception as e:
print(f" ✗ Error expanding CIDR {cidr_str}: {e}")
skipped_count += 1
continue
print(f" ✓ Expanded {expanded_count} IPs from CIDRs")
if skipped_count > 0:
print(f" ⚠ Skipped {skipped_count} CIDRs (too large or errors)")
# Step 7: Remove settings columns from site_cidrs (now only at IP level)
print("Removing settings columns from site_cidrs...")
# Re-inspect to get current columns
site_cidrs_columns = [col['name'] for col in inspector.get_columns('site_cidrs')]
if 'expected_ping' in site_cidrs_columns:
try:
op.drop_column('site_cidrs', 'expected_ping')
print("Dropped expected_ping from site_cidrs")
except Exception as e:
print(f"Error dropping expected_ping: {e}")
else:
print("expected_ping already dropped from site_cidrs")
if 'expected_tcp_ports' in site_cidrs_columns:
try:
op.drop_column('site_cidrs', 'expected_tcp_ports')
print("Dropped expected_tcp_ports from site_cidrs")
except Exception as e:
print(f"Error dropping expected_tcp_ports: {e}")
else:
print("expected_tcp_ports already dropped from site_cidrs")
if 'expected_udp_ports' in site_cidrs_columns:
try:
op.drop_column('site_cidrs', 'expected_udp_ports')
print("Dropped expected_udp_ports from site_cidrs")
except Exception as e:
print(f"Error dropping expected_udp_ports: {e}")
else:
print("expected_udp_ports already dropped from site_cidrs")
# Print summary
total_sites = connection.execute(text('SELECT COUNT(*) FROM sites')).scalar()
total_cidrs = connection.execute(text('SELECT COUNT(*) FROM site_cidrs')).scalar()
total_ips = connection.execute(text('SELECT COUNT(*) FROM site_ips')).scalar()
print("\n✓ Migration 008 complete: CIDRs expanded to individual IPs")
print(f" - Total sites: {total_sites}")
print(f" - Total CIDRs: {total_cidrs}")
print(f" - Total IPs: {total_ips}")
def downgrade():
"""
Revert schema changes (restore CIDR-level settings).
Note: This will lose per-IP granularity!
"""
connection = op.get_bind()
print("Rolling back to CIDR-level settings...")
# Step 1: Add settings columns back to site_cidrs
op.add_column('site_cidrs', sa.Column('expected_ping', sa.Boolean(), nullable=True))
op.add_column('site_cidrs', sa.Column('expected_tcp_ports', sa.Text(), nullable=True))
op.add_column('site_cidrs', sa.Column('expected_udp_ports', sa.Text(), nullable=True))
# Step 2: Populate CIDR settings from first IP in each CIDR (approximation)
connection.execute(text("""
UPDATE site_cidrs
SET
expected_ping = (
SELECT expected_ping FROM site_ips
WHERE site_ips.site_cidr_id = site_cidrs.id
LIMIT 1
),
expected_tcp_ports = (
SELECT expected_tcp_ports FROM site_ips
WHERE site_ips.site_cidr_id = site_cidrs.id
LIMIT 1
),
expected_udp_ports = (
SELECT expected_udp_ports FROM site_ips
WHERE site_ips.site_cidr_id = site_cidrs.id
LIMIT 1
)
"""))
# Step 3: Delete auto-expanded IPs (keep only original overrides)
# In practice, this is difficult to determine, so we'll keep all IPs
# and just remove the schema changes
# Step 4: Drop new unique constraint and restore old one
op.drop_constraint('uix_site_ip_address', 'site_ips', type_='unique')
op.create_unique_constraint('uix_site_cidr_ip', 'site_ips', ['site_cidr_id', 'ip_address'])
# Step 5: Make site_cidr_id NOT NULL again
op.alter_column('site_ips', 'site_cidr_id', nullable=False)
# Step 6: Drop site_id column and related constraints
op.drop_index(op.f('ix_site_ips_site_id'), table_name='site_ips')
op.drop_constraint('fk_site_ips_site_id', 'site_ips', type_='foreignkey')
op.drop_column('site_ips', 'site_id')
print("✓ Downgrade complete: Reverted to CIDR-level settings")

View File

@@ -0,0 +1,210 @@
"""Remove CIDR table - make sites IP-only
Revision ID: 009
Revises: 008
Create Date: 2025-11-19
This migration removes the SiteCIDR table entirely, making sites purely
IP-based. CIDRs are now only used as a convenience for bulk IP addition,
not stored as permanent entities.
Changes:
- Set all site_ips.site_cidr_id to NULL (preserve all IPs)
- Drop foreign key from site_ips to site_cidrs
- Drop site_cidrs table
- Remove site_cidr_id column from site_ips
All existing IPs are preserved. They become "standalone" IPs without
a CIDR parent.
"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy import text
# revision identifiers, used by Alembic
revision = '009'
down_revision = '008'
branch_labels = None
depends_on = None
def upgrade():
"""
Remove CIDR table and make all IPs standalone.
"""
connection = op.get_bind()
inspector = sa.inspect(connection)
print("\n=== Migration 009: Remove CIDR Table ===\n")
# Get counts before migration
try:
total_cidrs = connection.execute(text('SELECT COUNT(*) FROM site_cidrs')).scalar()
total_ips = connection.execute(text('SELECT COUNT(*) FROM site_ips')).scalar()
ips_with_cidr = connection.execute(text(
'SELECT COUNT(*) FROM site_ips WHERE site_cidr_id IS NOT NULL'
)).scalar()
print(f"Before migration:")
print(f" - Total CIDRs: {total_cidrs}")
print(f" - Total IPs: {total_ips}")
print(f" - IPs linked to CIDRs: {ips_with_cidr}")
print(f" - Standalone IPs: {total_ips - ips_with_cidr}\n")
except Exception as e:
print(f"Could not get pre-migration stats: {e}\n")
# Step 1: Set all site_cidr_id to NULL (preserve all IPs as standalone)
print("Step 1: Converting all IPs to standalone (nulling CIDR associations)...")
try:
result = connection.execute(text("""
UPDATE site_ips
SET site_cidr_id = NULL
WHERE site_cidr_id IS NOT NULL
"""))
print(f" ✓ Converted {result.rowcount} IPs to standalone\n")
except Exception as e:
print(f" ⚠ Error or already done: {e}\n")
# Step 2: Drop foreign key constraint from site_ips to site_cidrs
print("Step 2: Dropping foreign key constraint from site_ips to site_cidrs...")
foreign_keys = inspector.get_foreign_keys('site_ips')
fk_to_drop = None
for fk in foreign_keys:
if fk['referred_table'] == 'site_cidrs':
fk_to_drop = fk['name']
break
if fk_to_drop:
try:
op.drop_constraint(fk_to_drop, 'site_ips', type_='foreignkey')
print(f" ✓ Dropped foreign key constraint: {fk_to_drop}\n")
except Exception as e:
print(f" ⚠ Could not drop foreign key: {e}\n")
else:
print(" ⚠ Foreign key constraint not found or already dropped\n")
# Step 3: Drop index on site_cidr_id (if exists)
print("Step 3: Dropping index on site_cidr_id...")
indexes = inspector.get_indexes('site_ips')
index_to_drop = None
for idx in indexes:
if 'site_cidr_id' in idx['column_names']:
index_to_drop = idx['name']
break
if index_to_drop:
try:
op.drop_index(index_to_drop, table_name='site_ips')
print(f" ✓ Dropped index: {index_to_drop}\n")
except Exception as e:
print(f" ⚠ Could not drop index: {e}\n")
else:
print(" ⚠ Index not found or already dropped\n")
# Step 4: Drop site_cidrs table
print("Step 4: Dropping site_cidrs table...")
tables = inspector.get_table_names()
if 'site_cidrs' in tables:
try:
op.drop_table('site_cidrs')
print(" ✓ Dropped site_cidrs table\n")
except Exception as e:
print(f" ⚠ Could not drop table: {e}\n")
else:
print(" ⚠ Table site_cidrs not found or already dropped\n")
# Step 5: Drop site_cidr_id column from site_ips
print("Step 5: Dropping site_cidr_id column from site_ips...")
site_ips_columns = [col['name'] for col in inspector.get_columns('site_ips')]
if 'site_cidr_id' in site_ips_columns:
try:
op.drop_column('site_ips', 'site_cidr_id')
print(" ✓ Dropped site_cidr_id column from site_ips\n")
except Exception as e:
print(f" ⚠ Could not drop column: {e}\n")
else:
print(" ⚠ Column site_cidr_id not found or already dropped\n")
# Get counts after migration
try:
final_ips = connection.execute(text('SELECT COUNT(*) FROM site_ips')).scalar()
total_sites = connection.execute(text('SELECT COUNT(*) FROM sites')).scalar()
print("After migration:")
print(f" - Total sites: {total_sites}")
print(f" - Total IPs (all standalone): {final_ips}")
print(f" - CIDRs: N/A (table removed)")
except Exception as e:
print(f"Could not get post-migration stats: {e}")
print("\n✓ Migration 009 complete: Sites are now IP-only")
print(" All IPs preserved as standalone. CIDRs can still be used")
print(" via the API/UI for bulk IP creation, but are not stored.\n")
def downgrade():
"""
Recreate site_cidrs table (CANNOT restore original CIDR associations).
WARNING: This downgrade creates an empty site_cidrs table structure but
cannot restore the original CIDR-to-IP associations since that data was
deleted. All IPs will remain standalone.
"""
connection = op.get_bind()
print("\n=== Downgrade 009: Recreate CIDR Table Structure ===\n")
print("⚠ WARNING: Cannot restore original CIDR associations!")
print(" The site_cidrs table structure will be recreated but will be empty.")
print(" All IPs will remain standalone. This is a PARTIAL downgrade.\n")
# Step 1: Recreate site_cidrs table (empty)
print("Step 1: Recreating site_cidrs table structure...")
try:
op.create_table(
'site_cidrs',
sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
sa.Column('site_id', sa.Integer(), nullable=False),
sa.Column('cidr', sa.String(length=45), nullable=False, comment='CIDR notation (e.g., 10.0.0.0/24)'),
sa.Column('created_at', sa.DateTime(), nullable=False),
sa.PrimaryKeyConstraint('id'),
sa.ForeignKeyConstraint(['site_id'], ['sites.id'], ),
sa.UniqueConstraint('site_id', 'cidr', name='uix_site_cidr')
)
print(" ✓ Recreated site_cidrs table (empty)\n")
except Exception as e:
print(f" ⚠ Could not create table: {e}\n")
# Step 2: Add site_cidr_id column back to site_ips (nullable)
print("Step 2: Adding site_cidr_id column back to site_ips...")
try:
op.add_column('site_ips', sa.Column('site_cidr_id', sa.Integer(), nullable=True, comment='FK to site_cidrs (optional, for grouping)'))
print(" ✓ Added site_cidr_id column (nullable)\n")
except Exception as e:
print(f" ⚠ Could not add column: {e}\n")
# Step 3: Add foreign key constraint
print("Step 3: Adding foreign key constraint...")
try:
op.create_foreign_key('fk_site_ips_site_cidr_id', 'site_ips', 'site_cidrs', ['site_cidr_id'], ['id'])
print(" ✓ Created foreign key constraint\n")
except Exception as e:
print(f" ⚠ Could not create foreign key: {e}\n")
# Step 4: Add index on site_cidr_id
print("Step 4: Adding index on site_cidr_id...")
try:
op.create_index('ix_site_ips_site_cidr_id', 'site_ips', ['site_cidr_id'], unique=False)
print(" ✓ Created index on site_cidr_id\n")
except Exception as e:
print(f" ⚠ Could not create index: {e}\n")
print("✓ Downgrade complete: CIDR table structure restored (but empty)")
print(" All IPs remain standalone. You would need to manually recreate")
print(" CIDR records and associate IPs with them.\n")

View File

@@ -0,0 +1,53 @@
"""Add config_id to alert_rules table
Revision ID: 010
Revises: 009
Create Date: 2025-11-19
This migration adds config_id foreign key to alert_rules table to replace
the config_file column, completing the migration from file-based to
database-based configurations.
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic
revision = '010'
down_revision = '009'
branch_labels = None
depends_on = None
def upgrade():
"""
Add config_id to alert_rules table and remove config_file.
"""
with op.batch_alter_table('alert_rules', schema=None) as batch_op:
# Add config_id column with foreign key
batch_op.add_column(sa.Column('config_id', sa.Integer(), nullable=True, comment='FK to scan_configs table'))
batch_op.create_index('ix_alert_rules_config_id', ['config_id'], unique=False)
batch_op.create_foreign_key('fk_alert_rules_config_id', 'scan_configs', ['config_id'], ['id'])
# Remove the old config_file column
batch_op.drop_column('config_file')
print("✓ Migration complete: AlertRule now uses config_id")
print(" - Added config_id foreign key to alert_rules table")
print(" - Removed deprecated config_file column")
def downgrade():
"""Remove config_id and restore config_file on alert_rules."""
with op.batch_alter_table('alert_rules', schema=None) as batch_op:
# Remove foreign key and config_id column
batch_op.drop_constraint('fk_alert_rules_config_id', type_='foreignkey')
batch_op.drop_index('ix_alert_rules_config_id')
batch_op.drop_column('config_id')
# Restore config_file column
batch_op.add_column(sa.Column('config_file', sa.String(255), nullable=True, comment='Optional: specific config file this rule applies to'))
print("✓ Downgrade complete: AlertRule config_id removed, config_file restored")