diff --git a/app/migrations/versions/008_expand_cidrs_to_ips.py b/app/migrations/versions/008_expand_cidrs_to_ips.py new file mode 100644 index 0000000..274466c --- /dev/null +++ b/app/migrations/versions/008_expand_cidrs_to_ips.py @@ -0,0 +1,270 @@ +"""Expand CIDRs to individual IPs with per-IP settings + +Revision ID: 008 +Revises: 007 +Create Date: 2025-11-19 + +This migration changes the site architecture to automatically expand CIDRs into +individual IPs in the database. Each IP has its own port and ping settings. + +Changes: +- Add site_id to site_ips (direct link to sites, support standalone IPs) +- Make site_cidr_id nullable (IPs can exist without a CIDR parent) +- Remove settings from site_cidrs (settings now only at IP level) +- Add unique constraint: no duplicate IPs within a site +- Expand existing CIDRs to individual IPs +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy import text +import ipaddress + + +# revision identifiers, used by Alembic +revision = '008' +down_revision = '007' +branch_labels = None +depends_on = None + + +def upgrade(): + """ + Modify schema to support per-IP settings and auto-expand CIDRs. + """ + + connection = op.get_bind() + + # Check if site_id column already exists + inspector = sa.inspect(connection) + site_ips_columns = [col['name'] for col in inspector.get_columns('site_ips')] + site_cidrs_columns = [col['name'] for col in inspector.get_columns('site_cidrs')] + + # Step 1: Add site_id column to site_ips (will be populated from site_cidr_id) + if 'site_id' not in site_ips_columns: + print("Adding site_id column to site_ips...") + op.add_column('site_ips', sa.Column('site_id', sa.Integer(), nullable=True, comment='FK to sites (direct link)')) + else: + print("site_id column already exists in site_ips, skipping...") + + # Step 2: Populate site_id from site_cidr_id (before we make it nullable) + print("Populating site_id from existing site_cidr relationships...") + connection.execute(text(""" + UPDATE site_ips + SET site_id = ( + SELECT site_id + FROM site_cidrs + WHERE site_cidrs.id = site_ips.site_cidr_id + ) + WHERE site_cidr_id IS NOT NULL + """)) + + # Step 3: Make site_id NOT NULL and add foreign key + # Check if foreign key exists before creating + try: + op.alter_column('site_ips', 'site_id', nullable=False) + print("Made site_id NOT NULL") + except Exception as e: + print(f"site_id already NOT NULL or error: {e}") + + # Check if foreign key exists + try: + op.create_foreign_key('fk_site_ips_site_id', 'site_ips', 'sites', ['site_id'], ['id']) + print("Created foreign key fk_site_ips_site_id") + except Exception as e: + print(f"Foreign key already exists or error: {e}") + + # Check if index exists + try: + op.create_index(op.f('ix_site_ips_site_id'), 'site_ips', ['site_id'], unique=False) + print("Created index ix_site_ips_site_id") + except Exception as e: + print(f"Index already exists or error: {e}") + + # Step 4: Make site_cidr_id nullable (for standalone IPs) + try: + op.alter_column('site_ips', 'site_cidr_id', nullable=True) + print("Made site_cidr_id nullable") + except Exception as e: + print(f"site_cidr_id already nullable or error: {e}") + + # Step 5: Drop old unique constraint and create new one (site_id, ip_address) + # This prevents duplicate IPs within a site (across all CIDRs and standalone) + try: + op.drop_constraint('uix_site_cidr_ip', 'site_ips', type_='unique') + print("Dropped old constraint uix_site_cidr_ip") + except Exception as e: + print(f"Constraint already dropped or doesn't exist: {e}") + + try: + op.create_unique_constraint('uix_site_ip_address', 'site_ips', ['site_id', 'ip_address']) + print("Created new constraint uix_site_ip_address") + except Exception as e: + print(f"Constraint already exists or error: {e}") + + # Step 6: Expand existing CIDRs to individual IPs + print("Expanding existing CIDRs to individual IPs...") + + # Get all existing CIDRs + cidrs = connection.execute(text(""" + SELECT id, site_id, cidr, expected_ping, expected_tcp_ports, expected_udp_ports + FROM site_cidrs + """)).fetchall() + + expanded_count = 0 + skipped_count = 0 + + for cidr_row in cidrs: + cidr_id, site_id, cidr_str, expected_ping, expected_tcp_ports, expected_udp_ports = cidr_row + + try: + # Parse CIDR + network = ipaddress.ip_network(cidr_str, strict=False) + + # Check size - skip if too large (> /24 for IPv4, > /64 for IPv6) + if isinstance(network, ipaddress.IPv4Network) and network.prefixlen < 24: + print(f" ⚠ Skipping large CIDR {cidr_str} (>{network.num_addresses} IPs)") + skipped_count += 1 + continue + elif isinstance(network, ipaddress.IPv6Network) and network.prefixlen < 64: + print(f" ⚠ Skipping large CIDR {cidr_str} (>{network.num_addresses} IPs)") + skipped_count += 1 + continue + + # Expand to individual IPs + for ip in network.hosts() if network.num_addresses > 2 else [network.network_address]: + ip_str = str(ip) + + # Check if this IP already exists (from previous IP overrides) + existing = connection.execute(text(""" + SELECT id FROM site_ips + WHERE site_cidr_id = :cidr_id AND ip_address = :ip_address + """), {'cidr_id': cidr_id, 'ip_address': ip_str}).fetchone() + + if not existing: + # Insert new IP with settings from CIDR + connection.execute(text(""" + INSERT INTO site_ips ( + site_id, site_cidr_id, ip_address, + expected_ping, expected_tcp_ports, expected_udp_ports, + created_at + ) + VALUES ( + :site_id, :cidr_id, :ip_address, + :expected_ping, :expected_tcp_ports, :expected_udp_ports, + datetime('now') + ) + """), { + 'site_id': site_id, + 'cidr_id': cidr_id, + 'ip_address': ip_str, + 'expected_ping': expected_ping, + 'expected_tcp_ports': expected_tcp_ports, + 'expected_udp_ports': expected_udp_ports + }) + expanded_count += 1 + + except Exception as e: + print(f" ✗ Error expanding CIDR {cidr_str}: {e}") + skipped_count += 1 + continue + + print(f" ✓ Expanded {expanded_count} IPs from CIDRs") + if skipped_count > 0: + print(f" ⚠ Skipped {skipped_count} CIDRs (too large or errors)") + + # Step 7: Remove settings columns from site_cidrs (now only at IP level) + print("Removing settings columns from site_cidrs...") + # Re-inspect to get current columns + site_cidrs_columns = [col['name'] for col in inspector.get_columns('site_cidrs')] + + if 'expected_ping' in site_cidrs_columns: + try: + op.drop_column('site_cidrs', 'expected_ping') + print("Dropped expected_ping from site_cidrs") + except Exception as e: + print(f"Error dropping expected_ping: {e}") + else: + print("expected_ping already dropped from site_cidrs") + + if 'expected_tcp_ports' in site_cidrs_columns: + try: + op.drop_column('site_cidrs', 'expected_tcp_ports') + print("Dropped expected_tcp_ports from site_cidrs") + except Exception as e: + print(f"Error dropping expected_tcp_ports: {e}") + else: + print("expected_tcp_ports already dropped from site_cidrs") + + if 'expected_udp_ports' in site_cidrs_columns: + try: + op.drop_column('site_cidrs', 'expected_udp_ports') + print("Dropped expected_udp_ports from site_cidrs") + except Exception as e: + print(f"Error dropping expected_udp_ports: {e}") + else: + print("expected_udp_ports already dropped from site_cidrs") + + # Print summary + total_sites = connection.execute(text('SELECT COUNT(*) FROM sites')).scalar() + total_cidrs = connection.execute(text('SELECT COUNT(*) FROM site_cidrs')).scalar() + total_ips = connection.execute(text('SELECT COUNT(*) FROM site_ips')).scalar() + + print("\n✓ Migration 008 complete: CIDRs expanded to individual IPs") + print(f" - Total sites: {total_sites}") + print(f" - Total CIDRs: {total_cidrs}") + print(f" - Total IPs: {total_ips}") + + +def downgrade(): + """ + Revert schema changes (restore CIDR-level settings). + Note: This will lose per-IP granularity! + """ + + connection = op.get_bind() + + print("Rolling back to CIDR-level settings...") + + # Step 1: Add settings columns back to site_cidrs + op.add_column('site_cidrs', sa.Column('expected_ping', sa.Boolean(), nullable=True)) + op.add_column('site_cidrs', sa.Column('expected_tcp_ports', sa.Text(), nullable=True)) + op.add_column('site_cidrs', sa.Column('expected_udp_ports', sa.Text(), nullable=True)) + + # Step 2: Populate CIDR settings from first IP in each CIDR (approximation) + connection.execute(text(""" + UPDATE site_cidrs + SET + expected_ping = ( + SELECT expected_ping FROM site_ips + WHERE site_ips.site_cidr_id = site_cidrs.id + LIMIT 1 + ), + expected_tcp_ports = ( + SELECT expected_tcp_ports FROM site_ips + WHERE site_ips.site_cidr_id = site_cidrs.id + LIMIT 1 + ), + expected_udp_ports = ( + SELECT expected_udp_ports FROM site_ips + WHERE site_ips.site_cidr_id = site_cidrs.id + LIMIT 1 + ) + """)) + + # Step 3: Delete auto-expanded IPs (keep only original overrides) + # In practice, this is difficult to determine, so we'll keep all IPs + # and just remove the schema changes + + # Step 4: Drop new unique constraint and restore old one + op.drop_constraint('uix_site_ip_address', 'site_ips', type_='unique') + op.create_unique_constraint('uix_site_cidr_ip', 'site_ips', ['site_cidr_id', 'ip_address']) + + # Step 5: Make site_cidr_id NOT NULL again + op.alter_column('site_ips', 'site_cidr_id', nullable=False) + + # Step 6: Drop site_id column and related constraints + op.drop_index(op.f('ix_site_ips_site_id'), table_name='site_ips') + op.drop_constraint('fk_site_ips_site_id', 'site_ips', type_='foreignkey') + op.drop_column('site_ips', 'site_id') + + print("✓ Downgrade complete: Reverted to CIDR-level settings") diff --git a/app/migrations/versions/009_remove_cidrs.py b/app/migrations/versions/009_remove_cidrs.py new file mode 100644 index 0000000..3f2bef3 --- /dev/null +++ b/app/migrations/versions/009_remove_cidrs.py @@ -0,0 +1,210 @@ +"""Remove CIDR table - make sites IP-only + +Revision ID: 009 +Revises: 008 +Create Date: 2025-11-19 + +This migration removes the SiteCIDR table entirely, making sites purely +IP-based. CIDRs are now only used as a convenience for bulk IP addition, +not stored as permanent entities. + +Changes: +- Set all site_ips.site_cidr_id to NULL (preserve all IPs) +- Drop foreign key from site_ips to site_cidrs +- Drop site_cidrs table +- Remove site_cidr_id column from site_ips + +All existing IPs are preserved. They become "standalone" IPs without +a CIDR parent. +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy import text + + +# revision identifiers, used by Alembic +revision = '009' +down_revision = '008' +branch_labels = None +depends_on = None + + +def upgrade(): + """ + Remove CIDR table and make all IPs standalone. + """ + + connection = op.get_bind() + inspector = sa.inspect(connection) + + print("\n=== Migration 009: Remove CIDR Table ===\n") + + # Get counts before migration + try: + total_cidrs = connection.execute(text('SELECT COUNT(*) FROM site_cidrs')).scalar() + total_ips = connection.execute(text('SELECT COUNT(*) FROM site_ips')).scalar() + ips_with_cidr = connection.execute(text( + 'SELECT COUNT(*) FROM site_ips WHERE site_cidr_id IS NOT NULL' + )).scalar() + + print(f"Before migration:") + print(f" - Total CIDRs: {total_cidrs}") + print(f" - Total IPs: {total_ips}") + print(f" - IPs linked to CIDRs: {ips_with_cidr}") + print(f" - Standalone IPs: {total_ips - ips_with_cidr}\n") + except Exception as e: + print(f"Could not get pre-migration stats: {e}\n") + + # Step 1: Set all site_cidr_id to NULL (preserve all IPs as standalone) + print("Step 1: Converting all IPs to standalone (nulling CIDR associations)...") + try: + result = connection.execute(text(""" + UPDATE site_ips + SET site_cidr_id = NULL + WHERE site_cidr_id IS NOT NULL + """)) + print(f" ✓ Converted {result.rowcount} IPs to standalone\n") + except Exception as e: + print(f" ⚠ Error or already done: {e}\n") + + # Step 2: Drop foreign key constraint from site_ips to site_cidrs + print("Step 2: Dropping foreign key constraint from site_ips to site_cidrs...") + foreign_keys = inspector.get_foreign_keys('site_ips') + fk_to_drop = None + + for fk in foreign_keys: + if fk['referred_table'] == 'site_cidrs': + fk_to_drop = fk['name'] + break + + if fk_to_drop: + try: + op.drop_constraint(fk_to_drop, 'site_ips', type_='foreignkey') + print(f" ✓ Dropped foreign key constraint: {fk_to_drop}\n") + except Exception as e: + print(f" ⚠ Could not drop foreign key: {e}\n") + else: + print(" ⚠ Foreign key constraint not found or already dropped\n") + + # Step 3: Drop index on site_cidr_id (if exists) + print("Step 3: Dropping index on site_cidr_id...") + indexes = inspector.get_indexes('site_ips') + index_to_drop = None + + for idx in indexes: + if 'site_cidr_id' in idx['column_names']: + index_to_drop = idx['name'] + break + + if index_to_drop: + try: + op.drop_index(index_to_drop, table_name='site_ips') + print(f" ✓ Dropped index: {index_to_drop}\n") + except Exception as e: + print(f" ⚠ Could not drop index: {e}\n") + else: + print(" ⚠ Index not found or already dropped\n") + + # Step 4: Drop site_cidrs table + print("Step 4: Dropping site_cidrs table...") + tables = inspector.get_table_names() + + if 'site_cidrs' in tables: + try: + op.drop_table('site_cidrs') + print(" ✓ Dropped site_cidrs table\n") + except Exception as e: + print(f" ⚠ Could not drop table: {e}\n") + else: + print(" ⚠ Table site_cidrs not found or already dropped\n") + + # Step 5: Drop site_cidr_id column from site_ips + print("Step 5: Dropping site_cidr_id column from site_ips...") + site_ips_columns = [col['name'] for col in inspector.get_columns('site_ips')] + + if 'site_cidr_id' in site_ips_columns: + try: + op.drop_column('site_ips', 'site_cidr_id') + print(" ✓ Dropped site_cidr_id column from site_ips\n") + except Exception as e: + print(f" ⚠ Could not drop column: {e}\n") + else: + print(" ⚠ Column site_cidr_id not found or already dropped\n") + + # Get counts after migration + try: + final_ips = connection.execute(text('SELECT COUNT(*) FROM site_ips')).scalar() + total_sites = connection.execute(text('SELECT COUNT(*) FROM sites')).scalar() + + print("After migration:") + print(f" - Total sites: {total_sites}") + print(f" - Total IPs (all standalone): {final_ips}") + print(f" - CIDRs: N/A (table removed)") + except Exception as e: + print(f"Could not get post-migration stats: {e}") + + print("\n✓ Migration 009 complete: Sites are now IP-only") + print(" All IPs preserved as standalone. CIDRs can still be used") + print(" via the API/UI for bulk IP creation, but are not stored.\n") + + +def downgrade(): + """ + Recreate site_cidrs table (CANNOT restore original CIDR associations). + + WARNING: This downgrade creates an empty site_cidrs table structure but + cannot restore the original CIDR-to-IP associations since that data was + deleted. All IPs will remain standalone. + """ + + connection = op.get_bind() + + print("\n=== Downgrade 009: Recreate CIDR Table Structure ===\n") + print("⚠ WARNING: Cannot restore original CIDR associations!") + print(" The site_cidrs table structure will be recreated but will be empty.") + print(" All IPs will remain standalone. This is a PARTIAL downgrade.\n") + + # Step 1: Recreate site_cidrs table (empty) + print("Step 1: Recreating site_cidrs table structure...") + try: + op.create_table( + 'site_cidrs', + sa.Column('id', sa.Integer(), autoincrement=True, nullable=False), + sa.Column('site_id', sa.Integer(), nullable=False), + sa.Column('cidr', sa.String(length=45), nullable=False, comment='CIDR notation (e.g., 10.0.0.0/24)'), + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.PrimaryKeyConstraint('id'), + sa.ForeignKeyConstraint(['site_id'], ['sites.id'], ), + sa.UniqueConstraint('site_id', 'cidr', name='uix_site_cidr') + ) + print(" ✓ Recreated site_cidrs table (empty)\n") + except Exception as e: + print(f" ⚠ Could not create table: {e}\n") + + # Step 2: Add site_cidr_id column back to site_ips (nullable) + print("Step 2: Adding site_cidr_id column back to site_ips...") + try: + op.add_column('site_ips', sa.Column('site_cidr_id', sa.Integer(), nullable=True, comment='FK to site_cidrs (optional, for grouping)')) + print(" ✓ Added site_cidr_id column (nullable)\n") + except Exception as e: + print(f" ⚠ Could not add column: {e}\n") + + # Step 3: Add foreign key constraint + print("Step 3: Adding foreign key constraint...") + try: + op.create_foreign_key('fk_site_ips_site_cidr_id', 'site_ips', 'site_cidrs', ['site_cidr_id'], ['id']) + print(" ✓ Created foreign key constraint\n") + except Exception as e: + print(f" ⚠ Could not create foreign key: {e}\n") + + # Step 4: Add index on site_cidr_id + print("Step 4: Adding index on site_cidr_id...") + try: + op.create_index('ix_site_ips_site_cidr_id', 'site_ips', ['site_cidr_id'], unique=False) + print(" ✓ Created index on site_cidr_id\n") + except Exception as e: + print(f" ⚠ Could not create index: {e}\n") + + print("✓ Downgrade complete: CIDR table structure restored (but empty)") + print(" All IPs remain standalone. You would need to manually recreate") + print(" CIDR records and associate IPs with them.\n") diff --git a/app/migrations/versions/010_alert_rules_config_id.py b/app/migrations/versions/010_alert_rules_config_id.py new file mode 100644 index 0000000..6f83e4a --- /dev/null +++ b/app/migrations/versions/010_alert_rules_config_id.py @@ -0,0 +1,53 @@ +"""Add config_id to alert_rules table + +Revision ID: 010 +Revises: 009 +Create Date: 2025-11-19 + +This migration adds config_id foreign key to alert_rules table to replace +the config_file column, completing the migration from file-based to +database-based configurations. +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic +revision = '010' +down_revision = '009' +branch_labels = None +depends_on = None + + +def upgrade(): + """ + Add config_id to alert_rules table and remove config_file. + """ + + with op.batch_alter_table('alert_rules', schema=None) as batch_op: + # Add config_id column with foreign key + batch_op.add_column(sa.Column('config_id', sa.Integer(), nullable=True, comment='FK to scan_configs table')) + batch_op.create_index('ix_alert_rules_config_id', ['config_id'], unique=False) + batch_op.create_foreign_key('fk_alert_rules_config_id', 'scan_configs', ['config_id'], ['id']) + + # Remove the old config_file column + batch_op.drop_column('config_file') + + print("✓ Migration complete: AlertRule now uses config_id") + print(" - Added config_id foreign key to alert_rules table") + print(" - Removed deprecated config_file column") + + +def downgrade(): + """Remove config_id and restore config_file on alert_rules.""" + + with op.batch_alter_table('alert_rules', schema=None) as batch_op: + # Remove foreign key and config_id column + batch_op.drop_constraint('fk_alert_rules_config_id', type_='foreignkey') + batch_op.drop_index('ix_alert_rules_config_id') + batch_op.drop_column('config_id') + + # Restore config_file column + batch_op.add_column(sa.Column('config_file', sa.String(255), nullable=True, comment='Optional: specific config file this rule applies to')) + + print("✓ Downgrade complete: AlertRule config_id removed, config_file restored") diff --git a/app/src/scanner.py b/app/src/scanner.py index ee0c6d8..7881783 100644 --- a/app/src/scanner.py +++ b/app/src/scanner.py @@ -261,14 +261,14 @@ class SneakyScanner: """ Load a site definition from the database. + IPs are pre-expanded in the database, so we just load them directly. + Args: site_name: Name of the site to load Returns: - Site definition dict with expanded IPs, or None if not found + Site definition dict with IPs, or None if not found """ - import ipaddress - try: # Import database modules import os @@ -281,7 +281,7 @@ class SneakyScanner: from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker, joinedload - from web.models import Site, SiteCIDR + from web.models import Site # Get database URL from environment database_url = os.environ.get('DATABASE_URL', 'sqlite:///./sneakyscanner.db') @@ -291,12 +291,10 @@ class SneakyScanner: Session = sessionmaker(bind=engine) session = Session() - # Query site with CIDRs and IP overrides + # Query site with all IPs (CIDRs are already expanded) site = ( session.query(Site) - .options( - joinedload(Site.cidrs).joinedload(SiteCIDR.ips) - ) + .options(joinedload(Site.ips)) .filter(Site.name == site_name) .first() ) @@ -305,60 +303,25 @@ class SneakyScanner: session.close() return None - # Expand CIDRs to IP list + # Load all IPs directly from database (already expanded) expanded_ips = [] - for cidr_obj in site.cidrs: - cidr = cidr_obj.cidr - expected_ping = cidr_obj.expected_ping - expected_tcp_ports = json.loads(cidr_obj.expected_tcp_ports) if cidr_obj.expected_tcp_ports else [] - expected_udp_ports = json.loads(cidr_obj.expected_udp_ports) if cidr_obj.expected_udp_ports else [] + for ip_obj in site.ips: + # Get settings from IP (no need to merge with CIDR defaults) + expected_ping = ip_obj.expected_ping if ip_obj.expected_ping is not None else False + expected_tcp_ports = json.loads(ip_obj.expected_tcp_ports) if ip_obj.expected_tcp_ports else [] + expected_udp_ports = json.loads(ip_obj.expected_udp_ports) if ip_obj.expected_udp_ports else [] - # Build IP override map - override_map = {} - for ip_override in cidr_obj.ips: - override_map[ip_override.ip_address] = { - 'expected_ping': ip_override.expected_ping if ip_override.expected_ping is not None else expected_ping, - 'expected_tcp_ports': json.loads(ip_override.expected_tcp_ports) if ip_override.expected_tcp_ports else expected_tcp_ports, - 'expected_udp_ports': json.loads(ip_override.expected_udp_ports) if ip_override.expected_udp_ports else expected_udp_ports + ip_config = { + 'address': ip_obj.ip_address, + 'expected': { + 'ping': expected_ping, + 'tcp_ports': expected_tcp_ports, + 'udp_ports': expected_udp_ports } + } - # Expand CIDR to IP list - try: - network = ipaddress.ip_network(cidr, strict=False) - ip_list = [str(ip) for ip in network.hosts()] - - if not ip_list: - ip_list = [str(network.network_address)] - - for ip_address in ip_list: - # Check if this IP has an override - if ip_address in override_map: - override = override_map[ip_address] - ip_config = { - 'address': ip_address, - 'expected': { - 'ping': override['expected_ping'], - 'tcp_ports': override['expected_tcp_ports'], - 'udp_ports': override['expected_udp_ports'] - } - } - else: - # Use CIDR-level defaults - ip_config = { - 'address': ip_address, - 'expected': { - 'ping': expected_ping if expected_ping is not None else False, - 'tcp_ports': expected_tcp_ports, - 'udp_ports': expected_udp_ports - } - } - - expanded_ips.append(ip_config) - - except ValueError as e: - print(f"WARNING: Invalid CIDR '{cidr}' in site '{site_name}': {e}", file=sys.stderr) - continue + expanded_ips.append(ip_config) session.close() diff --git a/app/web/api/alerts.py b/app/web/api/alerts.py index 2ad23eb..8315dae 100644 --- a/app/web/api/alerts.py +++ b/app/web/api/alerts.py @@ -169,7 +169,8 @@ def list_alert_rules(): 'webhook_enabled': rule.webhook_enabled, 'severity': rule.severity, 'filter_conditions': json.loads(rule.filter_conditions) if rule.filter_conditions else None, - 'config_file': rule.config_file, + 'config_id': rule.config_id, + 'config_title': rule.config.title if rule.config else None, 'created_at': rule.created_at.isoformat(), 'updated_at': rule.updated_at.isoformat() if rule.updated_at else None }) @@ -195,7 +196,7 @@ def create_alert_rule(): webhook_enabled: Send webhook for this rule (default: false) severity: Alert severity (critical, warning, info) filter_conditions: JSON object with filter conditions - config_file: Optional config file to apply rule to + config_id: Optional config ID to apply rule to Returns: JSON response with created rule @@ -226,6 +227,17 @@ def create_alert_rule(): }), 400 try: + # Validate config_id if provided + config_id = data.get('config_id') + if config_id: + from web.models import ScanConfig + config = current_app.db_session.query(ScanConfig).filter_by(id=config_id).first() + if not config: + return jsonify({ + 'status': 'error', + 'message': f'Config with ID {config_id} not found' + }), 400 + # Create new rule rule = AlertRule( name=data.get('name', f"{data['rule_type']} rule"), @@ -236,7 +248,7 @@ def create_alert_rule(): webhook_enabled=data.get('webhook_enabled', False), severity=data.get('severity', 'warning'), filter_conditions=json.dumps(data['filter_conditions']) if data.get('filter_conditions') else None, - config_file=data.get('config_file'), + config_id=config_id, created_at=datetime.now(timezone.utc), updated_at=datetime.now(timezone.utc) ) @@ -257,7 +269,8 @@ def create_alert_rule(): 'webhook_enabled': rule.webhook_enabled, 'severity': rule.severity, 'filter_conditions': json.loads(rule.filter_conditions) if rule.filter_conditions else None, - 'config_file': rule.config_file, + 'config_id': rule.config_id, + 'config_title': rule.config.title if rule.config else None, 'created_at': rule.created_at.isoformat(), 'updated_at': rule.updated_at.isoformat() } @@ -288,7 +301,7 @@ def update_alert_rule(rule_id): webhook_enabled: Send webhook for this rule (optional) severity: Alert severity (optional) filter_conditions: JSON object with filter conditions (optional) - config_file: Config file to apply rule to (optional) + config_id: Config ID to apply rule to (optional) Returns: JSON response with update status @@ -312,6 +325,18 @@ def update_alert_rule(rule_id): }), 400 try: + # Validate config_id if provided + if 'config_id' in data: + config_id = data['config_id'] + if config_id: + from web.models import ScanConfig + config = current_app.db_session.query(ScanConfig).filter_by(id=config_id).first() + if not config: + return jsonify({ + 'status': 'error', + 'message': f'Config with ID {config_id} not found' + }), 400 + # Update fields if provided if 'name' in data: rule.name = data['name'] @@ -327,8 +352,8 @@ def update_alert_rule(rule_id): rule.severity = data['severity'] if 'filter_conditions' in data: rule.filter_conditions = json.dumps(data['filter_conditions']) if data['filter_conditions'] else None - if 'config_file' in data: - rule.config_file = data['config_file'] + if 'config_id' in data: + rule.config_id = data['config_id'] rule.updated_at = datetime.now(timezone.utc) @@ -347,7 +372,8 @@ def update_alert_rule(rule_id): 'webhook_enabled': rule.webhook_enabled, 'severity': rule.severity, 'filter_conditions': json.loads(rule.filter_conditions) if rule.filter_conditions else None, - 'config_file': rule.config_file, + 'config_id': rule.config_id, + 'config_title': rule.config.title if rule.config else None, 'created_at': rule.created_at.isoformat(), 'updated_at': rule.updated_at.isoformat() } diff --git a/app/web/api/configs.py b/app/web/api/configs.py index a8c49d0..f3346e3 100644 --- a/app/web/api/configs.py +++ b/app/web/api/configs.py @@ -174,7 +174,7 @@ def get_config(config_id: int): "id": 1, "name": "Production DC", "description": "...", - "cidr_count": 5 + "ip_count": 5 } ], "created_at": "2025-11-19T10:30:00Z", diff --git a/app/web/api/scans.py b/app/web/api/scans.py index afde8d5..b2095d8 100644 --- a/app/web/api/scans.py +++ b/app/web/api/scans.py @@ -129,7 +129,7 @@ def trigger_scan(): Trigger a new scan. Request body: - config_file: Path to YAML config file + config_id: Database config ID (required) Returns: JSON response with scan_id and status @@ -137,25 +137,35 @@ def trigger_scan(): try: # Get request data data = request.get_json() or {} - config_file = data.get('config_file') + config_id = data.get('config_id') # Validate required fields - if not config_file: - logger.warning("Scan trigger request missing config_file") + if not config_id: + logger.warning("Scan trigger request missing config_id") return jsonify({ 'error': 'Invalid request', - 'message': 'config_file is required' + 'message': 'config_id is required' + }), 400 + + # Validate config_id is an integer + try: + config_id = int(config_id) + except (TypeError, ValueError): + logger.warning(f"Invalid config_id type: {config_id}") + return jsonify({ + 'error': 'Invalid request', + 'message': 'config_id must be an integer' }), 400 # Trigger scan via service scan_service = ScanService(current_app.db_session) scan_id = scan_service.trigger_scan( - config_file=config_file, + config_id=config_id, triggered_by='api', scheduler=current_app.scheduler ) - logger.info(f"Scan {scan_id} triggered via API: config={config_file}") + logger.info(f"Scan {scan_id} triggered via API: config_id={config_id}") return jsonify({ 'scan_id': scan_id, @@ -164,10 +174,10 @@ def trigger_scan(): }), 201 except ValueError as e: - # Config file validation error + # Config validation error error_message = str(e) - logger.warning(f"Invalid config file: {error_message}") - logger.warning(f"Request data: config_file='{config_file}'") + logger.warning(f"Invalid config: {error_message}") + logger.warning(f"Request data: config_id='{config_id}'") return jsonify({ 'error': 'Invalid request', 'message': error_message diff --git a/app/web/api/sites.py b/app/web/api/sites.py index d5a4e01..6440c3e 100644 --- a/app/web/api/sites.py +++ b/app/web/api/sites.py @@ -158,21 +158,12 @@ def create_site(): }), 400 description = data.get('description') - cidrs = data.get('cidrs', []) - # Validate cidrs is a list - if not isinstance(cidrs, list): - return jsonify({ - 'error': 'Invalid request', - 'message': 'cidrs must be a list' - }), 400 - - # Create site + # Create site (empty initially) site_service = SiteService(current_app.db_session) site = site_service.create_site( name=name, - description=description, - cidrs=cidrs if cidrs else None + description=description ) logger.info(f"Created site '{name}' (id={site['id']})") @@ -294,135 +285,178 @@ def delete_site(site_id): }), 500 -@bp.route('//cidrs', methods=['POST']) +@bp.route('//ips/bulk', methods=['POST']) @api_auth_required -def add_cidr(site_id): +def bulk_add_ips(site_id): """ - Add a CIDR range to a site. + Bulk add IPs to a site from CIDR or list. Args: site_id: Site ID Request body: - cidr: CIDR notation (required, e.g., "10.0.0.0/24") - expected_ping: Expected ping response (optional) - expected_tcp_ports: List of expected TCP ports (optional) - expected_udp_ports: List of expected UDP ports (optional) + source_type: "cidr" or "list" (required) + cidr: CIDR notation if source_type="cidr" (e.g., "10.0.0.0/24") + ips: List of IP addresses if source_type="list" (e.g., ["10.0.0.1", "10.0.0.2"]) + expected_ping: Expected ping response for all IPs (optional) + expected_tcp_ports: List of expected TCP ports for all IPs (optional) + expected_udp_ports: List of expected UDP ports for all IPs (optional) Returns: - JSON response with created CIDR data + JSON response with count of IPs added and any errors """ try: data = request.get_json() or {} - # Validate required fields - cidr = data.get('cidr') - if not cidr: - logger.warning("CIDR creation request missing cidr") + source_type = data.get('source_type') + if source_type not in ['cidr', 'list']: return jsonify({ 'error': 'Invalid request', - 'message': 'cidr is required' + 'message': 'source_type must be "cidr" or "list"' }), 400 expected_ping = data.get('expected_ping') expected_tcp_ports = data.get('expected_tcp_ports', []) expected_udp_ports = data.get('expected_udp_ports', []) - # Add CIDR site_service = SiteService(current_app.db_session) - cidr_data = site_service.add_cidr( - site_id=site_id, - cidr=cidr, - expected_ping=expected_ping, - expected_tcp_ports=expected_tcp_ports, - expected_udp_ports=expected_udp_ports - ) - logger.info(f"Added CIDR '{cidr}' to site {site_id}") - return jsonify(cidr_data), 201 + if source_type == 'cidr': + cidr = data.get('cidr') + if not cidr: + return jsonify({ + 'error': 'Invalid request', + 'message': 'cidr is required when source_type="cidr"' + }), 400 + + result = site_service.bulk_add_ips_from_cidr( + site_id=site_id, + cidr=cidr, + expected_ping=expected_ping, + expected_tcp_ports=expected_tcp_ports, + expected_udp_ports=expected_udp_ports + ) + + logger.info(f"Bulk added {result['ip_count']} IPs from CIDR '{cidr}' to site {site_id}") + return jsonify(result), 201 + + else: # source_type == 'list' + ip_list = data.get('ips', []) + if not isinstance(ip_list, list): + return jsonify({ + 'error': 'Invalid request', + 'message': 'ips must be a list when source_type="list"' + }), 400 + + result = site_service.bulk_add_ips_from_list( + site_id=site_id, + ip_list=ip_list, + expected_ping=expected_ping, + expected_tcp_ports=expected_tcp_ports, + expected_udp_ports=expected_udp_ports + ) + + logger.info(f"Bulk added {result['ip_count']} IPs from list to site {site_id}") + return jsonify(result), 201 except ValueError as e: - logger.warning(f"Invalid CIDR creation request: {str(e)}") + logger.warning(f"Invalid bulk IP request: {str(e)}") return jsonify({ 'error': 'Invalid request', 'message': str(e) }), 400 except SQLAlchemyError as e: - logger.error(f"Database error adding CIDR to site {site_id}: {str(e)}") + logger.error(f"Database error bulk adding IPs to site {site_id}: {str(e)}") return jsonify({ 'error': 'Database error', - 'message': 'Failed to add CIDR' + 'message': 'Failed to add IPs' }), 500 except Exception as e: - logger.error(f"Unexpected error adding CIDR to site {site_id}: {str(e)}", exc_info=True) + logger.error(f"Unexpected error bulk adding IPs to site {site_id}: {str(e)}", exc_info=True) return jsonify({ 'error': 'Internal server error', 'message': 'An unexpected error occurred' }), 500 -@bp.route('//cidrs/', methods=['DELETE']) +@bp.route('//ips', methods=['GET']) @api_auth_required -def remove_cidr(site_id, cidr_id): +def list_ips(site_id): """ - Remove a CIDR range from a site. + List IPs in a site with pagination. - Prevents removal if it's the last CIDR. - - Args: - site_id: Site ID - cidr_id: CIDR ID + Query params: + page: Page number (default: 1) + per_page: Items per page (default: 50, max: 200) Returns: - JSON response with success message + JSON response with IPs list and pagination info """ try: - site_service = SiteService(current_app.db_session) - site_service.remove_cidr(site_id, cidr_id) + # Get and validate query parameters + page = request.args.get('page', 1, type=int) + per_page = request.args.get('per_page', 50, type=int) + + # Validate pagination params + page, per_page = validate_page_params(page, per_page, max_per_page=200) + + # Get IPs from service + site_service = SiteService(current_app.db_session) + paginated_result = site_service.list_ips( + site_id=site_id, + page=page, + per_page=per_page + ) + + logger.info(f"Listed IPs for site {site_id}: page={page}, per_page={per_page}, total={paginated_result.total}") - logger.info(f"Removed CIDR {cidr_id} from site {site_id}") return jsonify({ - 'message': f'CIDR {cidr_id} removed successfully' + 'ips': paginated_result.items, + 'total': paginated_result.total, + 'page': paginated_result.page, + 'per_page': paginated_result.per_page, + 'total_pages': paginated_result.pages, + 'has_prev': paginated_result.has_prev, + 'has_next': paginated_result.has_next }) except ValueError as e: - logger.warning(f"Cannot remove CIDR {cidr_id} from site {site_id}: {str(e)}") + logger.warning(f"Invalid request parameters: {str(e)}") return jsonify({ 'error': 'Invalid request', 'message': str(e) }), 400 except SQLAlchemyError as e: - logger.error(f"Database error removing CIDR {cidr_id} from site {site_id}: {str(e)}") + logger.error(f"Database error listing IPs for site {site_id}: {str(e)}") return jsonify({ 'error': 'Database error', - 'message': 'Failed to remove CIDR' + 'message': 'Failed to retrieve IPs' }), 500 except Exception as e: - logger.error(f"Unexpected error removing CIDR {cidr_id} from site {site_id}: {str(e)}", exc_info=True) + logger.error(f"Unexpected error listing IPs for site {site_id}: {str(e)}", exc_info=True) return jsonify({ 'error': 'Internal server error', 'message': 'An unexpected error occurred' }), 500 -@bp.route('//cidrs//ips', methods=['POST']) +@bp.route('//ips', methods=['POST']) @api_auth_required -def add_ip_override(site_id, cidr_id): +def add_standalone_ip(site_id): """ - Add an IP-level expectation override within a CIDR. + Add a standalone IP (without CIDR parent) to a site. Args: - site_id: Site ID (for validation) - cidr_id: CIDR ID + site_id: Site ID Request body: ip_address: IP address (required) - expected_ping: Override ping expectation (optional) - expected_tcp_ports: Override TCP ports expectation (optional) - expected_udp_ports: Override UDP ports expectation (optional) + expected_ping: Expected ping response (optional) + expected_tcp_ports: List of expected TCP ports (optional) + expected_udp_ports: List of expected UDP ports (optional) Returns: - JSON response with created IP override data + JSON response with created IP data """ try: data = request.get_json() or {} @@ -430,7 +464,7 @@ def add_ip_override(site_id, cidr_id): # Validate required fields ip_address = data.get('ip_address') if not ip_address: - logger.warning("IP override creation request missing ip_address") + logger.warning("Standalone IP creation request missing ip_address") return jsonify({ 'error': 'Invalid request', 'message': 'ip_address is required' @@ -440,76 +474,133 @@ def add_ip_override(site_id, cidr_id): expected_tcp_ports = data.get('expected_tcp_ports', []) expected_udp_ports = data.get('expected_udp_ports', []) - # Add IP override + # Add standalone IP site_service = SiteService(current_app.db_session) - ip_data = site_service.add_ip_override( - cidr_id=cidr_id, + ip_data = site_service.add_standalone_ip( + site_id=site_id, ip_address=ip_address, expected_ping=expected_ping, expected_tcp_ports=expected_tcp_ports, expected_udp_ports=expected_udp_ports ) - logger.info(f"Added IP override '{ip_address}' to CIDR {cidr_id} in site {site_id}") + logger.info(f"Added standalone IP '{ip_address}' to site {site_id}") return jsonify(ip_data), 201 except ValueError as e: - logger.warning(f"Invalid IP override creation request: {str(e)}") + logger.warning(f"Invalid standalone IP creation request: {str(e)}") return jsonify({ 'error': 'Invalid request', 'message': str(e) }), 400 except SQLAlchemyError as e: - logger.error(f"Database error adding IP override to CIDR {cidr_id}: {str(e)}") + logger.error(f"Database error adding standalone IP to site {site_id}: {str(e)}") return jsonify({ 'error': 'Database error', - 'message': 'Failed to add IP override' + 'message': 'Failed to add IP' }), 500 except Exception as e: - logger.error(f"Unexpected error adding IP override to CIDR {cidr_id}: {str(e)}", exc_info=True) + logger.error(f"Unexpected error adding standalone IP to site {site_id}: {str(e)}", exc_info=True) return jsonify({ 'error': 'Internal server error', 'message': 'An unexpected error occurred' }), 500 -@bp.route('//cidrs//ips/', methods=['DELETE']) +@bp.route('//ips/', methods=['PUT']) @api_auth_required -def remove_ip_override(site_id, cidr_id, ip_id): +def update_ip_settings(site_id, ip_id): """ - Remove an IP-level override. + Update settings for an individual IP. Args: - site_id: Site ID (for validation) - cidr_id: CIDR ID - ip_id: IP override ID + site_id: Site ID + ip_id: IP ID + + Request body: + expected_ping: New ping expectation (optional) + expected_tcp_ports: New TCP ports expectation (optional) + expected_udp_ports: New UDP ports expectation (optional) + + Returns: + JSON response with updated IP data + """ + try: + data = request.get_json() or {} + + expected_ping = data.get('expected_ping') + expected_tcp_ports = data.get('expected_tcp_ports') + expected_udp_ports = data.get('expected_udp_ports') + + # Update IP settings + site_service = SiteService(current_app.db_session) + ip_data = site_service.update_ip_settings( + site_id=site_id, + ip_id=ip_id, + expected_ping=expected_ping, + expected_tcp_ports=expected_tcp_ports, + expected_udp_ports=expected_udp_ports + ) + + logger.info(f"Updated IP {ip_id} in site {site_id}") + return jsonify(ip_data) + + except ValueError as e: + logger.warning(f"Invalid IP update request: {str(e)}") + return jsonify({ + 'error': 'Invalid request', + 'message': str(e) + }), 400 + except SQLAlchemyError as e: + logger.error(f"Database error updating IP {ip_id} in site {site_id}: {str(e)}") + return jsonify({ + 'error': 'Database error', + 'message': 'Failed to update IP' + }), 500 + except Exception as e: + logger.error(f"Unexpected error updating IP {ip_id} in site {site_id}: {str(e)}", exc_info=True) + return jsonify({ + 'error': 'Internal server error', + 'message': 'An unexpected error occurred' + }), 500 + + +@bp.route('//ips/', methods=['DELETE']) +@api_auth_required +def remove_ip(site_id, ip_id): + """ + Remove an IP from a site. + + Args: + site_id: Site ID + ip_id: IP ID Returns: JSON response with success message """ try: site_service = SiteService(current_app.db_session) - site_service.remove_ip_override(cidr_id, ip_id) + site_service.remove_ip(site_id, ip_id) - logger.info(f"Removed IP override {ip_id} from CIDR {cidr_id} in site {site_id}") + logger.info(f"Removed IP {ip_id} from site {site_id}") return jsonify({ - 'message': f'IP override {ip_id} removed successfully' + 'message': f'IP {ip_id} removed successfully' }) except ValueError as e: - logger.warning(f"Cannot remove IP override {ip_id}: {str(e)}") + logger.warning(f"Cannot remove IP {ip_id}: {str(e)}") return jsonify({ 'error': 'Invalid request', 'message': str(e) }), 400 except SQLAlchemyError as e: - logger.error(f"Database error removing IP override {ip_id}: {str(e)}") + logger.error(f"Database error removing IP {ip_id}: {str(e)}") return jsonify({ 'error': 'Database error', - 'message': 'Failed to remove IP override' + 'message': 'Failed to remove IP' }), 500 except Exception as e: - logger.error(f"Unexpected error removing IP override {ip_id}: {str(e)}", exc_info=True) + logger.error(f"Unexpected error removing IP {ip_id}: {str(e)}", exc_info=True) return jsonify({ 'error': 'Internal server error', 'message': 'An unexpected error occurred' diff --git a/app/web/config.py b/app/web/config.py index 11277bc..9906b0c 100644 --- a/app/web/config.py +++ b/app/web/config.py @@ -7,7 +7,7 @@ that are managed by developers, not stored in the database. # Application metadata APP_NAME = 'SneakyScanner' -APP_VERSION = '1.0.0-phase5' +APP_VERSION = '1.0.0-alpha' # Repository URL REPO_URL = 'https://git.sneakygeek.net/sneakygeek/SneakyScan' diff --git a/app/web/models.py b/app/web/models.py index 0805f53..6bd4d99 100644 --- a/app/web/models.py +++ b/app/web/models.py @@ -267,7 +267,7 @@ class Site(Base): updated_at = Column(DateTime, nullable=False, default=datetime.utcnow, onupdate=datetime.utcnow, comment="Last modification time") # Relationships - cidrs = relationship('SiteCIDR', back_populates='site', cascade='all, delete-orphan') + ips = relationship('SiteIP', back_populates='site', cascade='all, delete-orphan') scan_associations = relationship('ScanSiteAssociation', back_populates='site') config_associations = relationship('ScanConfigSite', back_populates='site') @@ -275,59 +275,29 @@ class Site(Base): return f"" -class SiteCIDR(Base): - """ - CIDR ranges associated with a site. - - Each site must have at least one CIDR range. CIDR-level expectations - (ping, ports) apply to all IPs in the range unless overridden at the IP level. - """ - __tablename__ = 'site_cidrs' - - id = Column(Integer, primary_key=True, autoincrement=True) - site_id = Column(Integer, ForeignKey('sites.id'), nullable=False, index=True) - cidr = Column(String(45), nullable=False, comment="CIDR notation (e.g., 10.0.0.0/24)") - expected_ping = Column(Boolean, nullable=True, default=False, comment="Expected ping response for this CIDR") - expected_tcp_ports = Column(Text, nullable=True, comment="JSON array of expected TCP ports") - expected_udp_ports = Column(Text, nullable=True, comment="JSON array of expected UDP ports") - created_at = Column(DateTime, nullable=False, default=datetime.utcnow, comment="CIDR creation time") - - # Relationships - site = relationship('Site', back_populates='cidrs') - ips = relationship('SiteIP', back_populates='cidr', cascade='all, delete-orphan') - - # Index for efficient CIDR lookups within a site - __table_args__ = ( - UniqueConstraint('site_id', 'cidr', name='uix_site_cidr'), - ) - - def __repr__(self): - return f"" - - class SiteIP(Base): """ - IP-level expectation overrides within a CIDR range. + Individual IP addresses with their own settings. - Allows fine-grained control where specific IPs within a CIDR have - different expectations than the CIDR-level defaults. + Each IP is directly associated with a site and has its own port and ping settings. + IPs are standalone entities - CIDRs are only used as a convenience for bulk creation. """ __tablename__ = 'site_ips' id = Column(Integer, primary_key=True, autoincrement=True) - site_cidr_id = Column(Integer, ForeignKey('site_cidrs.id'), nullable=False, index=True) + site_id = Column(Integer, ForeignKey('sites.id'), nullable=False, index=True, comment="FK to sites") ip_address = Column(String(45), nullable=False, comment="IPv4 or IPv6 address") - expected_ping = Column(Boolean, nullable=True, comment="Override ping expectation for this IP") - expected_tcp_ports = Column(Text, nullable=True, comment="JSON array of expected TCP ports (overrides CIDR)") - expected_udp_ports = Column(Text, nullable=True, comment="JSON array of expected UDP ports (overrides CIDR)") - created_at = Column(DateTime, nullable=False, default=datetime.utcnow, comment="IP override creation time") + expected_ping = Column(Boolean, nullable=True, comment="Expected ping response for this IP") + expected_tcp_ports = Column(Text, nullable=True, comment="JSON array of expected TCP ports") + expected_udp_ports = Column(Text, nullable=True, comment="JSON array of expected UDP ports") + created_at = Column(DateTime, nullable=False, default=datetime.utcnow, comment="IP creation time") # Relationships - cidr = relationship('SiteCIDR', back_populates='ips') + site = relationship('Site', back_populates='ips') - # Index for efficient IP lookups + # Index for efficient IP lookups - prevent duplicate IPs within a site __table_args__ = ( - UniqueConstraint('site_cidr_id', 'ip_address', name='uix_site_cidr_ip'), + UniqueConstraint('site_id', 'ip_address', name='uix_site_ip_address'), ) def __repr__(self): @@ -507,12 +477,13 @@ class AlertRule(Base): webhook_enabled = Column(Boolean, nullable=False, default=False, comment="Send webhook for this rule?") severity = Column(String(20), nullable=True, comment="Alert severity: critical, warning, info") filter_conditions = Column(Text, nullable=True, comment="JSON filter conditions for the rule") - config_file = Column(String(255), nullable=True, comment="Optional: specific config file this rule applies to") + config_id = Column(Integer, ForeignKey('scan_configs.id'), nullable=True, index=True, comment="Optional: specific config this rule applies to") created_at = Column(DateTime, nullable=False, default=datetime.utcnow, comment="Rule creation time") updated_at = Column(DateTime, nullable=True, comment="Last update time") # Relationships alerts = relationship("Alert", back_populates="rule", cascade="all, delete-orphan") + config = relationship("ScanConfig", backref="alert_rules") def __repr__(self): return f"" diff --git a/app/web/routes/main.py b/app/web/routes/main.py index c1607c6..99277d5 100644 --- a/app/web/routes/main.py +++ b/app/web/routes/main.py @@ -35,20 +35,7 @@ def dashboard(): Returns: Rendered dashboard template """ - import os - - # Get list of available config files - configs_dir = '/app/configs' - config_files = [] - - try: - if os.path.exists(configs_dir): - config_files = [f for f in os.listdir(configs_dir) if f.endswith(('.yaml', '.yml'))] - config_files.sort() - except Exception as e: - logger.error(f"Error listing config files: {e}") - - return render_template('dashboard.html', config_files=config_files) + return render_template('dashboard.html') @bp.route('/scans') @@ -60,20 +47,7 @@ def scans(): Returns: Rendered scans list template """ - import os - - # Get list of available config files - configs_dir = '/app/configs' - config_files = [] - - try: - if os.path.exists(configs_dir): - config_files = [f for f in os.listdir(configs_dir) if f.endswith(('.yaml', '.yml'))] - config_files.sort() - except Exception as e: - logger.error(f"Error listing config files: {e}") - - return render_template('scans.html', config_files=config_files) + return render_template('scans.html') @bp.route('/scans/') @@ -299,7 +273,6 @@ def alert_rules(): Returns: Rendered alert rules template """ - import os from flask import current_app from web.models import AlertRule @@ -317,19 +290,7 @@ def alert_rules(): if rules is None: rules = [] - # Get list of available config files - configs_dir = '/app/configs' - config_files = [] - - try: - if os.path.exists(configs_dir): - config_files = [f for f in os.listdir(configs_dir) if f.endswith(('.yaml', '.yml'))] - config_files.sort() - except Exception as e: - logger.error(f"Error listing config files: {e}") - return render_template( 'alert_rules.html', - rules=rules, - config_files=config_files + rules=rules ) diff --git a/app/web/services/config_service.py b/app/web/services/config_service.py index bd6c6b4..651cb7b 100644 --- a/app/web/services/config_service.py +++ b/app/web/services/config_service.py @@ -129,7 +129,7 @@ class ConfigService: 'id': site.id, 'name': site.name, 'description': site.description, - 'cidr_count': len(site.cidrs) + 'ip_count': len(site.ips) }) return { diff --git a/app/web/services/site_service.py b/app/web/services/site_service.py index 0a063ad..646c334 100644 --- a/app/web/services/site_service.py +++ b/app/web/services/site_service.py @@ -16,7 +16,7 @@ from sqlalchemy.orm import Session, joinedload from sqlalchemy.exc import IntegrityError from web.models import ( - Site, SiteCIDR, SiteIP, ScanSiteAssociation + Site, SiteIP, ScanSiteAssociation ) from web.utils.pagination import paginate, PaginatedResult @@ -40,34 +40,26 @@ class SiteService: """ self.db = db_session - def create_site(self, name: str, description: Optional[str] = None, - cidrs: Optional[List[Dict[str, Any]]] = None) -> Dict[str, Any]: + def create_site(self, name: str, description: Optional[str] = None) -> Dict[str, Any]: """ - Create a new site with optional CIDR ranges. + Create a new site. Args: name: Unique site name description: Optional site description - cidrs: List of CIDR definitions with format: - [{"cidr": "10.0.0.0/24", "expected_ping": true, - "expected_tcp_ports": [22, 80], "expected_udp_ports": [53]}] Returns: Dictionary with created site data Raises: - ValueError: If site name already exists or validation fails + ValueError: If site name already exists """ # Validate site name is unique existing = self.db.query(Site).filter(Site.name == name).first() if existing: raise ValueError(f"Site with name '{name}' already exists") - # Validate we have at least one CIDR if provided - if cidrs is not None and len(cidrs) == 0: - raise ValueError("Site must have at least one CIDR range") - - # Create site + # Create site (can be empty, IPs added separately) site = Site( name=name, description=description, @@ -76,17 +68,10 @@ class SiteService: ) self.db.add(site) - self.db.flush() # Get site.id without committing - - # Add CIDRs if provided - if cidrs: - for cidr_data in cidrs: - self._add_cidr_to_site(site, cidr_data) - self.db.commit() self.db.refresh(site) - logger.info(f"Created site '{name}' (id={site.id}) with {len(cidrs or [])} CIDR(s)") + logger.info(f"Created site '{name}' (id={site.id})") return self._site_to_dict(site) @@ -171,7 +156,7 @@ class SiteService: def get_site(self, site_id: int) -> Optional[Dict[str, Any]]: """ - Get site details with all CIDRs and IP overrides. + Get site details. Args: site_id: Site ID to retrieve @@ -181,9 +166,6 @@ class SiteService: """ site = ( self.db.query(Site) - .options( - joinedload(Site.cidrs).joinedload(SiteCIDR.ips) - ) .filter(Site.id == site_id) .first() ) @@ -205,9 +187,6 @@ class SiteService: """ site = ( self.db.query(Site) - .options( - joinedload(Site.cidrs).joinedload(SiteCIDR.ips) - ) .filter(Site.name == name) .first() ) @@ -230,7 +209,6 @@ class SiteService: """ query = ( self.db.query(Site) - .options(joinedload(Site.cidrs)) .order_by(Site.name) ) @@ -245,160 +223,211 @@ class SiteService: """ sites = ( self.db.query(Site) - .options(joinedload(Site.cidrs)) .order_by(Site.name) .all() ) return [self._site_to_dict(site) for site in sites] - def add_cidr(self, site_id: int, cidr: str, expected_ping: Optional[bool] = None, - expected_tcp_ports: Optional[List[int]] = None, - expected_udp_ports: Optional[List[int]] = None) -> Dict[str, Any]: + def bulk_add_ips_from_cidr(self, site_id: int, cidr: str, + expected_ping: Optional[bool] = None, + expected_tcp_ports: Optional[List[int]] = None, + expected_udp_ports: Optional[List[int]] = None) -> Dict[str, Any]: """ - Add a CIDR range to a site. + Expand a CIDR range and add all IPs to a site. + + CIDRs are NOT stored - they are just used to generate IP records. Args: site_id: Site ID cidr: CIDR notation (e.g., "10.0.0.0/24") - expected_ping: Expected ping response for IPs in this CIDR + expected_ping: Expected ping response for all IPs + expected_tcp_ports: List of expected TCP ports for all IPs + expected_udp_ports: List of expected UDP ports for all IPs + + Returns: + Dictionary with: + - cidr: The CIDR that was expanded + - ip_count: Number of IPs created + - ips_added: List of IP addresses created + - ips_skipped: List of IPs that already existed + + Raises: + ValueError: If site not found or CIDR is invalid/too large + """ + site = self.db.query(Site).filter(Site.id == site_id).first() + if not site: + raise ValueError(f"Site with id {site_id} not found") + + # Validate CIDR format and size + try: + network = ipaddress.ip_network(cidr, strict=False) + except ValueError as e: + raise ValueError(f"Invalid CIDR notation '{cidr}': {str(e)}") + + # Enforce CIDR size limits (max /24 for IPv4, /64 for IPv6) + if isinstance(network, ipaddress.IPv4Network) and network.prefixlen < 24: + raise ValueError( + f"CIDR '{cidr}' is too large ({network.num_addresses} IPs). " + f"Maximum allowed is /24 (256 IPs) for IPv4." + ) + elif isinstance(network, ipaddress.IPv6Network) and network.prefixlen < 64: + raise ValueError( + f"CIDR '{cidr}' is too large. " + f"Maximum allowed is /64 for IPv6." + ) + + # Expand CIDR to individual IPs (no cidr_id since we're not storing CIDR) + ip_count, ips_added, ips_skipped = self._expand_cidr_to_ips( + site_id=site_id, + network=network, + expected_ping=expected_ping, + expected_tcp_ports=expected_tcp_ports or [], + expected_udp_ports=expected_udp_ports or [] + ) + + site.updated_at = datetime.utcnow() + self.db.commit() + + logger.info( + f"Expanded CIDR '{cidr}' for site {site_id} ('{site.name}'): " + f"added {ip_count} IPs, skipped {len(ips_skipped)} duplicates" + ) + + return { + 'cidr': cidr, + 'ip_count': ip_count, + 'ips_added': ips_added, + 'ips_skipped': ips_skipped + } + + def bulk_add_ips_from_list(self, site_id: int, ip_list: List[str], + expected_ping: Optional[bool] = None, + expected_tcp_ports: Optional[List[int]] = None, + expected_udp_ports: Optional[List[int]] = None) -> Dict[str, Any]: + """ + Add multiple IPs from a list (e.g., from CSV/text import). + + Args: + site_id: Site ID + ip_list: List of IP addresses as strings + expected_ping: Expected ping response for all IPs + expected_tcp_ports: List of expected TCP ports for all IPs + expected_udp_ports: List of expected UDP ports for all IPs + + Returns: + Dictionary with: + - ip_count: Number of IPs successfully created + - ips_added: List of IP addresses created + - ips_skipped: List of IPs that already existed + - errors: List of validation errors {ip: error_message} + + Raises: + ValueError: If site not found + """ + site = self.db.query(Site).filter(Site.id == site_id).first() + if not site: + raise ValueError(f"Site with id {site_id} not found") + + ips_added = [] + ips_skipped = [] + errors = [] + + for ip_str in ip_list: + ip_str = ip_str.strip() + if not ip_str: + continue # Skip empty lines + + # Validate IP format + try: + ipaddress.ip_address(ip_str) + except ValueError as e: + errors.append({'ip': ip_str, 'error': f"Invalid IP address: {str(e)}"}) + continue + + # Check for duplicate (across all IPs in the site) + existing = ( + self.db.query(SiteIP) + .filter(SiteIP.site_id == site_id, SiteIP.ip_address == ip_str) + .first() + ) + if existing: + ips_skipped.append(ip_str) + continue + + # Create IP record + try: + ip_obj = SiteIP( + site_id=site_id, + ip_address=ip_str, + expected_ping=expected_ping, + expected_tcp_ports=json.dumps(expected_tcp_ports or []), + expected_udp_ports=json.dumps(expected_udp_ports or []), + created_at=datetime.utcnow() + ) + + self.db.add(ip_obj) + ips_added.append(ip_str) + except Exception as e: + errors.append({'ip': ip_str, 'error': f"Database error: {str(e)}"}) + + site.updated_at = datetime.utcnow() + self.db.commit() + + logger.info( + f"Bulk added {len(ips_added)} IPs to site {site_id} ('{site.name}'), " + f"skipped {len(ips_skipped)} duplicates, {len(errors)} errors" + ) + + return { + 'ip_count': len(ips_added), + 'ips_added': ips_added, + 'ips_skipped': ips_skipped, + 'errors': errors + } + + def add_standalone_ip(self, site_id: int, ip_address: str, + expected_ping: Optional[bool] = None, + expected_tcp_ports: Optional[List[int]] = None, + expected_udp_ports: Optional[List[int]] = None) -> Dict[str, Any]: + """ + Add a standalone IP (without a CIDR parent) to a site. + + Args: + site_id: Site ID + ip_address: IP address to add + expected_ping: Expected ping response expected_tcp_ports: List of expected TCP ports expected_udp_ports: List of expected UDP ports Returns: - Dictionary with CIDR data + Dictionary with IP data Raises: - ValueError: If site not found, CIDR is invalid, or already exists + ValueError: If site not found, IP is invalid, or already exists """ site = self.db.query(Site).filter(Site.id == site_id).first() if not site: raise ValueError(f"Site with id {site_id} not found") - # Validate CIDR format - try: - ipaddress.ip_network(cidr, strict=False) - except ValueError as e: - raise ValueError(f"Invalid CIDR notation '{cidr}': {str(e)}") - - # Check for duplicate CIDR - existing = ( - self.db.query(SiteCIDR) - .filter(SiteCIDR.site_id == site_id, SiteCIDR.cidr == cidr) - .first() - ) - if existing: - raise ValueError(f"CIDR '{cidr}' already exists for this site") - - # Create CIDR - cidr_obj = SiteCIDR( - site_id=site_id, - cidr=cidr, - expected_ping=expected_ping, - expected_tcp_ports=json.dumps(expected_tcp_ports or []), - expected_udp_ports=json.dumps(expected_udp_ports or []), - created_at=datetime.utcnow() - ) - - self.db.add(cidr_obj) - site.updated_at = datetime.utcnow() - self.db.commit() - self.db.refresh(cidr_obj) - - logger.info(f"Added CIDR '{cidr}' to site {site_id} ('{site.name}')") - - return self._cidr_to_dict(cidr_obj) - - def remove_cidr(self, site_id: int, cidr_id: int) -> None: - """ - Remove a CIDR range from a site. - - Prevents removal if it's the last CIDR (sites must have at least one CIDR). - - Args: - site_id: Site ID - cidr_id: CIDR ID to remove - - Raises: - ValueError: If CIDR not found or it's the last CIDR - """ - site = self.db.query(Site).filter(Site.id == site_id).first() - if not site: - raise ValueError(f"Site with id {site_id} not found") - - cidr = ( - self.db.query(SiteCIDR) - .filter(SiteCIDR.id == cidr_id, SiteCIDR.site_id == site_id) - .first() - ) - if not cidr: - raise ValueError(f"CIDR with id {cidr_id} not found for site {site_id}") - - # Check if this is the last CIDR - cidr_count = ( - self.db.query(func.count(SiteCIDR.id)) - .filter(SiteCIDR.site_id == site_id) - .scalar() - ) - - if cidr_count <= 1: - raise ValueError( - f"Cannot remove CIDR '{cidr.cidr}': site must have at least one CIDR range" - ) - - self.db.delete(cidr) - site.updated_at = datetime.utcnow() - self.db.commit() - - logger.info(f"Removed CIDR '{cidr.cidr}' from site {site_id} ('{site.name}')") - - def add_ip_override(self, cidr_id: int, ip_address: str, - expected_ping: Optional[bool] = None, - expected_tcp_ports: Optional[List[int]] = None, - expected_udp_ports: Optional[List[int]] = None) -> Dict[str, Any]: - """ - Add an IP-level expectation override within a CIDR. - - Args: - cidr_id: CIDR ID - ip_address: IP address to override - expected_ping: Override ping expectation - expected_tcp_ports: Override TCP ports expectation - expected_udp_ports: Override UDP ports expectation - - Returns: - Dictionary with IP override data - - Raises: - ValueError: If CIDR not found, IP is invalid, or not in CIDR range - """ - cidr = self.db.query(SiteCIDR).filter(SiteCIDR.id == cidr_id).first() - if not cidr: - raise ValueError(f"CIDR with id {cidr_id} not found") - # Validate IP format try: - ip_obj = ipaddress.ip_address(ip_address) + ipaddress.ip_address(ip_address) except ValueError as e: raise ValueError(f"Invalid IP address '{ip_address}': {str(e)}") - # Validate IP is within CIDR range - network = ipaddress.ip_network(cidr.cidr, strict=False) - if ip_obj not in network: - raise ValueError(f"IP address '{ip_address}' is not within CIDR '{cidr.cidr}'") - - # Check for duplicate + # Check for duplicate (across all IPs in the site) existing = ( self.db.query(SiteIP) - .filter(SiteIP.site_cidr_id == cidr_id, SiteIP.ip_address == ip_address) + .filter(SiteIP.site_id == site_id, SiteIP.ip_address == ip_address) .first() ) if existing: - raise ValueError(f"IP override for '{ip_address}' already exists in this CIDR") + raise ValueError(f"IP '{ip_address}' already exists in this site") - # Create IP override - ip_override = SiteIP( - site_cidr_id=cidr_id, + # Create IP + ip_obj = SiteIP( + site_id=site_id, ip_address=ip_address, expected_ping=expected_ping, expected_tcp_ports=json.dumps(expected_tcp_ports or []), @@ -406,38 +435,102 @@ class SiteService: created_at=datetime.utcnow() ) - self.db.add(ip_override) + self.db.add(ip_obj) + site.updated_at = datetime.utcnow() self.db.commit() - self.db.refresh(ip_override) + self.db.refresh(ip_obj) - logger.info(f"Added IP override '{ip_address}' to CIDR {cidr_id} ('{cidr.cidr}')") + logger.info(f"Added IP '{ip_address}' to site {site_id} ('{site.name}')") - return self._ip_override_to_dict(ip_override) + return self._ip_to_dict(ip_obj) - def remove_ip_override(self, cidr_id: int, ip_id: int) -> None: + def update_ip_settings(self, site_id: int, ip_id: int, + expected_ping: Optional[bool] = None, + expected_tcp_ports: Optional[List[int]] = None, + expected_udp_ports: Optional[List[int]] = None) -> Dict[str, Any]: """ - Remove an IP-level override. + Update settings for an individual IP. Args: - cidr_id: CIDR ID - ip_id: IP override ID to remove + site_id: Site ID + ip_id: IP ID to update + expected_ping: New ping expectation (if provided) + expected_tcp_ports: New TCP ports expectation (if provided) + expected_udp_ports: New UDP ports expectation (if provided) + + Returns: + Dictionary with updated IP data Raises: - ValueError: If IP override not found + ValueError: If IP not found """ - ip_override = ( + ip_obj = ( self.db.query(SiteIP) - .filter(SiteIP.id == ip_id, SiteIP.site_cidr_id == cidr_id) + .filter(SiteIP.id == ip_id, SiteIP.site_id == site_id) .first() ) - if not ip_override: - raise ValueError(f"IP override with id {ip_id} not found for CIDR {cidr_id}") + if not ip_obj: + raise ValueError(f"IP with id {ip_id} not found for site {site_id}") - ip_address = ip_override.ip_address - self.db.delete(ip_override) + # Update settings if provided + if expected_ping is not None: + ip_obj.expected_ping = expected_ping + if expected_tcp_ports is not None: + ip_obj.expected_tcp_ports = json.dumps(expected_tcp_ports) + if expected_udp_ports is not None: + ip_obj.expected_udp_ports = json.dumps(expected_udp_ports) + + self.db.commit() + self.db.refresh(ip_obj) + + logger.info(f"Updated settings for IP '{ip_obj.ip_address}' in site {site_id}") + + return self._ip_to_dict(ip_obj) + + def remove_ip(self, site_id: int, ip_id: int) -> None: + """ + Remove an IP from a site. + + Args: + site_id: Site ID + ip_id: IP ID to remove + + Raises: + ValueError: If IP not found + """ + ip_obj = ( + self.db.query(SiteIP) + .filter(SiteIP.id == ip_id, SiteIP.site_id == site_id) + .first() + ) + if not ip_obj: + raise ValueError(f"IP with id {ip_id} not found for site {site_id}") + + ip_address = ip_obj.ip_address + self.db.delete(ip_obj) self.db.commit() - logger.info(f"Removed IP override '{ip_address}' from CIDR {cidr_id}") + logger.info(f"Removed IP '{ip_address}' from site {site_id}") + + def list_ips(self, site_id: int, page: int = 1, per_page: int = 50) -> PaginatedResult: + """ + List IPs in a site with pagination. + + Args: + site_id: Site ID + page: Page number (1-indexed) + per_page: Number of items per page + + Returns: + PaginatedResult with IP data + """ + query = ( + self.db.query(SiteIP) + .filter(SiteIP.site_id == site_id) + .order_by(SiteIP.ip_address) + ) + + return paginate(query, page, per_page, self._ip_to_dict) def get_scan_usage(self, site_id: int) -> List[Dict[str, Any]]: """ @@ -470,59 +563,91 @@ class SiteService: # Private helper methods - def _add_cidr_to_site(self, site: Site, cidr_data: Dict[str, Any]) -> SiteCIDR: - """Helper to add CIDR during site creation.""" - cidr = cidr_data.get('cidr') - if not cidr: - raise ValueError("CIDR 'cidr' field is required") + def _expand_cidr_to_ips(self, site_id: int, + network: ipaddress.IPv4Network | ipaddress.IPv6Network, + expected_ping: Optional[bool], + expected_tcp_ports: List[int], + expected_udp_ports: List[int]) -> tuple[int, List[str], List[str]]: + """ + Expand a CIDR to individual IP addresses. - # Validate CIDR format - try: - ipaddress.ip_network(cidr, strict=False) - except ValueError as e: - raise ValueError(f"Invalid CIDR notation '{cidr}': {str(e)}") + Args: + site_id: Site ID + network: ipaddress network object + expected_ping: Default ping setting for all IPs + expected_tcp_ports: Default TCP ports for all IPs + expected_udp_ports: Default UDP ports for all IPs - cidr_obj = SiteCIDR( - site_id=site.id, - cidr=cidr, - expected_ping=cidr_data.get('expected_ping'), - expected_tcp_ports=json.dumps(cidr_data.get('expected_tcp_ports', [])), - expected_udp_ports=json.dumps(cidr_data.get('expected_udp_ports', [])), - created_at=datetime.utcnow() - ) + Returns: + Tuple of (count of IPs created, list of IPs added, list of IPs skipped) + """ + ip_count = 0 + ips_added = [] + ips_skipped = [] - self.db.add(cidr_obj) - return cidr_obj + # For /32 or /128 (single host), use the network address + # For larger ranges, use hosts() to exclude network/broadcast addresses + if network.num_addresses == 1: + ip_list = [network.network_address] + elif network.num_addresses == 2: + # For /31 networks (point-to-point), both addresses are usable + ip_list = [network.network_address, network.broadcast_address] + else: + # Use hosts() to get usable IPs (excludes network and broadcast) + ip_list = list(network.hosts()) + + for ip in ip_list: + ip_str = str(ip) + + # Check for duplicate + existing = ( + self.db.query(SiteIP) + .filter(SiteIP.site_id == site_id, SiteIP.ip_address == ip_str) + .first() + ) + if existing: + ips_skipped.append(ip_str) + continue + + # Create SiteIP entry + ip_obj = SiteIP( + site_id=site_id, + ip_address=ip_str, + expected_ping=expected_ping, + expected_tcp_ports=json.dumps(expected_tcp_ports), + expected_udp_ports=json.dumps(expected_udp_ports), + created_at=datetime.utcnow() + ) + + self.db.add(ip_obj) + ips_added.append(ip_str) + ip_count += 1 + + return ip_count, ips_added, ips_skipped def _site_to_dict(self, site: Site) -> Dict[str, Any]: """Convert Site model to dictionary.""" + # Count IPs for this site + ip_count = ( + self.db.query(func.count(SiteIP.id)) + .filter(SiteIP.site_id == site.id) + .scalar() or 0 + ) + return { 'id': site.id, 'name': site.name, 'description': site.description, 'created_at': site.created_at.isoformat() if site.created_at else None, 'updated_at': site.updated_at.isoformat() if site.updated_at else None, - 'cidrs': [self._cidr_to_dict(cidr) for cidr in site.cidrs] if hasattr(site, 'cidrs') else [] + 'ip_count': ip_count } - def _cidr_to_dict(self, cidr: SiteCIDR) -> Dict[str, Any]: - """Convert SiteCIDR model to dictionary.""" - return { - 'id': cidr.id, - 'site_id': cidr.site_id, - 'cidr': cidr.cidr, - 'expected_ping': cidr.expected_ping, - 'expected_tcp_ports': json.loads(cidr.expected_tcp_ports) if cidr.expected_tcp_ports else [], - 'expected_udp_ports': json.loads(cidr.expected_udp_ports) if cidr.expected_udp_ports else [], - 'created_at': cidr.created_at.isoformat() if cidr.created_at else None, - 'ip_overrides': [self._ip_override_to_dict(ip) for ip in cidr.ips] if hasattr(cidr, 'ips') else [] - } - - def _ip_override_to_dict(self, ip: SiteIP) -> Dict[str, Any]: + def _ip_to_dict(self, ip: SiteIP) -> Dict[str, Any]: """Convert SiteIP model to dictionary.""" return { 'id': ip.id, - 'site_cidr_id': ip.site_cidr_id, + 'site_id': ip.site_id, 'ip_address': ip.ip_address, 'expected_ping': ip.expected_ping, 'expected_tcp_ports': json.loads(ip.expected_tcp_ports) if ip.expected_tcp_ports else [], diff --git a/app/web/templates/alert_rules.html b/app/web/templates/alert_rules.html index e359126..4b15c31 100644 --- a/app/web/templates/alert_rules.html +++ b/app/web/templates/alert_rules.html @@ -96,8 +96,8 @@ {% endif %} - {% if rule.config_file %} - {{ rule.config_file }} + {% if rule.config %} + {{ rule.config.title }} {% else %} All Configs {% endif %} @@ -209,20 +209,9 @@ - - {% if config_files %} - Select a specific config file to limit this rule, or leave as "All Configs" to apply to all scans - {% else %} - No config files found. Upload a config in the Configs section to see available options. - {% endif %} + + Select a specific config to limit this rule, or leave as "All Configs" to apply to all scans @@ -272,12 +261,51 @@