sneakygeek/SneakyScan
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Migrate from file-based configs to database with per-IP site configuration

Browse Source 
Major architectural changes:
   - Replace YAML config files with database-stored ScanConfig model
   - Remove CIDR block support in favor of individual IP addresses per site
   - Each IP now has its own expected_ping, expected_tcp_ports, expected_udp_ports
   - AlertRule now uses config_id FK instead of config_file string

   API changes:
   - POST /api/scans now requires config_id instead of config_file
   - Alert rules API uses config_id with validation
   - All config dropdowns fetch from /api/configs dynamically

   Template updates:
   - scans.html, dashboard.html, alert_rules.html load configs via API
   - Display format: Config Title (X sites) in dropdowns
   - Removed Jinja2 config_files loops

   Migrations:
   - 008: Expand CIDRs to individual IPs with per-IP port configs
   - 009: Remove CIDR-related columns
   - 010: Add config_id to alert_rules, remove config_file
This commit is contained in:
Phillip Tarrant
2025-11-19 19:40:34 -06:00
parent 034f146fa1
commit 0ec338e252
21 changed files with 2004 additions and 686 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
app/web/api/alerts.py
View File

@@ -169,7 +169,8 @@ def list_alert_rules():
'webhook_enabled': rule.webhook_enabled,
'severity': rule.severity,
'filter_conditions': json.loads(rule.filter_conditions) if rule.filter_conditions else None,
'config_file': rule.config_file,
'config_id': rule.config_id,
'config_title': rule.config.title if rule.config else None,
'created_at': rule.created_at.isoformat(),
'updated_at': rule.updated_at.isoformat() if rule.updated_at else None
})
@@ -195,7 +196,7 @@ def create_alert_rule():
webhook_enabled: Send webhook for this rule (default: false)
severity: Alert severity (critical, warning, info)
filter_conditions: JSON object with filter conditions
config_file: Optional config file to apply rule to
config_id: Optional config ID to apply rule to
Returns:
JSON response with created rule
@@ -226,6 +227,17 @@ def create_alert_rule():
}), 400
try:
# Validate config_id if provided
config_id = data.get('config_id')
if config_id:
from web.models import ScanConfig
config = current_app.db_session.query(ScanConfig).filter_by(id=config_id).first()
if not config:
return jsonify({
'status': 'error',
'message': f'Config with ID {config_id} not found'
}), 400
# Create new rule
rule = AlertRule(
name=data.get('name', f"{data['rule_type']} rule"),
@@ -236,7 +248,7 @@ def create_alert_rule():
webhook_enabled=data.get('webhook_enabled', False),
severity=data.get('severity', 'warning'),
filter_conditions=json.dumps(data['filter_conditions']) if data.get('filter_conditions') else None,
config_file=data.get('config_file'),
config_id=config_id,
created_at=datetime.now(timezone.utc),
updated_at=datetime.now(timezone.utc)
)
@@ -257,7 +269,8 @@ def create_alert_rule():
'webhook_enabled': rule.webhook_enabled,
'severity': rule.severity,
'filter_conditions': json.loads(rule.filter_conditions) if rule.filter_conditions else None,
'config_file': rule.config_file,
'config_id': rule.config_id,
'config_title': rule.config.title if rule.config else None,
'created_at': rule.created_at.isoformat(),
'updated_at': rule.updated_at.isoformat()
}
@@ -288,7 +301,7 @@ def update_alert_rule(rule_id):
webhook_enabled: Send webhook for this rule (optional)
severity: Alert severity (optional)
filter_conditions: JSON object with filter conditions (optional)
config_file: Config file to apply rule to (optional)
config_id: Config ID to apply rule to (optional)
Returns:
JSON response with update status
@@ -312,6 +325,18 @@ def update_alert_rule(rule_id):
}), 400
try:
# Validate config_id if provided
if 'config_id' in data:
config_id = data['config_id']
if config_id:
from web.models import ScanConfig
config = current_app.db_session.query(ScanConfig).filter_by(id=config_id).first()
if not config:
return jsonify({
'status': 'error',
'message': f'Config with ID {config_id} not found'
}), 400
# Update fields if provided
if 'name' in data:
rule.name = data['name']
@@ -327,8 +352,8 @@ def update_alert_rule(rule_id):
rule.severity = data['severity']
if 'filter_conditions' in data:
rule.filter_conditions = json.dumps(data['filter_conditions']) if data['filter_conditions'] else None
if 'config_file' in data:
rule.config_file = data['config_file']
if 'config_id' in data:
rule.config_id = data['config_id']
rule.updated_at = datetime.now(timezone.utc)
@@ -347,7 +372,8 @@ def update_alert_rule(rule_id):
'webhook_enabled': rule.webhook_enabled,
'severity': rule.severity,
'filter_conditions': json.loads(rule.filter_conditions) if rule.filter_conditions else None,
'config_file': rule.config_file,
'config_id': rule.config_id,
'config_title': rule.config.title if rule.config else None,
'created_at': rule.created_at.isoformat(),
'updated_at': rule.updated_at.isoformat()
}

2
app/web/api/configs.py
View File

@@ -174,7 +174,7 @@ def get_config(config_id: int):
"id": 1,
"name": "Production DC",
"description": "...",
"cidr_count": 5
"ip_count": 5
}
],
"created_at": "2025-11-19T10:30:00Z",

30
app/web/api/scans.py
View File

@@ -129,7 +129,7 @@ def trigger_scan():
Trigger a new scan.
Request body:
config_file: Path to YAML config file
config_id: Database config ID (required)
Returns:
JSON response with scan_id and status
@@ -137,25 +137,35 @@ def trigger_scan():
try:
# Get request data
data = request.get_json() or {}
config_file = data.get('config_file')
config_id = data.get('config_id')
# Validate required fields
if not config_file:
logger.warning("Scan trigger request missing config_file")
if not config_id:
logger.warning("Scan trigger request missing config_id")
return jsonify({
'error': 'Invalid request',
'message': 'config_file is required'
'message': 'config_id is required'
}), 400
# Validate config_id is an integer
try:
config_id = int(config_id)
except (TypeError, ValueError):
logger.warning(f"Invalid config_id type: {config_id}")
return jsonify({
'error': 'Invalid request',
'message': 'config_id must be an integer'
}), 400
# Trigger scan via service
scan_service = ScanService(current_app.db_session)
scan_id = scan_service.trigger_scan(
config_file=config_file,
config_id=config_id,
triggered_by='api',
scheduler=current_app.scheduler
)
logger.info(f"Scan {scan_id} triggered via API: config={config_file}")
logger.info(f"Scan {scan_id} triggered via API: config_id={config_id}")
return jsonify({
'scan_id': scan_id,
@@ -164,10 +174,10 @@ def trigger_scan():
}), 201
except ValueError as e:
# Config file validation error
# Config validation error
error_message = str(e)
logger.warning(f"Invalid config file: {error_message}")
logger.warning(f"Request data: config_file='{config_file}'")
logger.warning(f"Invalid config: {error_message}")
logger.warning(f"Request data: config_id='{config_id}'")
return jsonify({
'error': 'Invalid request',
'message': error_message

263
app/web/api/sites.py
View File

@@ -158,21 +158,12 @@ def create_site():
}), 400
description = data.get('description')
cidrs = data.get('cidrs', [])
# Validate cidrs is a list
if not isinstance(cidrs, list):
return jsonify({
'error': 'Invalid request',
'message': 'cidrs must be a list'
}), 400
# Create site
# Create site (empty initially)
site_service = SiteService(current_app.db_session)
site = site_service.create_site(
name=name,
description=description,
cidrs=cidrs if cidrs else None
description=description
)
logger.info(f"Created site '{name}' (id={site['id']})")
@@ -294,135 +285,178 @@ def delete_site(site_id):
}), 500
@bp.route('/<int:site_id>/cidrs', methods=['POST'])
@bp.route('/<int:site_id>/ips/bulk', methods=['POST'])
@api_auth_required
def add_cidr(site_id):
def bulk_add_ips(site_id):
"""
Add a CIDR range to a site.
Bulk add IPs to a site from CIDR or list.
Args:
site_id: Site ID
Request body:
cidr: CIDR notation (required, e.g., "10.0.0.0/24")
expected_ping: Expected ping response (optional)
expected_tcp_ports: List of expected TCP ports (optional)
expected_udp_ports: List of expected UDP ports (optional)
source_type: "cidr" or "list" (required)
cidr: CIDR notation if source_type="cidr" (e.g., "10.0.0.0/24")
ips: List of IP addresses if source_type="list" (e.g., ["10.0.0.1", "10.0.0.2"])
expected_ping: Expected ping response for all IPs (optional)
expected_tcp_ports: List of expected TCP ports for all IPs (optional)
expected_udp_ports: List of expected UDP ports for all IPs (optional)
Returns:
JSON response with created CIDR data
JSON response with count of IPs added and any errors
"""
try:
data = request.get_json() or {}
# Validate required fields
cidr = data.get('cidr')
if not cidr:
logger.warning("CIDR creation request missing cidr")
source_type = data.get('source_type')
if source_type not in ['cidr', 'list']:
return jsonify({
'error': 'Invalid request',
'message': 'cidr is required'
'message': 'source_type must be "cidr" or "list"'
}), 400
expected_ping = data.get('expected_ping')
expected_tcp_ports = data.get('expected_tcp_ports', [])
expected_udp_ports = data.get('expected_udp_ports', [])
# Add CIDR
site_service = SiteService(current_app.db_session)
cidr_data = site_service.add_cidr(
site_id=site_id,
cidr=cidr,
expected_ping=expected_ping,
expected_tcp_ports=expected_tcp_ports,
expected_udp_ports=expected_udp_ports
)
logger.info(f"Added CIDR '{cidr}' to site {site_id}")
return jsonify(cidr_data), 201
if source_type == 'cidr':
cidr = data.get('cidr')
if not cidr:
return jsonify({
'error': 'Invalid request',
'message': 'cidr is required when source_type="cidr"'
}), 400
result = site_service.bulk_add_ips_from_cidr(
site_id=site_id,
cidr=cidr,
expected_ping=expected_ping,
expected_tcp_ports=expected_tcp_ports,
expected_udp_ports=expected_udp_ports
)
logger.info(f"Bulk added {result['ip_count']} IPs from CIDR '{cidr}' to site {site_id}")
return jsonify(result), 201
else: # source_type == 'list'
ip_list = data.get('ips', [])
if not isinstance(ip_list, list):
return jsonify({
'error': 'Invalid request',
'message': 'ips must be a list when source_type="list"'
}), 400
result = site_service.bulk_add_ips_from_list(
site_id=site_id,
ip_list=ip_list,
expected_ping=expected_ping,
expected_tcp_ports=expected_tcp_ports,
expected_udp_ports=expected_udp_ports
)
logger.info(f"Bulk added {result['ip_count']} IPs from list to site {site_id}")
return jsonify(result), 201
except ValueError as e:
logger.warning(f"Invalid CIDR creation request: {str(e)}")
logger.warning(f"Invalid bulk IP request: {str(e)}")
return jsonify({
'error': 'Invalid request',
'message': str(e)
}), 400
except SQLAlchemyError as e:
logger.error(f"Database error adding CIDR to site {site_id}: {str(e)}")
logger.error(f"Database error bulk adding IPs to site {site_id}: {str(e)}")
return jsonify({
'error': 'Database error',
'message': 'Failed to add CIDR'
'message': 'Failed to add IPs'
}), 500
except Exception as e:
logger.error(f"Unexpected error adding CIDR to site {site_id}: {str(e)}", exc_info=True)
logger.error(f"Unexpected error bulk adding IPs to site {site_id}: {str(e)}", exc_info=True)
return jsonify({
'error': 'Internal server error',
'message': 'An unexpected error occurred'
}), 500
@bp.route('/<int:site_id>/cidrs/<int:cidr_id>', methods=['DELETE'])
@bp.route('/<int:site_id>/ips', methods=['GET'])
@api_auth_required
def remove_cidr(site_id, cidr_id):
def list_ips(site_id):
"""
Remove a CIDR range from a site.
List IPs in a site with pagination.
Prevents removal if it's the last CIDR.
Args:
site_id: Site ID
cidr_id: CIDR ID
Query params:
page: Page number (default: 1)
per_page: Items per page (default: 50, max: 200)
Returns:
JSON response with success message
JSON response with IPs list and pagination info
"""
try:
site_service = SiteService(current_app.db_session)
site_service.remove_cidr(site_id, cidr_id)
# Get and validate query parameters
page = request.args.get('page', 1, type=int)
per_page = request.args.get('per_page', 50, type=int)
# Validate pagination params
page, per_page = validate_page_params(page, per_page, max_per_page=200)
# Get IPs from service
site_service = SiteService(current_app.db_session)
paginated_result = site_service.list_ips(
site_id=site_id,
page=page,
per_page=per_page
)
logger.info(f"Listed IPs for site {site_id}: page={page}, per_page={per_page}, total={paginated_result.total}")
logger.info(f"Removed CIDR {cidr_id} from site {site_id}")
return jsonify({
'message': f'CIDR {cidr_id} removed successfully'
'ips': paginated_result.items,
'total': paginated_result.total,
'page': paginated_result.page,
'per_page': paginated_result.per_page,
'total_pages': paginated_result.pages,
'has_prev': paginated_result.has_prev,
'has_next': paginated_result.has_next
})
except ValueError as e:
logger.warning(f"Cannot remove CIDR {cidr_id} from site {site_id}: {str(e)}")
logger.warning(f"Invalid request parameters: {str(e)}")
return jsonify({
'error': 'Invalid request',
'message': str(e)
}), 400
except SQLAlchemyError as e:
logger.error(f"Database error removing CIDR {cidr_id} from site {site_id}: {str(e)}")
logger.error(f"Database error listing IPs for site {site_id}: {str(e)}")
return jsonify({
'error': 'Database error',
'message': 'Failed to remove CIDR'
'message': 'Failed to retrieve IPs'
}), 500
except Exception as e:
logger.error(f"Unexpected error removing CIDR {cidr_id} from site {site_id}: {str(e)}", exc_info=True)
logger.error(f"Unexpected error listing IPs for site {site_id}: {str(e)}", exc_info=True)
return jsonify({
'error': 'Internal server error',
'message': 'An unexpected error occurred'
}), 500
@bp.route('/<int:site_id>/cidrs/<int:cidr_id>/ips', methods=['POST'])
@bp.route('/<int:site_id>/ips', methods=['POST'])
@api_auth_required
def add_ip_override(site_id, cidr_id):
def add_standalone_ip(site_id):
"""
Add an IP-level expectation override within a CIDR.
Add a standalone IP (without CIDR parent) to a site.
Args:
site_id: Site ID (for validation)
cidr_id: CIDR ID
site_id: Site ID
Request body:
ip_address: IP address (required)
expected_ping: Override ping expectation (optional)
expected_tcp_ports: Override TCP ports expectation (optional)
expected_udp_ports: Override UDP ports expectation (optional)
expected_ping: Expected ping response (optional)
expected_tcp_ports: List of expected TCP ports (optional)
expected_udp_ports: List of expected UDP ports (optional)
Returns:
JSON response with created IP override data
JSON response with created IP data
"""
try:
data = request.get_json() or {}
@@ -430,7 +464,7 @@ def add_ip_override(site_id, cidr_id):
# Validate required fields
ip_address = data.get('ip_address')
if not ip_address:
logger.warning("IP override creation request missing ip_address")
logger.warning("Standalone IP creation request missing ip_address")
return jsonify({
'error': 'Invalid request',
'message': 'ip_address is required'
@@ -440,76 +474,133 @@ def add_ip_override(site_id, cidr_id):
expected_tcp_ports = data.get('expected_tcp_ports', [])
expected_udp_ports = data.get('expected_udp_ports', [])
# Add IP override
# Add standalone IP
site_service = SiteService(current_app.db_session)
ip_data = site_service.add_ip_override(
cidr_id=cidr_id,
ip_data = site_service.add_standalone_ip(
site_id=site_id,
ip_address=ip_address,
expected_ping=expected_ping,
expected_tcp_ports=expected_tcp_ports,
expected_udp_ports=expected_udp_ports
)
logger.info(f"Added IP override '{ip_address}' to CIDR {cidr_id} in site {site_id}")
logger.info(f"Added standalone IP '{ip_address}' to site {site_id}")
return jsonify(ip_data), 201
except ValueError as e:
logger.warning(f"Invalid IP override creation request: {str(e)}")
logger.warning(f"Invalid standalone IP creation request: {str(e)}")
return jsonify({
'error': 'Invalid request',
'message': str(e)
}), 400
except SQLAlchemyError as e:
logger.error(f"Database error adding IP override to CIDR {cidr_id}: {str(e)}")
logger.error(f"Database error adding standalone IP to site {site_id}: {str(e)}")
return jsonify({
'error': 'Database error',
'message': 'Failed to add IP override'
'message': 'Failed to add IP'
}), 500
except Exception as e:
logger.error(f"Unexpected error adding IP override to CIDR {cidr_id}: {str(e)}", exc_info=True)
logger.error(f"Unexpected error adding standalone IP to site {site_id}: {str(e)}", exc_info=True)
return jsonify({
'error': 'Internal server error',
'message': 'An unexpected error occurred'
}), 500
@bp.route('/<int:site_id>/cidrs/<int:cidr_id>/ips/<int:ip_id>', methods=['DELETE'])
@bp.route('/<int:site_id>/ips/<int:ip_id>', methods=['PUT'])
@api_auth_required
def remove_ip_override(site_id, cidr_id, ip_id):
def update_ip_settings(site_id, ip_id):
"""
Remove an IP-level override.
Update settings for an individual IP.
Args:
site_id: Site ID (for validation)
cidr_id: CIDR ID
ip_id: IP override ID
site_id: Site ID
ip_id: IP ID
Request body:
expected_ping: New ping expectation (optional)
expected_tcp_ports: New TCP ports expectation (optional)
expected_udp_ports: New UDP ports expectation (optional)
Returns:
JSON response with updated IP data
"""
try:
data = request.get_json() or {}
expected_ping = data.get('expected_ping')
expected_tcp_ports = data.get('expected_tcp_ports')
expected_udp_ports = data.get('expected_udp_ports')
# Update IP settings
site_service = SiteService(current_app.db_session)
ip_data = site_service.update_ip_settings(
site_id=site_id,
ip_id=ip_id,
expected_ping=expected_ping,
expected_tcp_ports=expected_tcp_ports,
expected_udp_ports=expected_udp_ports
)
logger.info(f"Updated IP {ip_id} in site {site_id}")
return jsonify(ip_data)
except ValueError as e:
logger.warning(f"Invalid IP update request: {str(e)}")
return jsonify({
'error': 'Invalid request',
'message': str(e)
}), 400
except SQLAlchemyError as e:
logger.error(f"Database error updating IP {ip_id} in site {site_id}: {str(e)}")
return jsonify({
'error': 'Database error',
'message': 'Failed to update IP'
}), 500
except Exception as e:
logger.error(f"Unexpected error updating IP {ip_id} in site {site_id}: {str(e)}", exc_info=True)
return jsonify({
'error': 'Internal server error',
'message': 'An unexpected error occurred'
}), 500
@bp.route('/<int:site_id>/ips/<int:ip_id>', methods=['DELETE'])
@api_auth_required
def remove_ip(site_id, ip_id):
"""
Remove an IP from a site.
Args:
site_id: Site ID
ip_id: IP ID
Returns:
JSON response with success message
"""
try:
site_service = SiteService(current_app.db_session)
site_service.remove_ip_override(cidr_id, ip_id)
site_service.remove_ip(site_id, ip_id)
logger.info(f"Removed IP override {ip_id} from CIDR {cidr_id} in site {site_id}")
logger.info(f"Removed IP {ip_id} from site {site_id}")
return jsonify({
'message': f'IP override {ip_id} removed successfully'
'message': f'IP {ip_id} removed successfully'
})
except ValueError as e:
logger.warning(f"Cannot remove IP override {ip_id}: {str(e)}")
logger.warning(f"Cannot remove IP {ip_id}: {str(e)}")
return jsonify({
'error': 'Invalid request',
'message': str(e)
}), 400
except SQLAlchemyError as e:
logger.error(f"Database error removing IP override {ip_id}: {str(e)}")
logger.error(f"Database error removing IP {ip_id}: {str(e)}")
return jsonify({
'error': 'Database error',
'message': 'Failed to remove IP override'
'message': 'Failed to remove IP'
}), 500
except Exception as e:
logger.error(f"Unexpected error removing IP override {ip_id}: {str(e)}", exc_info=True)
logger.error(f"Unexpected error removing IP {ip_id}: {str(e)}", exc_info=True)
return jsonify({
'error': 'Internal server error',
'message': 'An unexpected error occurred'