diff --git a/app/migrations/versions/012_add_scan_progress.py b/app/migrations/versions/012_add_scan_progress.py new file mode 100644 index 0000000..2da3309 --- /dev/null +++ b/app/migrations/versions/012_add_scan_progress.py @@ -0,0 +1,58 @@ +"""Add scan progress tracking + +Revision ID: 012 +Revises: 011 +Create Date: 2024-01-01 00:00:00.000000 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '012' +down_revision = '011' +branch_labels = None +depends_on = None + + +def upgrade(): + # Add progress tracking columns to scans table + op.add_column('scans', sa.Column('current_phase', sa.String(50), nullable=True, + comment='Current scan phase: ping, tcp_scan, udp_scan, service_detection, http_analysis')) + op.add_column('scans', sa.Column('total_ips', sa.Integer(), nullable=True, + comment='Total number of IPs to scan')) + op.add_column('scans', sa.Column('completed_ips', sa.Integer(), nullable=True, default=0, + comment='Number of IPs completed in current phase')) + + # Create scan_progress table for per-IP progress tracking + op.create_table( + 'scan_progress', + sa.Column('id', sa.Integer(), primary_key=True, autoincrement=True), + sa.Column('scan_id', sa.Integer(), sa.ForeignKey('scans.id'), nullable=False, index=True), + sa.Column('ip_address', sa.String(45), nullable=False, comment='IP address being scanned'), + sa.Column('site_name', sa.String(255), nullable=True, comment='Site name this IP belongs to'), + sa.Column('phase', sa.String(50), nullable=False, + comment='Phase: ping, tcp_scan, udp_scan, service_detection, http_analysis'), + sa.Column('status', sa.String(20), nullable=False, default='pending', + comment='pending, in_progress, completed, failed'), + sa.Column('ping_result', sa.Boolean(), nullable=True, comment='Ping response result'), + sa.Column('tcp_ports', sa.Text(), nullable=True, comment='JSON array of discovered TCP ports'), + sa.Column('udp_ports', sa.Text(), nullable=True, comment='JSON array of discovered UDP ports'), + sa.Column('services', sa.Text(), nullable=True, comment='JSON array of detected services'), + sa.Column('created_at', sa.DateTime(), nullable=False, server_default=sa.func.now(), + comment='Entry creation time'), + sa.Column('updated_at', sa.DateTime(), nullable=False, server_default=sa.func.now(), + onupdate=sa.func.now(), comment='Last update time'), + sa.UniqueConstraint('scan_id', 'ip_address', name='uix_scan_progress_ip') + ) + + +def downgrade(): + # Drop scan_progress table + op.drop_table('scan_progress') + + # Remove progress tracking columns from scans table + op.drop_column('scans', 'completed_ips') + op.drop_column('scans', 'total_ips') + op.drop_column('scans', 'current_phase') diff --git a/app/src/scanner.py b/app/src/scanner.py index 80c99d8..3159fcc 100644 --- a/app/src/scanner.py +++ b/app/src/scanner.py @@ -13,7 +13,7 @@ import time import zipfile from datetime import datetime from pathlib import Path -from typing import Dict, List, Any +from typing import Dict, List, Any, Callable, Optional import xml.etree.ElementTree as ET import yaml @@ -833,10 +833,17 @@ class SneakyScanner: return all_results - def scan(self) -> Dict[str, Any]: + def scan(self, progress_callback: Optional[Callable] = None) -> Dict[str, Any]: """ Perform complete scan based on configuration + Args: + progress_callback: Optional callback function for progress updates. + Called with (phase, ip, data) where: + - phase: 'init', 'ping', 'tcp_scan', 'udp_scan', 'service_detection', 'http_analysis' + - ip: IP address being processed (or None for phase start) + - data: Dict with progress data (results, counts, etc.) + Returns: Dictionary containing scan results """ @@ -873,16 +880,36 @@ class SneakyScanner: all_ips = sorted(list(all_ips)) print(f"Total IPs to scan: {len(all_ips)}", flush=True) + # Report initialization with total IP count + if progress_callback: + progress_callback('init', None, { + 'total_ips': len(all_ips), + 'ip_to_site': ip_to_site + }) + # Perform ping scan print(f"\n[1/5] Performing ping scan on {len(all_ips)} IPs...", flush=True) + if progress_callback: + progress_callback('ping', None, {'status': 'starting'}) ping_results = self._run_ping_scan(all_ips) + # Report ping results + if progress_callback: + progress_callback('ping', None, { + 'status': 'completed', + 'results': ping_results + }) + # Perform TCP scan (all ports) print(f"\n[2/5] Performing TCP scan on {len(all_ips)} IPs (ports 0-65535)...", flush=True) + if progress_callback: + progress_callback('tcp_scan', None, {'status': 'starting'}) tcp_results = self._run_masscan(all_ips, '0-65535', 'tcp') # Perform UDP scan (all ports) print(f"\n[3/5] Performing UDP scan on {len(all_ips)} IPs (ports 0-65535)...", flush=True) + if progress_callback: + progress_callback('udp_scan', None, {'status': 'starting'}) udp_results = self._run_masscan(all_ips, '0-65535', 'udp') # Organize results by IP @@ -918,8 +945,23 @@ class SneakyScanner: results_by_ip[ip]['actual']['tcp_ports'].sort() results_by_ip[ip]['actual']['udp_ports'].sort() + # Report TCP/UDP scan results with discovered ports per IP + if progress_callback: + tcp_udp_results = {} + for ip in all_ips: + tcp_udp_results[ip] = { + 'tcp_ports': results_by_ip[ip]['actual']['tcp_ports'], + 'udp_ports': results_by_ip[ip]['actual']['udp_ports'] + } + progress_callback('tcp_scan', None, { + 'status': 'completed', + 'results': tcp_udp_results + }) + # Perform service detection on TCP ports print(f"\n[4/5] Performing service detection on discovered TCP ports...", flush=True) + if progress_callback: + progress_callback('service_detection', None, {'status': 'starting'}) ip_ports = {ip: results_by_ip[ip]['actual']['tcp_ports'] for ip in all_ips} service_results = self._run_nmap_service_detection(ip_ports) @@ -928,10 +970,26 @@ class SneakyScanner: if ip in results_by_ip: results_by_ip[ip]['actual']['services'] = services + # Report service detection results + if progress_callback: + progress_callback('service_detection', None, { + 'status': 'completed', + 'results': service_results + }) + # Perform HTTP/HTTPS analysis on web services print(f"\n[5/5] Analyzing HTTP/HTTPS services and SSL/TLS configuration...", flush=True) + if progress_callback: + progress_callback('http_analysis', None, {'status': 'starting'}) http_results = self._run_http_analysis(service_results) + # Report HTTP analysis completion + if progress_callback: + progress_callback('http_analysis', None, { + 'status': 'completed', + 'results': http_results + }) + # Merge HTTP analysis into service results for ip, port_results in http_results.items(): if ip in results_by_ip: diff --git a/app/web/api/scans.py b/app/web/api/scans.py index 042cb89..d4f25f4 100644 --- a/app/web/api/scans.py +++ b/app/web/api/scans.py @@ -5,11 +5,13 @@ Handles endpoints for triggering scans, listing scan history, and retrieving scan results. """ +import json import logging from flask import Blueprint, current_app, jsonify, request from sqlalchemy.exc import SQLAlchemyError from web.auth.decorators import api_auth_required +from web.models import Scan, ScanProgress from web.services.scan_service import ScanService from web.utils.pagination import validate_page_params @@ -281,6 +283,102 @@ def get_scan_status(scan_id): }), 500 +@bp.route('//progress', methods=['GET']) +@api_auth_required +def get_scan_progress(scan_id): + """ + Get detailed progress for a running scan including per-IP results. + + Args: + scan_id: Scan ID + + Returns: + JSON response with scan progress including: + - current_phase: Current scan phase + - total_ips: Total IPs being scanned + - completed_ips: Number of IPs completed in current phase + - progress_entries: List of per-IP progress with discovered results + """ + try: + session = current_app.db_session + + # Get scan record + scan = session.query(Scan).filter_by(id=scan_id).first() + if not scan: + logger.warning(f"Scan not found for progress check: {scan_id}") + return jsonify({ + 'error': 'Not found', + 'message': f'Scan with ID {scan_id} not found' + }), 404 + + # Get progress entries + progress_entries = session.query(ScanProgress).filter_by(scan_id=scan_id).all() + + # Build progress data + entries = [] + for entry in progress_entries: + entry_data = { + 'ip_address': entry.ip_address, + 'site_name': entry.site_name, + 'phase': entry.phase, + 'status': entry.status, + 'ping_result': entry.ping_result + } + + # Parse JSON fields + if entry.tcp_ports: + entry_data['tcp_ports'] = json.loads(entry.tcp_ports) + else: + entry_data['tcp_ports'] = [] + + if entry.udp_ports: + entry_data['udp_ports'] = json.loads(entry.udp_ports) + else: + entry_data['udp_ports'] = [] + + if entry.services: + entry_data['services'] = json.loads(entry.services) + else: + entry_data['services'] = [] + + entries.append(entry_data) + + # Sort entries by site name then IP (numerically) + def ip_sort_key(ip_str): + """Convert IP to tuple of integers for proper numeric sorting.""" + try: + return tuple(int(octet) for octet in ip_str.split('.')) + except (ValueError, AttributeError): + return (0, 0, 0, 0) + + entries.sort(key=lambda x: (x['site_name'] or '', ip_sort_key(x['ip_address']))) + + response = { + 'scan_id': scan_id, + 'status': scan.status, + 'current_phase': scan.current_phase or 'pending', + 'total_ips': scan.total_ips or 0, + 'completed_ips': scan.completed_ips or 0, + 'progress_entries': entries + } + + logger.debug(f"Retrieved progress for scan {scan_id}: phase={scan.current_phase}, {scan.completed_ips}/{scan.total_ips} IPs") + return jsonify(response) + + except SQLAlchemyError as e: + logger.error(f"Database error retrieving scan progress {scan_id}: {str(e)}") + return jsonify({ + 'error': 'Database error', + 'message': 'Failed to retrieve scan progress' + }), 500 + except Exception as e: + logger.error(f"Unexpected error retrieving scan progress {scan_id}: {str(e)}", exc_info=True) + return jsonify({ + 'error': 'Internal server error', + 'message': 'An unexpected error occurred' + }), 500 + + @bp.route('/by-ip/', methods=['GET']) @api_auth_required def get_scans_by_ip(ip_address): diff --git a/app/web/jobs/scan_job.py b/app/web/jobs/scan_job.py index d7fcb72..66d36e5 100644 --- a/app/web/jobs/scan_job.py +++ b/app/web/jobs/scan_job.py @@ -5,6 +5,7 @@ This module handles the execution of scans in background threads, updating database status and handling errors. """ +import json import logging import traceback from datetime import datetime @@ -14,13 +15,132 @@ from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker from src.scanner import SneakyScanner -from web.models import Scan +from web.models import Scan, ScanProgress from web.services.scan_service import ScanService from web.services.alert_service import AlertService logger = logging.getLogger(__name__) +def create_progress_callback(scan_id: int, session): + """ + Create a progress callback function for updating scan progress in database. + + Args: + scan_id: ID of the scan record + session: Database session + + Returns: + Callback function that accepts (phase, ip, data) + """ + ip_to_site = {} + + def progress_callback(phase: str, ip: str, data: dict): + """Update scan progress in database.""" + nonlocal ip_to_site + + try: + # Get scan record + scan = session.query(Scan).filter_by(id=scan_id).first() + if not scan: + return + + # Handle initialization phase + if phase == 'init': + scan.total_ips = data.get('total_ips', 0) + scan.completed_ips = 0 + scan.current_phase = 'ping' + ip_to_site = data.get('ip_to_site', {}) + + # Create progress entries for all IPs + for ip_addr, site_name in ip_to_site.items(): + progress = ScanProgress( + scan_id=scan_id, + ip_address=ip_addr, + site_name=site_name, + phase='pending', + status='pending' + ) + session.add(progress) + + session.commit() + return + + # Update current phase + if data.get('status') == 'starting': + scan.current_phase = phase + scan.completed_ips = 0 + session.commit() + return + + # Handle phase completion with results + if data.get('status') == 'completed': + results = data.get('results', {}) + + if phase == 'ping': + # Update progress entries with ping results + for ip_addr, ping_result in results.items(): + progress = session.query(ScanProgress).filter_by( + scan_id=scan_id, ip_address=ip_addr + ).first() + if progress: + progress.ping_result = ping_result + progress.phase = 'ping' + progress.status = 'completed' + + scan.completed_ips = len(results) + + elif phase == 'tcp_scan': + # Update progress entries with TCP/UDP port results + for ip_addr, port_data in results.items(): + progress = session.query(ScanProgress).filter_by( + scan_id=scan_id, ip_address=ip_addr + ).first() + if progress: + progress.tcp_ports = json.dumps(port_data.get('tcp_ports', [])) + progress.udp_ports = json.dumps(port_data.get('udp_ports', [])) + progress.phase = 'tcp_scan' + progress.status = 'completed' + + scan.completed_ips = len(results) + + elif phase == 'service_detection': + # Update progress entries with service detection results + for ip_addr, services in results.items(): + progress = session.query(ScanProgress).filter_by( + scan_id=scan_id, ip_address=ip_addr + ).first() + if progress: + # Simplify service data for storage + service_list = [] + for svc in services: + service_list.append({ + 'port': svc.get('port'), + 'service': svc.get('service', 'unknown'), + 'product': svc.get('product', ''), + 'version': svc.get('version', '') + }) + progress.services = json.dumps(service_list) + progress.phase = 'service_detection' + progress.status = 'completed' + + scan.completed_ips = len(results) + + elif phase == 'http_analysis': + # Mark HTTP analysis as complete + scan.current_phase = 'completed' + scan.completed_ips = scan.total_ips + + session.commit() + + except Exception as e: + logger.error(f"Progress callback error for scan {scan_id}: {str(e)}") + # Don't re-raise - we don't want to break the scan + session.rollback() + + return progress_callback + + def execute_scan(scan_id: int, config_id: int, db_url: str = None): """ Execute a scan in the background. @@ -66,10 +186,13 @@ def execute_scan(scan_id: int, config_id: int, db_url: str = None): # Initialize scanner with database config scanner = SneakyScanner(config_id=config_id) - # Execute scan + # Create progress callback + progress_callback = create_progress_callback(scan_id, session) + + # Execute scan with progress tracking logger.info(f"Scan {scan_id}: Running scanner...") start_time = datetime.utcnow() - report, timestamp = scanner.scan() + report, timestamp = scanner.scan(progress_callback=progress_callback) end_time = datetime.utcnow() scan_duration = (end_time - start_time).total_seconds() diff --git a/app/web/models.py b/app/web/models.py index caa543a..feeb130 100644 --- a/app/web/models.py +++ b/app/web/models.py @@ -59,6 +59,11 @@ class Scan(Base): completed_at = Column(DateTime, nullable=True, comment="Scan execution completion time") error_message = Column(Text, nullable=True, comment="Error message if scan failed") + # Progress tracking fields + current_phase = Column(String(50), nullable=True, comment="Current scan phase: ping, tcp_scan, udp_scan, service_detection, http_analysis") + total_ips = Column(Integer, nullable=True, comment="Total number of IPs to scan") + completed_ips = Column(Integer, nullable=True, default=0, comment="Number of IPs completed in current phase") + # Relationships sites = relationship('ScanSite', back_populates='scan', cascade='all, delete-orphan') ips = relationship('ScanIP', back_populates='scan', cascade='all, delete-orphan') @@ -70,6 +75,7 @@ class Scan(Base): schedule = relationship('Schedule', back_populates='scans') config = relationship('ScanConfig', back_populates='scans') site_associations = relationship('ScanSiteAssociation', back_populates='scan', cascade='all, delete-orphan') + progress_entries = relationship('ScanProgress', back_populates='scan', cascade='all, delete-orphan') def __repr__(self): return f"" @@ -244,6 +250,43 @@ class ScanTLSVersion(Base): return f"" +class ScanProgress(Base): + """ + Real-time progress tracking for individual IPs during scan execution. + + Stores intermediate results as they become available, allowing users to + see progress and results before the full scan completes. + """ + __tablename__ = 'scan_progress' + + id = Column(Integer, primary_key=True, autoincrement=True) + scan_id = Column(Integer, ForeignKey('scans.id'), nullable=False, index=True) + ip_address = Column(String(45), nullable=False, comment="IP address being scanned") + site_name = Column(String(255), nullable=True, comment="Site name this IP belongs to") + phase = Column(String(50), nullable=False, comment="Phase: ping, tcp_scan, udp_scan, service_detection, http_analysis") + status = Column(String(20), nullable=False, default='pending', comment="pending, in_progress, completed, failed") + + # Results data (stored as JSON) + ping_result = Column(Boolean, nullable=True, comment="Ping response result") + tcp_ports = Column(Text, nullable=True, comment="JSON array of discovered TCP ports") + udp_ports = Column(Text, nullable=True, comment="JSON array of discovered UDP ports") + services = Column(Text, nullable=True, comment="JSON array of detected services") + + created_at = Column(DateTime, nullable=False, default=datetime.utcnow, comment="Entry creation time") + updated_at = Column(DateTime, nullable=False, default=datetime.utcnow, onupdate=datetime.utcnow, comment="Last update time") + + # Relationships + scan = relationship('Scan', back_populates='progress_entries') + + # Index for efficient lookups + __table_args__ = ( + UniqueConstraint('scan_id', 'ip_address', name='uix_scan_progress_ip'), + ) + + def __repr__(self): + return f"" + + # ============================================================================ # Reusable Site Definition Tables # ============================================================================ diff --git a/app/web/templates/scan_detail.html b/app/web/templates/scan_detail.html index a0e9096..98f9157 100644 --- a/app/web/templates/scan_detail.html +++ b/app/web/templates/scan_detail.html @@ -84,6 +84,50 @@ + + +
@@ -222,6 +266,7 @@ const scanId = {{ scan_id }}; let scanData = null; let historyChart = null; // Store chart instance to prevent duplicates + let progressInterval = null; // Store progress polling interval // Show alert notification function showAlert(type, message) { @@ -247,16 +292,136 @@ loadScan().then(() => { findPreviousScan(); loadHistoricalChart(); + + // Start progress polling if scan is running + if (scanData && scanData.status === 'running') { + startProgressPolling(); + } }); - // Auto-refresh every 10 seconds if scan is running - setInterval(function() { - if (scanData && scanData.status === 'running') { - loadScan(); - } - }, 10000); }); + // Start polling for progress updates + function startProgressPolling() { + // Show progress section + document.getElementById('progress-section').style.display = 'block'; + + // Initial load + loadProgress(); + + // Poll every 3 seconds + progressInterval = setInterval(loadProgress, 3000); + } + + // Stop polling for progress updates + function stopProgressPolling() { + if (progressInterval) { + clearInterval(progressInterval); + progressInterval = null; + } + // Hide progress section when scan completes + document.getElementById('progress-section').style.display = 'none'; + } + + // Load progress data + async function loadProgress() { + try { + const response = await fetch(`/api/scans/${scanId}/progress`); + if (!response.ok) return; + + const progress = await response.json(); + + // Check if scan is still running + if (progress.status !== 'running') { + stopProgressPolling(); + loadScan(); // Refresh full scan data + return; + } + + renderProgress(progress); + } catch (error) { + console.error('Error loading progress:', error); + } + } + + // Render progress data + function renderProgress(progress) { + // Update phase display + const phaseNames = { + 'pending': 'Initializing', + 'ping': 'Ping Scan', + 'tcp_scan': 'TCP Port Scan', + 'udp_scan': 'UDP Port Scan', + 'service_detection': 'Service Detection', + 'http_analysis': 'HTTP/HTTPS Analysis', + 'completed': 'Completing' + }; + + const phaseName = phaseNames[progress.current_phase] || progress.current_phase; + document.getElementById('current-phase').textContent = phaseName; + + // Update progress count and bar + const total = progress.total_ips || 0; + const completed = progress.completed_ips || 0; + const percent = total > 0 ? Math.round((completed / total) * 100) : 0; + + document.getElementById('progress-count').textContent = `${completed} / ${total} IPs`; + document.getElementById('progress-bar').style.width = `${percent}%`; + + // Update progress table + const tbody = document.getElementById('progress-table-body'); + const entries = progress.progress_entries || []; + + if (entries.length === 0) { + tbody.innerHTML = 'Waiting for results...'; + return; + } + + let html = ''; + entries.forEach(entry => { + // Ping result + let pingDisplay = '-'; + if (entry.ping_result !== null && entry.ping_result !== undefined) { + pingDisplay = entry.ping_result + ? 'Yes' + : 'No'; + } + + // TCP ports + const tcpPorts = entry.tcp_ports || []; + let tcpDisplay = tcpPorts.length > 0 + ? `${tcpPorts.length} ${tcpPorts.slice(0, 5).join(', ')}${tcpPorts.length > 5 ? '...' : ''}` + : '-'; + + // UDP ports + const udpPorts = entry.udp_ports || []; + let udpDisplay = udpPorts.length > 0 + ? `${udpPorts.length}` + : '-'; + + // Services + const services = entry.services || []; + let svcDisplay = '-'; + if (services.length > 0) { + const svcNames = services.map(s => s.service || 'unknown').slice(0, 3); + svcDisplay = `${services.length} ${svcNames.join(', ')}${services.length > 3 ? '...' : ''}`; + } + + html += ` + + ${entry.site_name || '-'} + ${entry.ip_address} + ${pingDisplay} + ${tcpDisplay} + ${udpDisplay} + ${svcDisplay} + + `; + }); + + tbody.innerHTML = html; + } + // Load scan details async function loadScan() { const loadingEl = document.getElementById('scan-loading');