Add real-time scan progress tracking

- Add ScanProgress model and progress fields to Scan model
- Implement progress callback in scanner to report phase completion
- Update scan_job to write per-IP results to database during execution
- Add /api/scans/<id>/progress endpoint for progress polling
- Add progress section to scan detail page with live updates
- Progress table shows current phase, completion bar, and per-IP results
- Poll every 3 seconds during active scans
- Sort IPs numerically for proper ordering
- Add database migration for new tables/columns

app/web/api/scans.py

@@ -5,11 +5,13 @@ Handles endpoints for triggering scans, listing scan history, and retrieving
 scan results.
 """
 
+import json
 import logging
 from flask import Blueprint, current_app, jsonify, request
 from sqlalchemy.exc import SQLAlchemyError
 
 from web.auth.decorators import api_auth_required
+from web.models import Scan, ScanProgress
 from web.services.scan_service import ScanService
 from web.utils.pagination import validate_page_params
 
@@ -281,6 +283,102 @@ def get_scan_status(scan_id):
         }), 500
 
 
+@bp.route('/<int:scan_id>/progress', methods=['GET'])
+@api_auth_required
+def get_scan_progress(scan_id):
+    """
+    Get detailed progress for a running scan including per-IP results.
+
+    Args:
+        scan_id: Scan ID
+
+    Returns:
+        JSON response with scan progress including:
+        - current_phase: Current scan phase
+        - total_ips: Total IPs being scanned
+        - completed_ips: Number of IPs completed in current phase
+        - progress_entries: List of per-IP progress with discovered results
+    """
+    try:
+        session = current_app.db_session
+
+        # Get scan record
+        scan = session.query(Scan).filter_by(id=scan_id).first()
+        if not scan:
+            logger.warning(f"Scan not found for progress check: {scan_id}")
+            return jsonify({
+                'error': 'Not found',
+                'message': f'Scan with ID {scan_id} not found'
+            }), 404
+
+        # Get progress entries
+        progress_entries = session.query(ScanProgress).filter_by(scan_id=scan_id).all()
+
+        # Build progress data
+        entries = []
+        for entry in progress_entries:
+            entry_data = {
+                'ip_address': entry.ip_address,
+                'site_name': entry.site_name,
+                'phase': entry.phase,
+                'status': entry.status,
+                'ping_result': entry.ping_result
+            }
+
+            # Parse JSON fields
+            if entry.tcp_ports:
+                entry_data['tcp_ports'] = json.loads(entry.tcp_ports)
+            else:
+                entry_data['tcp_ports'] = []
+
+            if entry.udp_ports:
+                entry_data['udp_ports'] = json.loads(entry.udp_ports)
+            else:
+                entry_data['udp_ports'] = []
+
+            if entry.services:
+                entry_data['services'] = json.loads(entry.services)
+            else:
+                entry_data['services'] = []
+
+            entries.append(entry_data)
+
+        # Sort entries by site name then IP (numerically)
+        def ip_sort_key(ip_str):
+            """Convert IP to tuple of integers for proper numeric sorting."""
+            try:
+                return tuple(int(octet) for octet in ip_str.split('.'))
+            except (ValueError, AttributeError):
+                return (0, 0, 0, 0)
+
+        entries.sort(key=lambda x: (x['site_name'] or '', ip_sort_key(x['ip_address'])))
+
+        response = {
+            'scan_id': scan_id,
+            'status': scan.status,
+            'current_phase': scan.current_phase or 'pending',
+            'total_ips': scan.total_ips or 0,
+            'completed_ips': scan.completed_ips or 0,
+            'progress_entries': entries
+        }
+
+        logger.debug(f"Retrieved progress for scan {scan_id}: phase={scan.current_phase}, {scan.completed_ips}/{scan.total_ips} IPs")
+        return jsonify(response)
+
+    except SQLAlchemyError as e:
+        logger.error(f"Database error retrieving scan progress {scan_id}: {str(e)}")
+        return jsonify({
+            'error': 'Database error',
+            'message': 'Failed to retrieve scan progress'
+        }), 500
+    except Exception as e:
+        logger.error(f"Unexpected error retrieving scan progress {scan_id}: {str(e)}", exc_info=True)
+        return jsonify({
+            'error': 'Internal server error',
+            'message': 'An unexpected error occurred'
+        }), 500
+
+
 @bp.route('/by-ip/<ip_address>', methods=['GET'])
 @api_auth_required
 def get_scans_by_ip(ip_address):

app/web/jobs/scan_job.py

@@ -5,6 +5,7 @@ This module handles the execution of scans in background threads,
 updating database status and handling errors.
 """
 
+import json
 import logging
 import traceback
 from datetime import datetime
@@ -14,13 +15,132 @@ from sqlalchemy import create_engine
 from sqlalchemy.orm import sessionmaker
 
 from src.scanner import SneakyScanner
-from web.models import Scan
+from web.models import Scan, ScanProgress
 from web.services.scan_service import ScanService
 from web.services.alert_service import AlertService
 
 logger = logging.getLogger(__name__)
 
 
+def create_progress_callback(scan_id: int, session):
+    """
+    Create a progress callback function for updating scan progress in database.
+
+    Args:
+        scan_id: ID of the scan record
+        session: Database session
+
+    Returns:
+        Callback function that accepts (phase, ip, data)
+    """
+    ip_to_site = {}
+
+    def progress_callback(phase: str, ip: str, data: dict):
+        """Update scan progress in database."""
+        nonlocal ip_to_site
+
+        try:
+            # Get scan record
+            scan = session.query(Scan).filter_by(id=scan_id).first()
+            if not scan:
+                return
+
+            # Handle initialization phase
+            if phase == 'init':
+                scan.total_ips = data.get('total_ips', 0)
+                scan.completed_ips = 0
+                scan.current_phase = 'ping'
+                ip_to_site = data.get('ip_to_site', {})
+
+                # Create progress entries for all IPs
+                for ip_addr, site_name in ip_to_site.items():
+                    progress = ScanProgress(
+                        scan_id=scan_id,
+                        ip_address=ip_addr,
+                        site_name=site_name,
+                        phase='pending',
+                        status='pending'
+                    )
+                    session.add(progress)
+
+                session.commit()
+                return
+
+            # Update current phase
+            if data.get('status') == 'starting':
+                scan.current_phase = phase
+                scan.completed_ips = 0
+                session.commit()
+                return
+
+            # Handle phase completion with results
+            if data.get('status') == 'completed':
+                results = data.get('results', {})
+
+                if phase == 'ping':
+                    # Update progress entries with ping results
+                    for ip_addr, ping_result in results.items():
+                        progress = session.query(ScanProgress).filter_by(
+                            scan_id=scan_id, ip_address=ip_addr
+                        ).first()
+                        if progress:
+                            progress.ping_result = ping_result
+                            progress.phase = 'ping'
+                            progress.status = 'completed'
+
+                    scan.completed_ips = len(results)
+
+                elif phase == 'tcp_scan':
+                    # Update progress entries with TCP/UDP port results
+                    for ip_addr, port_data in results.items():
+                        progress = session.query(ScanProgress).filter_by(
+                            scan_id=scan_id, ip_address=ip_addr
+                        ).first()
+                        if progress:
+                            progress.tcp_ports = json.dumps(port_data.get('tcp_ports', []))
+                            progress.udp_ports = json.dumps(port_data.get('udp_ports', []))
+                            progress.phase = 'tcp_scan'
+                            progress.status = 'completed'
+
+                    scan.completed_ips = len(results)
+
+                elif phase == 'service_detection':
+                    # Update progress entries with service detection results
+                    for ip_addr, services in results.items():
+                        progress = session.query(ScanProgress).filter_by(
+                            scan_id=scan_id, ip_address=ip_addr
+                        ).first()
+                        if progress:
+                            # Simplify service data for storage
+                            service_list = []
+                            for svc in services:
+                                service_list.append({
+                                    'port': svc.get('port'),
+                                    'service': svc.get('service', 'unknown'),
+                                    'product': svc.get('product', ''),
+                                    'version': svc.get('version', '')
+                                })
+                            progress.services = json.dumps(service_list)
+                            progress.phase = 'service_detection'
+                            progress.status = 'completed'
+
+                    scan.completed_ips = len(results)
+
+                elif phase == 'http_analysis':
+                    # Mark HTTP analysis as complete
+                    scan.current_phase = 'completed'
+                    scan.completed_ips = scan.total_ips
+
+                session.commit()
+
+        except Exception as e:
+            logger.error(f"Progress callback error for scan {scan_id}: {str(e)}")
+            # Don't re-raise - we don't want to break the scan
+            session.rollback()
+
+    return progress_callback
+
+
 def execute_scan(scan_id: int, config_id: int, db_url: str = None):
     """
     Execute a scan in the background.
@@ -66,10 +186,13 @@ def execute_scan(scan_id: int, config_id: int, db_url: str = None):
         # Initialize scanner with database config
         scanner = SneakyScanner(config_id=config_id)
 
-        # Execute scan
+        # Create progress callback
+        progress_callback = create_progress_callback(scan_id, session)
+
+        # Execute scan with progress tracking
         logger.info(f"Scan {scan_id}: Running scanner...")
         start_time = datetime.utcnow()
-        report, timestamp = scanner.scan()
+        report, timestamp = scanner.scan(progress_callback=progress_callback)
         end_time = datetime.utcnow()
 
         scan_duration = (end_time - start_time).total_seconds()

app/web/models.py

@@ -59,6 +59,11 @@ class Scan(Base):
     completed_at = Column(DateTime, nullable=True, comment="Scan execution completion time")
     error_message = Column(Text, nullable=True, comment="Error message if scan failed")
 
+    # Progress tracking fields
+    current_phase = Column(String(50), nullable=True, comment="Current scan phase: ping, tcp_scan, udp_scan, service_detection, http_analysis")
+    total_ips = Column(Integer, nullable=True, comment="Total number of IPs to scan")
+    completed_ips = Column(Integer, nullable=True, default=0, comment="Number of IPs completed in current phase")
+
     # Relationships
     sites = relationship('ScanSite', back_populates='scan', cascade='all, delete-orphan')
     ips = relationship('ScanIP', back_populates='scan', cascade='all, delete-orphan')
@@ -70,6 +75,7 @@ class Scan(Base):
     schedule = relationship('Schedule', back_populates='scans')
     config = relationship('ScanConfig', back_populates='scans')
     site_associations = relationship('ScanSiteAssociation', back_populates='scan', cascade='all, delete-orphan')
+    progress_entries = relationship('ScanProgress', back_populates='scan', cascade='all, delete-orphan')
 
     def __repr__(self):
         return f"<Scan(id={self.id}, title='{self.title}', status='{self.status}')>"
@@ -244,6 +250,43 @@ class ScanTLSVersion(Base):
         return f"<ScanTLSVersion(id={self.id}, tls_version='{self.tls_version}', supported={self.supported})>"
 
 
+class ScanProgress(Base):
+    """
+    Real-time progress tracking for individual IPs during scan execution.
+
+    Stores intermediate results as they become available, allowing users to
+    see progress and results before the full scan completes.
+    """
+    __tablename__ = 'scan_progress'
+
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    scan_id = Column(Integer, ForeignKey('scans.id'), nullable=False, index=True)
+    ip_address = Column(String(45), nullable=False, comment="IP address being scanned")
+    site_name = Column(String(255), nullable=True, comment="Site name this IP belongs to")
+    phase = Column(String(50), nullable=False, comment="Phase: ping, tcp_scan, udp_scan, service_detection, http_analysis")
+    status = Column(String(20), nullable=False, default='pending', comment="pending, in_progress, completed, failed")
+
+    # Results data (stored as JSON)
+    ping_result = Column(Boolean, nullable=True, comment="Ping response result")
+    tcp_ports = Column(Text, nullable=True, comment="JSON array of discovered TCP ports")
+    udp_ports = Column(Text, nullable=True, comment="JSON array of discovered UDP ports")
+    services = Column(Text, nullable=True, comment="JSON array of detected services")
+
+    created_at = Column(DateTime, nullable=False, default=datetime.utcnow, comment="Entry creation time")
+    updated_at = Column(DateTime, nullable=False, default=datetime.utcnow, onupdate=datetime.utcnow, comment="Last update time")
+
+    # Relationships
+    scan = relationship('Scan', back_populates='progress_entries')
+
+    # Index for efficient lookups
+    __table_args__ = (
+        UniqueConstraint('scan_id', 'ip_address', name='uix_scan_progress_ip'),
+    )
+
+    def __repr__(self):
+        return f"<ScanProgress(id={self.id}, ip='{self.ip_address}', phase='{self.phase}', status='{self.status}')>"
+
+
 # ============================================================================
 # Reusable Site Definition Tables
 # ============================================================================

app/web/templates/scan_detail.html

@@ -84,6 +84,50 @@
         </div>
     </div>
 
+    <!-- Progress Section (shown when scan is running) -->
+    <div class="row mb-4" id="progress-section" style="display: none;">
+        <div class="col-12">
+            <div class="card">
+                <div class="card-header">
+                    <h5 class="mb-0" style="color: #60a5fa;">
+                        <i class="bi bi-hourglass-split"></i> Scan Progress
+                    </h5>
+                </div>
+                <div class="card-body">
+                    <!-- Phase and Progress Bar -->
+                    <div class="mb-3">
+                        <div class="d-flex justify-content-between align-items-center mb-2">
+                            <span>Current Phase: <strong id="current-phase">Initializing...</strong></span>
+                            <span id="progress-count">0 / 0 IPs</span>
+                        </div>
+                        <div class="progress" style="height: 20px; background-color: #334155;">
+                            <div id="progress-bar" class="progress-bar bg-info" role="progressbar" style="width: 0%"></div>
+                        </div>
+                    </div>
+
+                    <!-- Per-IP Results Table -->
+                    <div class="table-responsive" style="max-height: 400px; overflow-y: auto;">
+                        <table class="table table-sm">
+                            <thead style="position: sticky; top: 0; background-color: #1e293b;">
+                                <tr>
+                                    <th>Site</th>
+                                    <th>IP Address</th>
+                                    <th>Ping</th>
+                                    <th>TCP Ports</th>
+                                    <th>UDP Ports</th>
+                                    <th>Services</th>
+                                </tr>
+                            </thead>
+                            <tbody id="progress-table-body">
+                                <tr><td colspan="6" class="text-center text-muted">Waiting for results...</td></tr>
+                            </tbody>
+                        </table>
+                    </div>
+                </div>
+            </div>
+        </div>
+    </div>
+
     <!-- Stats Row -->
     <div class="row mb-4">
         <div class="col-md-3">
@@ -222,6 +266,7 @@
     const scanId = {{ scan_id }};
     let scanData = null;
     let historyChart = null; // Store chart instance to prevent duplicates
+    let progressInterval = null; // Store progress polling interval
 
     // Show alert notification
     function showAlert(type, message) {
@@ -247,16 +292,136 @@
         loadScan().then(() => {
             findPreviousScan();
             loadHistoricalChart();
+
+            // Start progress polling if scan is running
+            if (scanData && scanData.status === 'running') {
+                startProgressPolling();
+            }
         });
 
-        // Auto-refresh every 10 seconds if scan is running
-        setInterval(function() {
-            if (scanData && scanData.status === 'running') {
-                loadScan();
-            }
-        }, 10000);
     });
 
+    // Start polling for progress updates
+    function startProgressPolling() {
+        // Show progress section
+        document.getElementById('progress-section').style.display = 'block';
+
+        // Initial load
+        loadProgress();
+
+        // Poll every 3 seconds
+        progressInterval = setInterval(loadProgress, 3000);
+    }
+
+    // Stop polling for progress updates
+    function stopProgressPolling() {
+        if (progressInterval) {
+            clearInterval(progressInterval);
+            progressInterval = null;
+        }
+        // Hide progress section when scan completes
+        document.getElementById('progress-section').style.display = 'none';
+    }
+
+    // Load progress data
+    async function loadProgress() {
+        try {
+            const response = await fetch(`/api/scans/${scanId}/progress`);
+            if (!response.ok) return;
+
+            const progress = await response.json();
+
+            // Check if scan is still running
+            if (progress.status !== 'running') {
+                stopProgressPolling();
+                loadScan(); // Refresh full scan data
+                return;
+            }
+
+            renderProgress(progress);
+        } catch (error) {
+            console.error('Error loading progress:', error);
+        }
+    }
+
+    // Render progress data
+    function renderProgress(progress) {
+        // Update phase display
+        const phaseNames = {
+            'pending': 'Initializing',
+            'ping': 'Ping Scan',
+            'tcp_scan': 'TCP Port Scan',
+            'udp_scan': 'UDP Port Scan',
+            'service_detection': 'Service Detection',
+            'http_analysis': 'HTTP/HTTPS Analysis',
+            'completed': 'Completing'
+        };
+
+        const phaseName = phaseNames[progress.current_phase] || progress.current_phase;
+        document.getElementById('current-phase').textContent = phaseName;
+
+        // Update progress count and bar
+        const total = progress.total_ips || 0;
+        const completed = progress.completed_ips || 0;
+        const percent = total > 0 ? Math.round((completed / total) * 100) : 0;
+
+        document.getElementById('progress-count').textContent = `${completed} / ${total} IPs`;
+        document.getElementById('progress-bar').style.width = `${percent}%`;
+
+        // Update progress table
+        const tbody = document.getElementById('progress-table-body');
+        const entries = progress.progress_entries || [];
+
+        if (entries.length === 0) {
+            tbody.innerHTML = '<tr><td colspan="6" class="text-center text-muted">Waiting for results...</td></tr>';
+            return;
+        }
+
+        let html = '';
+        entries.forEach(entry => {
+            // Ping result
+            let pingDisplay = '-';
+            if (entry.ping_result !== null && entry.ping_result !== undefined) {
+                pingDisplay = entry.ping_result
+                    ? '<span class="badge badge-success">Yes</span>'
+                    : '<span class="badge badge-danger">No</span>';
+            }
+
+            // TCP ports
+            const tcpPorts = entry.tcp_ports || [];
+            let tcpDisplay = tcpPorts.length > 0
+                ? `<span class="badge bg-info">${tcpPorts.length}</span> <small class="text-muted">${tcpPorts.slice(0, 5).join(', ')}${tcpPorts.length > 5 ? '...' : ''}</small>`
+                : '-';
+
+            // UDP ports
+            const udpPorts = entry.udp_ports || [];
+            let udpDisplay = udpPorts.length > 0
+                ? `<span class="badge bg-info">${udpPorts.length}</span>`
+                : '-';
+
+            // Services
+            const services = entry.services || [];
+            let svcDisplay = '-';
+            if (services.length > 0) {
+                const svcNames = services.map(s => s.service || 'unknown').slice(0, 3);
+                svcDisplay = `<span class="badge bg-info">${services.length}</span> <small class="text-muted">${svcNames.join(', ')}${services.length > 3 ? '...' : ''}</small>`;
+            }
+
+            html += `
+                <tr class="scan-row">
+                    <td>${entry.site_name || '-'}</td>
+                    <td class="mono">${entry.ip_address}</td>
+                    <td>${pingDisplay}</td>
+                    <td>${tcpDisplay}</td>
+                    <td>${udpDisplay}</td>
+                    <td>${svcDisplay}</td>
+                </tr>
+            `;
+        });
+
+        tbody.innerHTML = html;
+    }
+
     // Load scan details
     async function loadScan() {
         const loadingEl = document.getElementById('scan-loading');