Add real-time scan progress tracking
- Add ScanProgress model and progress fields to Scan model - Implement progress callback in scanner to report phase completion - Update scan_job to write per-IP results to database during execution - Add /api/scans/<id>/progress endpoint for progress polling - Add progress section to scan detail page with live updates - Progress table shows current phase, completion bar, and per-IP results - Poll every 3 seconds during active scans - Sort IPs numerically for proper ordering - Add database migration for new tables/columns
This commit is contained in:
58
app/migrations/versions/012_add_scan_progress.py
Normal file
58
app/migrations/versions/012_add_scan_progress.py
Normal file
@@ -0,0 +1,58 @@
|
|||||||
|
"""Add scan progress tracking
|
||||||
|
|
||||||
|
Revision ID: 012
|
||||||
|
Revises: 011
|
||||||
|
Create Date: 2024-01-01 00:00:00.000000
|
||||||
|
|
||||||
|
"""
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
|
||||||
|
# revision identifiers, used by Alembic.
|
||||||
|
revision = '012'
|
||||||
|
down_revision = '011'
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade():
|
||||||
|
# Add progress tracking columns to scans table
|
||||||
|
op.add_column('scans', sa.Column('current_phase', sa.String(50), nullable=True,
|
||||||
|
comment='Current scan phase: ping, tcp_scan, udp_scan, service_detection, http_analysis'))
|
||||||
|
op.add_column('scans', sa.Column('total_ips', sa.Integer(), nullable=True,
|
||||||
|
comment='Total number of IPs to scan'))
|
||||||
|
op.add_column('scans', sa.Column('completed_ips', sa.Integer(), nullable=True, default=0,
|
||||||
|
comment='Number of IPs completed in current phase'))
|
||||||
|
|
||||||
|
# Create scan_progress table for per-IP progress tracking
|
||||||
|
op.create_table(
|
||||||
|
'scan_progress',
|
||||||
|
sa.Column('id', sa.Integer(), primary_key=True, autoincrement=True),
|
||||||
|
sa.Column('scan_id', sa.Integer(), sa.ForeignKey('scans.id'), nullable=False, index=True),
|
||||||
|
sa.Column('ip_address', sa.String(45), nullable=False, comment='IP address being scanned'),
|
||||||
|
sa.Column('site_name', sa.String(255), nullable=True, comment='Site name this IP belongs to'),
|
||||||
|
sa.Column('phase', sa.String(50), nullable=False,
|
||||||
|
comment='Phase: ping, tcp_scan, udp_scan, service_detection, http_analysis'),
|
||||||
|
sa.Column('status', sa.String(20), nullable=False, default='pending',
|
||||||
|
comment='pending, in_progress, completed, failed'),
|
||||||
|
sa.Column('ping_result', sa.Boolean(), nullable=True, comment='Ping response result'),
|
||||||
|
sa.Column('tcp_ports', sa.Text(), nullable=True, comment='JSON array of discovered TCP ports'),
|
||||||
|
sa.Column('udp_ports', sa.Text(), nullable=True, comment='JSON array of discovered UDP ports'),
|
||||||
|
sa.Column('services', sa.Text(), nullable=True, comment='JSON array of detected services'),
|
||||||
|
sa.Column('created_at', sa.DateTime(), nullable=False, server_default=sa.func.now(),
|
||||||
|
comment='Entry creation time'),
|
||||||
|
sa.Column('updated_at', sa.DateTime(), nullable=False, server_default=sa.func.now(),
|
||||||
|
onupdate=sa.func.now(), comment='Last update time'),
|
||||||
|
sa.UniqueConstraint('scan_id', 'ip_address', name='uix_scan_progress_ip')
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade():
|
||||||
|
# Drop scan_progress table
|
||||||
|
op.drop_table('scan_progress')
|
||||||
|
|
||||||
|
# Remove progress tracking columns from scans table
|
||||||
|
op.drop_column('scans', 'completed_ips')
|
||||||
|
op.drop_column('scans', 'total_ips')
|
||||||
|
op.drop_column('scans', 'current_phase')
|
||||||
@@ -13,7 +13,7 @@ import time
|
|||||||
import zipfile
|
import zipfile
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Dict, List, Any
|
from typing import Dict, List, Any, Callable, Optional
|
||||||
import xml.etree.ElementTree as ET
|
import xml.etree.ElementTree as ET
|
||||||
|
|
||||||
import yaml
|
import yaml
|
||||||
@@ -833,10 +833,17 @@ class SneakyScanner:
|
|||||||
|
|
||||||
return all_results
|
return all_results
|
||||||
|
|
||||||
def scan(self) -> Dict[str, Any]:
|
def scan(self, progress_callback: Optional[Callable] = None) -> Dict[str, Any]:
|
||||||
"""
|
"""
|
||||||
Perform complete scan based on configuration
|
Perform complete scan based on configuration
|
||||||
|
|
||||||
|
Args:
|
||||||
|
progress_callback: Optional callback function for progress updates.
|
||||||
|
Called with (phase, ip, data) where:
|
||||||
|
- phase: 'init', 'ping', 'tcp_scan', 'udp_scan', 'service_detection', 'http_analysis'
|
||||||
|
- ip: IP address being processed (or None for phase start)
|
||||||
|
- data: Dict with progress data (results, counts, etc.)
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Dictionary containing scan results
|
Dictionary containing scan results
|
||||||
"""
|
"""
|
||||||
@@ -873,16 +880,36 @@ class SneakyScanner:
|
|||||||
all_ips = sorted(list(all_ips))
|
all_ips = sorted(list(all_ips))
|
||||||
print(f"Total IPs to scan: {len(all_ips)}", flush=True)
|
print(f"Total IPs to scan: {len(all_ips)}", flush=True)
|
||||||
|
|
||||||
|
# Report initialization with total IP count
|
||||||
|
if progress_callback:
|
||||||
|
progress_callback('init', None, {
|
||||||
|
'total_ips': len(all_ips),
|
||||||
|
'ip_to_site': ip_to_site
|
||||||
|
})
|
||||||
|
|
||||||
# Perform ping scan
|
# Perform ping scan
|
||||||
print(f"\n[1/5] Performing ping scan on {len(all_ips)} IPs...", flush=True)
|
print(f"\n[1/5] Performing ping scan on {len(all_ips)} IPs...", flush=True)
|
||||||
|
if progress_callback:
|
||||||
|
progress_callback('ping', None, {'status': 'starting'})
|
||||||
ping_results = self._run_ping_scan(all_ips)
|
ping_results = self._run_ping_scan(all_ips)
|
||||||
|
|
||||||
|
# Report ping results
|
||||||
|
if progress_callback:
|
||||||
|
progress_callback('ping', None, {
|
||||||
|
'status': 'completed',
|
||||||
|
'results': ping_results
|
||||||
|
})
|
||||||
|
|
||||||
# Perform TCP scan (all ports)
|
# Perform TCP scan (all ports)
|
||||||
print(f"\n[2/5] Performing TCP scan on {len(all_ips)} IPs (ports 0-65535)...", flush=True)
|
print(f"\n[2/5] Performing TCP scan on {len(all_ips)} IPs (ports 0-65535)...", flush=True)
|
||||||
|
if progress_callback:
|
||||||
|
progress_callback('tcp_scan', None, {'status': 'starting'})
|
||||||
tcp_results = self._run_masscan(all_ips, '0-65535', 'tcp')
|
tcp_results = self._run_masscan(all_ips, '0-65535', 'tcp')
|
||||||
|
|
||||||
# Perform UDP scan (all ports)
|
# Perform UDP scan (all ports)
|
||||||
print(f"\n[3/5] Performing UDP scan on {len(all_ips)} IPs (ports 0-65535)...", flush=True)
|
print(f"\n[3/5] Performing UDP scan on {len(all_ips)} IPs (ports 0-65535)...", flush=True)
|
||||||
|
if progress_callback:
|
||||||
|
progress_callback('udp_scan', None, {'status': 'starting'})
|
||||||
udp_results = self._run_masscan(all_ips, '0-65535', 'udp')
|
udp_results = self._run_masscan(all_ips, '0-65535', 'udp')
|
||||||
|
|
||||||
# Organize results by IP
|
# Organize results by IP
|
||||||
@@ -918,8 +945,23 @@ class SneakyScanner:
|
|||||||
results_by_ip[ip]['actual']['tcp_ports'].sort()
|
results_by_ip[ip]['actual']['tcp_ports'].sort()
|
||||||
results_by_ip[ip]['actual']['udp_ports'].sort()
|
results_by_ip[ip]['actual']['udp_ports'].sort()
|
||||||
|
|
||||||
|
# Report TCP/UDP scan results with discovered ports per IP
|
||||||
|
if progress_callback:
|
||||||
|
tcp_udp_results = {}
|
||||||
|
for ip in all_ips:
|
||||||
|
tcp_udp_results[ip] = {
|
||||||
|
'tcp_ports': results_by_ip[ip]['actual']['tcp_ports'],
|
||||||
|
'udp_ports': results_by_ip[ip]['actual']['udp_ports']
|
||||||
|
}
|
||||||
|
progress_callback('tcp_scan', None, {
|
||||||
|
'status': 'completed',
|
||||||
|
'results': tcp_udp_results
|
||||||
|
})
|
||||||
|
|
||||||
# Perform service detection on TCP ports
|
# Perform service detection on TCP ports
|
||||||
print(f"\n[4/5] Performing service detection on discovered TCP ports...", flush=True)
|
print(f"\n[4/5] Performing service detection on discovered TCP ports...", flush=True)
|
||||||
|
if progress_callback:
|
||||||
|
progress_callback('service_detection', None, {'status': 'starting'})
|
||||||
ip_ports = {ip: results_by_ip[ip]['actual']['tcp_ports'] for ip in all_ips}
|
ip_ports = {ip: results_by_ip[ip]['actual']['tcp_ports'] for ip in all_ips}
|
||||||
service_results = self._run_nmap_service_detection(ip_ports)
|
service_results = self._run_nmap_service_detection(ip_ports)
|
||||||
|
|
||||||
@@ -928,10 +970,26 @@ class SneakyScanner:
|
|||||||
if ip in results_by_ip:
|
if ip in results_by_ip:
|
||||||
results_by_ip[ip]['actual']['services'] = services
|
results_by_ip[ip]['actual']['services'] = services
|
||||||
|
|
||||||
|
# Report service detection results
|
||||||
|
if progress_callback:
|
||||||
|
progress_callback('service_detection', None, {
|
||||||
|
'status': 'completed',
|
||||||
|
'results': service_results
|
||||||
|
})
|
||||||
|
|
||||||
# Perform HTTP/HTTPS analysis on web services
|
# Perform HTTP/HTTPS analysis on web services
|
||||||
print(f"\n[5/5] Analyzing HTTP/HTTPS services and SSL/TLS configuration...", flush=True)
|
print(f"\n[5/5] Analyzing HTTP/HTTPS services and SSL/TLS configuration...", flush=True)
|
||||||
|
if progress_callback:
|
||||||
|
progress_callback('http_analysis', None, {'status': 'starting'})
|
||||||
http_results = self._run_http_analysis(service_results)
|
http_results = self._run_http_analysis(service_results)
|
||||||
|
|
||||||
|
# Report HTTP analysis completion
|
||||||
|
if progress_callback:
|
||||||
|
progress_callback('http_analysis', None, {
|
||||||
|
'status': 'completed',
|
||||||
|
'results': http_results
|
||||||
|
})
|
||||||
|
|
||||||
# Merge HTTP analysis into service results
|
# Merge HTTP analysis into service results
|
||||||
for ip, port_results in http_results.items():
|
for ip, port_results in http_results.items():
|
||||||
if ip in results_by_ip:
|
if ip in results_by_ip:
|
||||||
|
|||||||
@@ -5,11 +5,13 @@ Handles endpoints for triggering scans, listing scan history, and retrieving
|
|||||||
scan results.
|
scan results.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
import logging
|
import logging
|
||||||
from flask import Blueprint, current_app, jsonify, request
|
from flask import Blueprint, current_app, jsonify, request
|
||||||
from sqlalchemy.exc import SQLAlchemyError
|
from sqlalchemy.exc import SQLAlchemyError
|
||||||
|
|
||||||
from web.auth.decorators import api_auth_required
|
from web.auth.decorators import api_auth_required
|
||||||
|
from web.models import Scan, ScanProgress
|
||||||
from web.services.scan_service import ScanService
|
from web.services.scan_service import ScanService
|
||||||
from web.utils.pagination import validate_page_params
|
from web.utils.pagination import validate_page_params
|
||||||
|
|
||||||
@@ -281,6 +283,102 @@ def get_scan_status(scan_id):
|
|||||||
}), 500
|
}), 500
|
||||||
|
|
||||||
|
|
||||||
|
@bp.route('/<int:scan_id>/progress', methods=['GET'])
|
||||||
|
@api_auth_required
|
||||||
|
def get_scan_progress(scan_id):
|
||||||
|
"""
|
||||||
|
Get detailed progress for a running scan including per-IP results.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
scan_id: Scan ID
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
JSON response with scan progress including:
|
||||||
|
- current_phase: Current scan phase
|
||||||
|
- total_ips: Total IPs being scanned
|
||||||
|
- completed_ips: Number of IPs completed in current phase
|
||||||
|
- progress_entries: List of per-IP progress with discovered results
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
session = current_app.db_session
|
||||||
|
|
||||||
|
# Get scan record
|
||||||
|
scan = session.query(Scan).filter_by(id=scan_id).first()
|
||||||
|
if not scan:
|
||||||
|
logger.warning(f"Scan not found for progress check: {scan_id}")
|
||||||
|
return jsonify({
|
||||||
|
'error': 'Not found',
|
||||||
|
'message': f'Scan with ID {scan_id} not found'
|
||||||
|
}), 404
|
||||||
|
|
||||||
|
# Get progress entries
|
||||||
|
progress_entries = session.query(ScanProgress).filter_by(scan_id=scan_id).all()
|
||||||
|
|
||||||
|
# Build progress data
|
||||||
|
entries = []
|
||||||
|
for entry in progress_entries:
|
||||||
|
entry_data = {
|
||||||
|
'ip_address': entry.ip_address,
|
||||||
|
'site_name': entry.site_name,
|
||||||
|
'phase': entry.phase,
|
||||||
|
'status': entry.status,
|
||||||
|
'ping_result': entry.ping_result
|
||||||
|
}
|
||||||
|
|
||||||
|
# Parse JSON fields
|
||||||
|
if entry.tcp_ports:
|
||||||
|
entry_data['tcp_ports'] = json.loads(entry.tcp_ports)
|
||||||
|
else:
|
||||||
|
entry_data['tcp_ports'] = []
|
||||||
|
|
||||||
|
if entry.udp_ports:
|
||||||
|
entry_data['udp_ports'] = json.loads(entry.udp_ports)
|
||||||
|
else:
|
||||||
|
entry_data['udp_ports'] = []
|
||||||
|
|
||||||
|
if entry.services:
|
||||||
|
entry_data['services'] = json.loads(entry.services)
|
||||||
|
else:
|
||||||
|
entry_data['services'] = []
|
||||||
|
|
||||||
|
entries.append(entry_data)
|
||||||
|
|
||||||
|
# Sort entries by site name then IP (numerically)
|
||||||
|
def ip_sort_key(ip_str):
|
||||||
|
"""Convert IP to tuple of integers for proper numeric sorting."""
|
||||||
|
try:
|
||||||
|
return tuple(int(octet) for octet in ip_str.split('.'))
|
||||||
|
except (ValueError, AttributeError):
|
||||||
|
return (0, 0, 0, 0)
|
||||||
|
|
||||||
|
entries.sort(key=lambda x: (x['site_name'] or '', ip_sort_key(x['ip_address'])))
|
||||||
|
|
||||||
|
response = {
|
||||||
|
'scan_id': scan_id,
|
||||||
|
'status': scan.status,
|
||||||
|
'current_phase': scan.current_phase or 'pending',
|
||||||
|
'total_ips': scan.total_ips or 0,
|
||||||
|
'completed_ips': scan.completed_ips or 0,
|
||||||
|
'progress_entries': entries
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.debug(f"Retrieved progress for scan {scan_id}: phase={scan.current_phase}, {scan.completed_ips}/{scan.total_ips} IPs")
|
||||||
|
return jsonify(response)
|
||||||
|
|
||||||
|
except SQLAlchemyError as e:
|
||||||
|
logger.error(f"Database error retrieving scan progress {scan_id}: {str(e)}")
|
||||||
|
return jsonify({
|
||||||
|
'error': 'Database error',
|
||||||
|
'message': 'Failed to retrieve scan progress'
|
||||||
|
}), 500
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Unexpected error retrieving scan progress {scan_id}: {str(e)}", exc_info=True)
|
||||||
|
return jsonify({
|
||||||
|
'error': 'Internal server error',
|
||||||
|
'message': 'An unexpected error occurred'
|
||||||
|
}), 500
|
||||||
|
|
||||||
|
|
||||||
@bp.route('/by-ip/<ip_address>', methods=['GET'])
|
@bp.route('/by-ip/<ip_address>', methods=['GET'])
|
||||||
@api_auth_required
|
@api_auth_required
|
||||||
def get_scans_by_ip(ip_address):
|
def get_scans_by_ip(ip_address):
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ This module handles the execution of scans in background threads,
|
|||||||
updating database status and handling errors.
|
updating database status and handling errors.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
import logging
|
import logging
|
||||||
import traceback
|
import traceback
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
@@ -14,13 +15,132 @@ from sqlalchemy import create_engine
|
|||||||
from sqlalchemy.orm import sessionmaker
|
from sqlalchemy.orm import sessionmaker
|
||||||
|
|
||||||
from src.scanner import SneakyScanner
|
from src.scanner import SneakyScanner
|
||||||
from web.models import Scan
|
from web.models import Scan, ScanProgress
|
||||||
from web.services.scan_service import ScanService
|
from web.services.scan_service import ScanService
|
||||||
from web.services.alert_service import AlertService
|
from web.services.alert_service import AlertService
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def create_progress_callback(scan_id: int, session):
|
||||||
|
"""
|
||||||
|
Create a progress callback function for updating scan progress in database.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
scan_id: ID of the scan record
|
||||||
|
session: Database session
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Callback function that accepts (phase, ip, data)
|
||||||
|
"""
|
||||||
|
ip_to_site = {}
|
||||||
|
|
||||||
|
def progress_callback(phase: str, ip: str, data: dict):
|
||||||
|
"""Update scan progress in database."""
|
||||||
|
nonlocal ip_to_site
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Get scan record
|
||||||
|
scan = session.query(Scan).filter_by(id=scan_id).first()
|
||||||
|
if not scan:
|
||||||
|
return
|
||||||
|
|
||||||
|
# Handle initialization phase
|
||||||
|
if phase == 'init':
|
||||||
|
scan.total_ips = data.get('total_ips', 0)
|
||||||
|
scan.completed_ips = 0
|
||||||
|
scan.current_phase = 'ping'
|
||||||
|
ip_to_site = data.get('ip_to_site', {})
|
||||||
|
|
||||||
|
# Create progress entries for all IPs
|
||||||
|
for ip_addr, site_name in ip_to_site.items():
|
||||||
|
progress = ScanProgress(
|
||||||
|
scan_id=scan_id,
|
||||||
|
ip_address=ip_addr,
|
||||||
|
site_name=site_name,
|
||||||
|
phase='pending',
|
||||||
|
status='pending'
|
||||||
|
)
|
||||||
|
session.add(progress)
|
||||||
|
|
||||||
|
session.commit()
|
||||||
|
return
|
||||||
|
|
||||||
|
# Update current phase
|
||||||
|
if data.get('status') == 'starting':
|
||||||
|
scan.current_phase = phase
|
||||||
|
scan.completed_ips = 0
|
||||||
|
session.commit()
|
||||||
|
return
|
||||||
|
|
||||||
|
# Handle phase completion with results
|
||||||
|
if data.get('status') == 'completed':
|
||||||
|
results = data.get('results', {})
|
||||||
|
|
||||||
|
if phase == 'ping':
|
||||||
|
# Update progress entries with ping results
|
||||||
|
for ip_addr, ping_result in results.items():
|
||||||
|
progress = session.query(ScanProgress).filter_by(
|
||||||
|
scan_id=scan_id, ip_address=ip_addr
|
||||||
|
).first()
|
||||||
|
if progress:
|
||||||
|
progress.ping_result = ping_result
|
||||||
|
progress.phase = 'ping'
|
||||||
|
progress.status = 'completed'
|
||||||
|
|
||||||
|
scan.completed_ips = len(results)
|
||||||
|
|
||||||
|
elif phase == 'tcp_scan':
|
||||||
|
# Update progress entries with TCP/UDP port results
|
||||||
|
for ip_addr, port_data in results.items():
|
||||||
|
progress = session.query(ScanProgress).filter_by(
|
||||||
|
scan_id=scan_id, ip_address=ip_addr
|
||||||
|
).first()
|
||||||
|
if progress:
|
||||||
|
progress.tcp_ports = json.dumps(port_data.get('tcp_ports', []))
|
||||||
|
progress.udp_ports = json.dumps(port_data.get('udp_ports', []))
|
||||||
|
progress.phase = 'tcp_scan'
|
||||||
|
progress.status = 'completed'
|
||||||
|
|
||||||
|
scan.completed_ips = len(results)
|
||||||
|
|
||||||
|
elif phase == 'service_detection':
|
||||||
|
# Update progress entries with service detection results
|
||||||
|
for ip_addr, services in results.items():
|
||||||
|
progress = session.query(ScanProgress).filter_by(
|
||||||
|
scan_id=scan_id, ip_address=ip_addr
|
||||||
|
).first()
|
||||||
|
if progress:
|
||||||
|
# Simplify service data for storage
|
||||||
|
service_list = []
|
||||||
|
for svc in services:
|
||||||
|
service_list.append({
|
||||||
|
'port': svc.get('port'),
|
||||||
|
'service': svc.get('service', 'unknown'),
|
||||||
|
'product': svc.get('product', ''),
|
||||||
|
'version': svc.get('version', '')
|
||||||
|
})
|
||||||
|
progress.services = json.dumps(service_list)
|
||||||
|
progress.phase = 'service_detection'
|
||||||
|
progress.status = 'completed'
|
||||||
|
|
||||||
|
scan.completed_ips = len(results)
|
||||||
|
|
||||||
|
elif phase == 'http_analysis':
|
||||||
|
# Mark HTTP analysis as complete
|
||||||
|
scan.current_phase = 'completed'
|
||||||
|
scan.completed_ips = scan.total_ips
|
||||||
|
|
||||||
|
session.commit()
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Progress callback error for scan {scan_id}: {str(e)}")
|
||||||
|
# Don't re-raise - we don't want to break the scan
|
||||||
|
session.rollback()
|
||||||
|
|
||||||
|
return progress_callback
|
||||||
|
|
||||||
|
|
||||||
def execute_scan(scan_id: int, config_id: int, db_url: str = None):
|
def execute_scan(scan_id: int, config_id: int, db_url: str = None):
|
||||||
"""
|
"""
|
||||||
Execute a scan in the background.
|
Execute a scan in the background.
|
||||||
@@ -66,10 +186,13 @@ def execute_scan(scan_id: int, config_id: int, db_url: str = None):
|
|||||||
# Initialize scanner with database config
|
# Initialize scanner with database config
|
||||||
scanner = SneakyScanner(config_id=config_id)
|
scanner = SneakyScanner(config_id=config_id)
|
||||||
|
|
||||||
# Execute scan
|
# Create progress callback
|
||||||
|
progress_callback = create_progress_callback(scan_id, session)
|
||||||
|
|
||||||
|
# Execute scan with progress tracking
|
||||||
logger.info(f"Scan {scan_id}: Running scanner...")
|
logger.info(f"Scan {scan_id}: Running scanner...")
|
||||||
start_time = datetime.utcnow()
|
start_time = datetime.utcnow()
|
||||||
report, timestamp = scanner.scan()
|
report, timestamp = scanner.scan(progress_callback=progress_callback)
|
||||||
end_time = datetime.utcnow()
|
end_time = datetime.utcnow()
|
||||||
|
|
||||||
scan_duration = (end_time - start_time).total_seconds()
|
scan_duration = (end_time - start_time).total_seconds()
|
||||||
|
|||||||
@@ -59,6 +59,11 @@ class Scan(Base):
|
|||||||
completed_at = Column(DateTime, nullable=True, comment="Scan execution completion time")
|
completed_at = Column(DateTime, nullable=True, comment="Scan execution completion time")
|
||||||
error_message = Column(Text, nullable=True, comment="Error message if scan failed")
|
error_message = Column(Text, nullable=True, comment="Error message if scan failed")
|
||||||
|
|
||||||
|
# Progress tracking fields
|
||||||
|
current_phase = Column(String(50), nullable=True, comment="Current scan phase: ping, tcp_scan, udp_scan, service_detection, http_analysis")
|
||||||
|
total_ips = Column(Integer, nullable=True, comment="Total number of IPs to scan")
|
||||||
|
completed_ips = Column(Integer, nullable=True, default=0, comment="Number of IPs completed in current phase")
|
||||||
|
|
||||||
# Relationships
|
# Relationships
|
||||||
sites = relationship('ScanSite', back_populates='scan', cascade='all, delete-orphan')
|
sites = relationship('ScanSite', back_populates='scan', cascade='all, delete-orphan')
|
||||||
ips = relationship('ScanIP', back_populates='scan', cascade='all, delete-orphan')
|
ips = relationship('ScanIP', back_populates='scan', cascade='all, delete-orphan')
|
||||||
@@ -70,6 +75,7 @@ class Scan(Base):
|
|||||||
schedule = relationship('Schedule', back_populates='scans')
|
schedule = relationship('Schedule', back_populates='scans')
|
||||||
config = relationship('ScanConfig', back_populates='scans')
|
config = relationship('ScanConfig', back_populates='scans')
|
||||||
site_associations = relationship('ScanSiteAssociation', back_populates='scan', cascade='all, delete-orphan')
|
site_associations = relationship('ScanSiteAssociation', back_populates='scan', cascade='all, delete-orphan')
|
||||||
|
progress_entries = relationship('ScanProgress', back_populates='scan', cascade='all, delete-orphan')
|
||||||
|
|
||||||
def __repr__(self):
|
def __repr__(self):
|
||||||
return f"<Scan(id={self.id}, title='{self.title}', status='{self.status}')>"
|
return f"<Scan(id={self.id}, title='{self.title}', status='{self.status}')>"
|
||||||
@@ -244,6 +250,43 @@ class ScanTLSVersion(Base):
|
|||||||
return f"<ScanTLSVersion(id={self.id}, tls_version='{self.tls_version}', supported={self.supported})>"
|
return f"<ScanTLSVersion(id={self.id}, tls_version='{self.tls_version}', supported={self.supported})>"
|
||||||
|
|
||||||
|
|
||||||
|
class ScanProgress(Base):
|
||||||
|
"""
|
||||||
|
Real-time progress tracking for individual IPs during scan execution.
|
||||||
|
|
||||||
|
Stores intermediate results as they become available, allowing users to
|
||||||
|
see progress and results before the full scan completes.
|
||||||
|
"""
|
||||||
|
__tablename__ = 'scan_progress'
|
||||||
|
|
||||||
|
id = Column(Integer, primary_key=True, autoincrement=True)
|
||||||
|
scan_id = Column(Integer, ForeignKey('scans.id'), nullable=False, index=True)
|
||||||
|
ip_address = Column(String(45), nullable=False, comment="IP address being scanned")
|
||||||
|
site_name = Column(String(255), nullable=True, comment="Site name this IP belongs to")
|
||||||
|
phase = Column(String(50), nullable=False, comment="Phase: ping, tcp_scan, udp_scan, service_detection, http_analysis")
|
||||||
|
status = Column(String(20), nullable=False, default='pending', comment="pending, in_progress, completed, failed")
|
||||||
|
|
||||||
|
# Results data (stored as JSON)
|
||||||
|
ping_result = Column(Boolean, nullable=True, comment="Ping response result")
|
||||||
|
tcp_ports = Column(Text, nullable=True, comment="JSON array of discovered TCP ports")
|
||||||
|
udp_ports = Column(Text, nullable=True, comment="JSON array of discovered UDP ports")
|
||||||
|
services = Column(Text, nullable=True, comment="JSON array of detected services")
|
||||||
|
|
||||||
|
created_at = Column(DateTime, nullable=False, default=datetime.utcnow, comment="Entry creation time")
|
||||||
|
updated_at = Column(DateTime, nullable=False, default=datetime.utcnow, onupdate=datetime.utcnow, comment="Last update time")
|
||||||
|
|
||||||
|
# Relationships
|
||||||
|
scan = relationship('Scan', back_populates='progress_entries')
|
||||||
|
|
||||||
|
# Index for efficient lookups
|
||||||
|
__table_args__ = (
|
||||||
|
UniqueConstraint('scan_id', 'ip_address', name='uix_scan_progress_ip'),
|
||||||
|
)
|
||||||
|
|
||||||
|
def __repr__(self):
|
||||||
|
return f"<ScanProgress(id={self.id}, ip='{self.ip_address}', phase='{self.phase}', status='{self.status}')>"
|
||||||
|
|
||||||
|
|
||||||
# ============================================================================
|
# ============================================================================
|
||||||
# Reusable Site Definition Tables
|
# Reusable Site Definition Tables
|
||||||
# ============================================================================
|
# ============================================================================
|
||||||
|
|||||||
@@ -84,6 +84,50 @@
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<!-- Progress Section (shown when scan is running) -->
|
||||||
|
<div class="row mb-4" id="progress-section" style="display: none;">
|
||||||
|
<div class="col-12">
|
||||||
|
<div class="card">
|
||||||
|
<div class="card-header">
|
||||||
|
<h5 class="mb-0" style="color: #60a5fa;">
|
||||||
|
<i class="bi bi-hourglass-split"></i> Scan Progress
|
||||||
|
</h5>
|
||||||
|
</div>
|
||||||
|
<div class="card-body">
|
||||||
|
<!-- Phase and Progress Bar -->
|
||||||
|
<div class="mb-3">
|
||||||
|
<div class="d-flex justify-content-between align-items-center mb-2">
|
||||||
|
<span>Current Phase: <strong id="current-phase">Initializing...</strong></span>
|
||||||
|
<span id="progress-count">0 / 0 IPs</span>
|
||||||
|
</div>
|
||||||
|
<div class="progress" style="height: 20px; background-color: #334155;">
|
||||||
|
<div id="progress-bar" class="progress-bar bg-info" role="progressbar" style="width: 0%"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Per-IP Results Table -->
|
||||||
|
<div class="table-responsive" style="max-height: 400px; overflow-y: auto;">
|
||||||
|
<table class="table table-sm">
|
||||||
|
<thead style="position: sticky; top: 0; background-color: #1e293b;">
|
||||||
|
<tr>
|
||||||
|
<th>Site</th>
|
||||||
|
<th>IP Address</th>
|
||||||
|
<th>Ping</th>
|
||||||
|
<th>TCP Ports</th>
|
||||||
|
<th>UDP Ports</th>
|
||||||
|
<th>Services</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody id="progress-table-body">
|
||||||
|
<tr><td colspan="6" class="text-center text-muted">Waiting for results...</td></tr>
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
<!-- Stats Row -->
|
<!-- Stats Row -->
|
||||||
<div class="row mb-4">
|
<div class="row mb-4">
|
||||||
<div class="col-md-3">
|
<div class="col-md-3">
|
||||||
@@ -222,6 +266,7 @@
|
|||||||
const scanId = {{ scan_id }};
|
const scanId = {{ scan_id }};
|
||||||
let scanData = null;
|
let scanData = null;
|
||||||
let historyChart = null; // Store chart instance to prevent duplicates
|
let historyChart = null; // Store chart instance to prevent duplicates
|
||||||
|
let progressInterval = null; // Store progress polling interval
|
||||||
|
|
||||||
// Show alert notification
|
// Show alert notification
|
||||||
function showAlert(type, message) {
|
function showAlert(type, message) {
|
||||||
@@ -247,16 +292,136 @@
|
|||||||
loadScan().then(() => {
|
loadScan().then(() => {
|
||||||
findPreviousScan();
|
findPreviousScan();
|
||||||
loadHistoricalChart();
|
loadHistoricalChart();
|
||||||
|
|
||||||
|
// Start progress polling if scan is running
|
||||||
|
if (scanData && scanData.status === 'running') {
|
||||||
|
startProgressPolling();
|
||||||
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
// Auto-refresh every 10 seconds if scan is running
|
|
||||||
setInterval(function() {
|
|
||||||
if (scanData && scanData.status === 'running') {
|
|
||||||
loadScan();
|
|
||||||
}
|
|
||||||
}, 10000);
|
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// Start polling for progress updates
|
||||||
|
function startProgressPolling() {
|
||||||
|
// Show progress section
|
||||||
|
document.getElementById('progress-section').style.display = 'block';
|
||||||
|
|
||||||
|
// Initial load
|
||||||
|
loadProgress();
|
||||||
|
|
||||||
|
// Poll every 3 seconds
|
||||||
|
progressInterval = setInterval(loadProgress, 3000);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stop polling for progress updates
|
||||||
|
function stopProgressPolling() {
|
||||||
|
if (progressInterval) {
|
||||||
|
clearInterval(progressInterval);
|
||||||
|
progressInterval = null;
|
||||||
|
}
|
||||||
|
// Hide progress section when scan completes
|
||||||
|
document.getElementById('progress-section').style.display = 'none';
|
||||||
|
}
|
||||||
|
|
||||||
|
// Load progress data
|
||||||
|
async function loadProgress() {
|
||||||
|
try {
|
||||||
|
const response = await fetch(`/api/scans/${scanId}/progress`);
|
||||||
|
if (!response.ok) return;
|
||||||
|
|
||||||
|
const progress = await response.json();
|
||||||
|
|
||||||
|
// Check if scan is still running
|
||||||
|
if (progress.status !== 'running') {
|
||||||
|
stopProgressPolling();
|
||||||
|
loadScan(); // Refresh full scan data
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
renderProgress(progress);
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Error loading progress:', error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Render progress data
|
||||||
|
function renderProgress(progress) {
|
||||||
|
// Update phase display
|
||||||
|
const phaseNames = {
|
||||||
|
'pending': 'Initializing',
|
||||||
|
'ping': 'Ping Scan',
|
||||||
|
'tcp_scan': 'TCP Port Scan',
|
||||||
|
'udp_scan': 'UDP Port Scan',
|
||||||
|
'service_detection': 'Service Detection',
|
||||||
|
'http_analysis': 'HTTP/HTTPS Analysis',
|
||||||
|
'completed': 'Completing'
|
||||||
|
};
|
||||||
|
|
||||||
|
const phaseName = phaseNames[progress.current_phase] || progress.current_phase;
|
||||||
|
document.getElementById('current-phase').textContent = phaseName;
|
||||||
|
|
||||||
|
// Update progress count and bar
|
||||||
|
const total = progress.total_ips || 0;
|
||||||
|
const completed = progress.completed_ips || 0;
|
||||||
|
const percent = total > 0 ? Math.round((completed / total) * 100) : 0;
|
||||||
|
|
||||||
|
document.getElementById('progress-count').textContent = `${completed} / ${total} IPs`;
|
||||||
|
document.getElementById('progress-bar').style.width = `${percent}%`;
|
||||||
|
|
||||||
|
// Update progress table
|
||||||
|
const tbody = document.getElementById('progress-table-body');
|
||||||
|
const entries = progress.progress_entries || [];
|
||||||
|
|
||||||
|
if (entries.length === 0) {
|
||||||
|
tbody.innerHTML = '<tr><td colspan="6" class="text-center text-muted">Waiting for results...</td></tr>';
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
let html = '';
|
||||||
|
entries.forEach(entry => {
|
||||||
|
// Ping result
|
||||||
|
let pingDisplay = '-';
|
||||||
|
if (entry.ping_result !== null && entry.ping_result !== undefined) {
|
||||||
|
pingDisplay = entry.ping_result
|
||||||
|
? '<span class="badge badge-success">Yes</span>'
|
||||||
|
: '<span class="badge badge-danger">No</span>';
|
||||||
|
}
|
||||||
|
|
||||||
|
// TCP ports
|
||||||
|
const tcpPorts = entry.tcp_ports || [];
|
||||||
|
let tcpDisplay = tcpPorts.length > 0
|
||||||
|
? `<span class="badge bg-info">${tcpPorts.length}</span> <small class="text-muted">${tcpPorts.slice(0, 5).join(', ')}${tcpPorts.length > 5 ? '...' : ''}</small>`
|
||||||
|
: '-';
|
||||||
|
|
||||||
|
// UDP ports
|
||||||
|
const udpPorts = entry.udp_ports || [];
|
||||||
|
let udpDisplay = udpPorts.length > 0
|
||||||
|
? `<span class="badge bg-info">${udpPorts.length}</span>`
|
||||||
|
: '-';
|
||||||
|
|
||||||
|
// Services
|
||||||
|
const services = entry.services || [];
|
||||||
|
let svcDisplay = '-';
|
||||||
|
if (services.length > 0) {
|
||||||
|
const svcNames = services.map(s => s.service || 'unknown').slice(0, 3);
|
||||||
|
svcDisplay = `<span class="badge bg-info">${services.length}</span> <small class="text-muted">${svcNames.join(', ')}${services.length > 3 ? '...' : ''}</small>`;
|
||||||
|
}
|
||||||
|
|
||||||
|
html += `
|
||||||
|
<tr class="scan-row">
|
||||||
|
<td>${entry.site_name || '-'}</td>
|
||||||
|
<td class="mono">${entry.ip_address}</td>
|
||||||
|
<td>${pingDisplay}</td>
|
||||||
|
<td>${tcpDisplay}</td>
|
||||||
|
<td>${udpDisplay}</td>
|
||||||
|
<td>${svcDisplay}</td>
|
||||||
|
</tr>
|
||||||
|
`;
|
||||||
|
});
|
||||||
|
|
||||||
|
tbody.innerHTML = html;
|
||||||
|
}
|
||||||
|
|
||||||
// Load scan details
|
// Load scan details
|
||||||
async function loadScan() {
|
async function loadScan() {
|
||||||
const loadingEl = document.getElementById('scan-loading');
|
const loadingEl = document.getElementById('scan-loading');
|
||||||
|
|||||||
Reference in New Issue
Block a user