stage 1 of doing new cidrs/ site setup
This commit is contained in:
@@ -29,17 +29,52 @@ sys.stderr.reconfigure(line_buffering=True)
|
||||
|
||||
|
||||
class SneakyScanner:
|
||||
"""Wrapper for masscan to perform network scans based on YAML config"""
|
||||
"""Wrapper for masscan to perform network scans based on YAML config or database config"""
|
||||
|
||||
def __init__(self, config_path: str, output_dir: str = "/app/output"):
|
||||
self.config_path = Path(config_path)
|
||||
def __init__(self, config_path: str = None, config_id: int = None, config_dict: Dict = None, output_dir: str = "/app/output"):
|
||||
"""
|
||||
Initialize scanner with configuration.
|
||||
|
||||
Args:
|
||||
config_path: Path to YAML config file (legacy)
|
||||
config_id: Database config ID (preferred)
|
||||
config_dict: Config dictionary (for direct use)
|
||||
output_dir: Output directory for scan results
|
||||
|
||||
Note: Provide exactly one of config_path, config_id, or config_dict
|
||||
"""
|
||||
if sum([config_path is not None, config_id is not None, config_dict is not None]) != 1:
|
||||
raise ValueError("Must provide exactly one of: config_path, config_id, or config_dict")
|
||||
|
||||
self.config_path = Path(config_path) if config_path else None
|
||||
self.config_id = config_id
|
||||
self.output_dir = Path(output_dir)
|
||||
self.output_dir.mkdir(parents=True, exist_ok=True)
|
||||
self.config = self._load_config()
|
||||
|
||||
if config_dict:
|
||||
self.config = config_dict
|
||||
# Process sites: resolve references and expand CIDRs
|
||||
if 'sites' in self.config:
|
||||
self.config['sites'] = self._resolve_sites(self.config['sites'])
|
||||
else:
|
||||
self.config = self._load_config()
|
||||
|
||||
self.screenshot_capture = None
|
||||
|
||||
def _load_config(self) -> Dict[str, Any]:
|
||||
"""Load and validate YAML configuration"""
|
||||
"""
|
||||
Load and validate configuration from file or database.
|
||||
|
||||
Supports three formats:
|
||||
1. Legacy: Sites with explicit IP lists
|
||||
2. Site references: Sites referencing database-stored sites
|
||||
3. Inline CIDRs: Sites with CIDR ranges
|
||||
"""
|
||||
# Load from database if config_id provided
|
||||
if self.config_id:
|
||||
return self._load_config_from_database(self.config_id)
|
||||
|
||||
# Load from YAML file
|
||||
if not self.config_path.exists():
|
||||
raise FileNotFoundError(f"Config file not found: {self.config_path}")
|
||||
|
||||
@@ -51,8 +86,293 @@ class SneakyScanner:
|
||||
if not config.get('sites'):
|
||||
raise ValueError("Config must include 'sites' field")
|
||||
|
||||
# Process sites: resolve references and expand CIDRs
|
||||
config['sites'] = self._resolve_sites(config['sites'])
|
||||
|
||||
return config
|
||||
|
||||
def _load_config_from_database(self, config_id: int) -> Dict[str, Any]:
|
||||
"""
|
||||
Load configuration from database by ID.
|
||||
|
||||
Args:
|
||||
config_id: Database config ID
|
||||
|
||||
Returns:
|
||||
Config dictionary with expanded sites
|
||||
|
||||
Raises:
|
||||
ValueError: If config not found or invalid
|
||||
"""
|
||||
try:
|
||||
# Import here to avoid circular dependencies and allow scanner to work standalone
|
||||
import os
|
||||
import sys
|
||||
|
||||
# Add parent directory to path for imports
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
from web.models import ScanConfig
|
||||
|
||||
# Create database session
|
||||
db_url = os.environ.get('DATABASE_URL', 'sqlite:////app/data/sneakyscanner.db')
|
||||
engine = create_engine(db_url)
|
||||
Session = sessionmaker(bind=engine)
|
||||
session = Session()
|
||||
|
||||
try:
|
||||
# Load config from database
|
||||
db_config = session.query(ScanConfig).filter_by(id=config_id).first()
|
||||
|
||||
if not db_config:
|
||||
raise ValueError(f"Config with ID {config_id} not found in database")
|
||||
|
||||
# Build config dict with site references
|
||||
config = {
|
||||
'title': db_config.title,
|
||||
'sites': []
|
||||
}
|
||||
|
||||
# Add each site as a site_ref
|
||||
for assoc in db_config.site_associations:
|
||||
site = assoc.site
|
||||
config['sites'].append({
|
||||
'site_ref': site.name
|
||||
})
|
||||
|
||||
# Process sites: resolve references and expand CIDRs
|
||||
config['sites'] = self._resolve_sites(config['sites'])
|
||||
|
||||
return config
|
||||
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
except ImportError as e:
|
||||
raise ValueError(f"Failed to load config from database (import error): {str(e)}")
|
||||
except Exception as e:
|
||||
raise ValueError(f"Failed to load config from database: {str(e)}")
|
||||
|
||||
def _resolve_sites(self, sites: List[Dict]) -> List[Dict]:
|
||||
"""
|
||||
Resolve site references and expand CIDRs to IP lists.
|
||||
|
||||
Converts all site formats into the legacy format (with explicit IPs)
|
||||
for compatibility with the existing scan logic.
|
||||
|
||||
Args:
|
||||
sites: List of site definitions from config
|
||||
|
||||
Returns:
|
||||
List of sites with expanded IP lists
|
||||
"""
|
||||
import ipaddress
|
||||
|
||||
resolved_sites = []
|
||||
|
||||
for site_def in sites:
|
||||
# Handle site references
|
||||
if 'site_ref' in site_def:
|
||||
site_ref = site_def['site_ref']
|
||||
# Load site from database
|
||||
site_data = self._load_site_from_database(site_ref)
|
||||
if site_data:
|
||||
resolved_sites.append(site_data)
|
||||
else:
|
||||
print(f"WARNING: Site reference '{site_ref}' not found in database", file=sys.stderr)
|
||||
continue
|
||||
|
||||
# Handle inline CIDR definitions
|
||||
if 'cidrs' in site_def:
|
||||
site_name = site_def.get('name', 'Unknown Site')
|
||||
expanded_ips = []
|
||||
|
||||
for cidr_def in site_def['cidrs']:
|
||||
cidr = cidr_def['cidr']
|
||||
expected_ping = cidr_def.get('expected_ping', False)
|
||||
expected_tcp_ports = cidr_def.get('expected_tcp_ports', [])
|
||||
expected_udp_ports = cidr_def.get('expected_udp_ports', [])
|
||||
|
||||
# Check if there are IP-level overrides (from database sites)
|
||||
ip_overrides = cidr_def.get('ip_overrides', [])
|
||||
override_map = {
|
||||
override['ip_address']: override
|
||||
for override in ip_overrides
|
||||
}
|
||||
|
||||
# Expand CIDR to IP list
|
||||
try:
|
||||
network = ipaddress.ip_network(cidr, strict=False)
|
||||
ip_list = [str(ip) for ip in network.hosts()]
|
||||
|
||||
# If network has only 1 address (like /32), hosts() returns empty
|
||||
if not ip_list:
|
||||
ip_list = [str(network.network_address)]
|
||||
|
||||
# Create IP config for each IP in the CIDR
|
||||
for ip_address in ip_list:
|
||||
# Check if this IP has an override
|
||||
if ip_address in override_map:
|
||||
override = override_map[ip_address]
|
||||
ip_config = {
|
||||
'address': ip_address,
|
||||
'expected': {
|
||||
'ping': override.get('expected_ping', expected_ping),
|
||||
'tcp_ports': override.get('expected_tcp_ports', expected_tcp_ports),
|
||||
'udp_ports': override.get('expected_udp_ports', expected_udp_ports)
|
||||
}
|
||||
}
|
||||
else:
|
||||
# Use CIDR-level defaults
|
||||
ip_config = {
|
||||
'address': ip_address,
|
||||
'expected': {
|
||||
'ping': expected_ping,
|
||||
'tcp_ports': expected_tcp_ports,
|
||||
'udp_ports': expected_udp_ports
|
||||
}
|
||||
}
|
||||
|
||||
expanded_ips.append(ip_config)
|
||||
|
||||
except ValueError as e:
|
||||
print(f"WARNING: Invalid CIDR '{cidr}': {e}", file=sys.stderr)
|
||||
continue
|
||||
|
||||
# Add expanded site
|
||||
resolved_sites.append({
|
||||
'name': site_name,
|
||||
'ips': expanded_ips
|
||||
})
|
||||
continue
|
||||
|
||||
# Legacy format: already has 'ips' list
|
||||
if 'ips' in site_def:
|
||||
resolved_sites.append(site_def)
|
||||
continue
|
||||
|
||||
print(f"WARNING: Site definition missing required fields: {site_def}", file=sys.stderr)
|
||||
|
||||
return resolved_sites
|
||||
|
||||
def _load_site_from_database(self, site_name: str) -> Dict[str, Any]:
|
||||
"""
|
||||
Load a site definition from the database.
|
||||
|
||||
Args:
|
||||
site_name: Name of the site to load
|
||||
|
||||
Returns:
|
||||
Site definition dict with expanded IPs, or None if not found
|
||||
"""
|
||||
import ipaddress
|
||||
|
||||
try:
|
||||
# Import database modules
|
||||
import os
|
||||
import sys
|
||||
|
||||
# Add parent directory to path if needed
|
||||
parent_dir = str(Path(__file__).parent.parent)
|
||||
if parent_dir not in sys.path:
|
||||
sys.path.insert(0, parent_dir)
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import sessionmaker, joinedload
|
||||
from web.models import Site, SiteCIDR
|
||||
|
||||
# Get database URL from environment
|
||||
database_url = os.environ.get('DATABASE_URL', 'sqlite:///./sneakyscanner.db')
|
||||
|
||||
# Create engine and session
|
||||
engine = create_engine(database_url)
|
||||
Session = sessionmaker(bind=engine)
|
||||
session = Session()
|
||||
|
||||
# Query site with CIDRs and IP overrides
|
||||
site = (
|
||||
session.query(Site)
|
||||
.options(
|
||||
joinedload(Site.cidrs).joinedload(SiteCIDR.ips)
|
||||
)
|
||||
.filter(Site.name == site_name)
|
||||
.first()
|
||||
)
|
||||
|
||||
if not site:
|
||||
session.close()
|
||||
return None
|
||||
|
||||
# Expand CIDRs to IP list
|
||||
expanded_ips = []
|
||||
|
||||
for cidr_obj in site.cidrs:
|
||||
cidr = cidr_obj.cidr
|
||||
expected_ping = cidr_obj.expected_ping
|
||||
expected_tcp_ports = json.loads(cidr_obj.expected_tcp_ports) if cidr_obj.expected_tcp_ports else []
|
||||
expected_udp_ports = json.loads(cidr_obj.expected_udp_ports) if cidr_obj.expected_udp_ports else []
|
||||
|
||||
# Build IP override map
|
||||
override_map = {}
|
||||
for ip_override in cidr_obj.ips:
|
||||
override_map[ip_override.ip_address] = {
|
||||
'expected_ping': ip_override.expected_ping if ip_override.expected_ping is not None else expected_ping,
|
||||
'expected_tcp_ports': json.loads(ip_override.expected_tcp_ports) if ip_override.expected_tcp_ports else expected_tcp_ports,
|
||||
'expected_udp_ports': json.loads(ip_override.expected_udp_ports) if ip_override.expected_udp_ports else expected_udp_ports
|
||||
}
|
||||
|
||||
# Expand CIDR to IP list
|
||||
try:
|
||||
network = ipaddress.ip_network(cidr, strict=False)
|
||||
ip_list = [str(ip) for ip in network.hosts()]
|
||||
|
||||
if not ip_list:
|
||||
ip_list = [str(network.network_address)]
|
||||
|
||||
for ip_address in ip_list:
|
||||
# Check if this IP has an override
|
||||
if ip_address in override_map:
|
||||
override = override_map[ip_address]
|
||||
ip_config = {
|
||||
'address': ip_address,
|
||||
'expected': {
|
||||
'ping': override['expected_ping'],
|
||||
'tcp_ports': override['expected_tcp_ports'],
|
||||
'udp_ports': override['expected_udp_ports']
|
||||
}
|
||||
}
|
||||
else:
|
||||
# Use CIDR-level defaults
|
||||
ip_config = {
|
||||
'address': ip_address,
|
||||
'expected': {
|
||||
'ping': expected_ping if expected_ping is not None else False,
|
||||
'tcp_ports': expected_tcp_ports,
|
||||
'udp_ports': expected_udp_ports
|
||||
}
|
||||
}
|
||||
|
||||
expanded_ips.append(ip_config)
|
||||
|
||||
except ValueError as e:
|
||||
print(f"WARNING: Invalid CIDR '{cidr}' in site '{site_name}': {e}", file=sys.stderr)
|
||||
continue
|
||||
|
||||
session.close()
|
||||
|
||||
return {
|
||||
'name': site.name,
|
||||
'ips': expanded_ips
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
print(f"ERROR: Failed to load site '{site_name}' from database: {e}", file=sys.stderr)
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return None
|
||||
|
||||
def _run_masscan(self, targets: List[str], ports: str, protocol: str) -> List[Dict]:
|
||||
"""
|
||||
Run masscan and return parsed results
|
||||
@@ -557,7 +877,10 @@ class SneakyScanner:
|
||||
Dictionary containing scan results
|
||||
"""
|
||||
print(f"Starting scan: {self.config['title']}", flush=True)
|
||||
print(f"Config: {self.config_path}", flush=True)
|
||||
if self.config_id:
|
||||
print(f"Config ID: {self.config_id}", flush=True)
|
||||
elif self.config_path:
|
||||
print(f"Config: {self.config_path}", flush=True)
|
||||
|
||||
# Record start time
|
||||
start_time = time.time()
|
||||
@@ -662,7 +985,8 @@ class SneakyScanner:
|
||||
'title': self.config['title'],
|
||||
'scan_time': datetime.utcnow().isoformat() + 'Z',
|
||||
'scan_duration': scan_duration,
|
||||
'config_file': str(self.config_path),
|
||||
'config_file': str(self.config_path) if self.config_path else None,
|
||||
'config_id': self.config_id,
|
||||
'sites': []
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user