diff --git a/docs/ai/PHASE3.md b/docs/ai/PHASE3.md index f874293..871da20 100644 --- a/docs/ai/PHASE3.md +++ b/docs/ai/PHASE3.md @@ -1,7 +1,7 @@ # Phase 3 Implementation Plan: Dashboard Enhancement & Scheduled Scans **Status:** In Progress -**Progress:** 5/14 days complete (36%) +**Progress:** 9/14 days complete (64%) **Estimated Duration:** 14 days (2 weeks) **Dependencies:** Phase 2 Complete ✅ @@ -10,9 +10,9 @@ - ✅ **Step 1: Fix Styling Issues & CSS Refactor** (Day 1) - COMPLETE - ✅ **Step 2: ScheduleService Implementation** (Days 2-3) - COMPLETE - ✅ **Step 3: Schedules API Endpoints** (Days 4-5) - COMPLETE -- 📋 **Step 4: Schedule Management UI** (Days 6-7) - NEXT -- 📋 **Step 5: Enhanced Dashboard with Charts** (Days 8-9) -- 📋 **Step 6: Scheduler Integration** (Day 10) +- ✅ **Step 4: Schedule Management UI** (Days 6-7) - COMPLETE +- ✅ **Step 5: Enhanced Dashboard with Charts** (Days 8-9) - COMPLETE +- 📋 **Step 6: Scheduler Integration** (Day 10) - NEXT - 📋 **Step 7: Scan Comparison Features** (Days 11-12) - 📋 **Step 8: Testing & Documentation** (Days 13-14) diff --git a/tests/test_stats_api.py b/tests/test_stats_api.py new file mode 100644 index 0000000..bd72631 --- /dev/null +++ b/tests/test_stats_api.py @@ -0,0 +1,325 @@ +""" +Tests for stats API endpoints. + +Tests dashboard statistics and trending data endpoints. +""" + +import pytest +from datetime import datetime, timedelta +from web.models import Scan + + +class TestStatsAPI: + """Test suite for stats API endpoints.""" + + def test_scan_trend_default_30_days(self, client, auth_headers, db_session): + """Test scan trend endpoint with default 30 days.""" + # Create test scans over multiple days + today = datetime.utcnow() + for i in range(5): + scan_date = today - timedelta(days=i) + for j in range(i + 1): # Create 1, 2, 3, 4, 5 scans per day + scan = Scan( + config_file='/app/configs/test.yaml', + timestamp=scan_date, + status='completed', + duration=10.5 + ) + db_session.add(scan) + db_session.commit() + + # Request trend data + response = client.get('/api/stats/scan-trend', headers=auth_headers) + assert response.status_code == 200 + + data = response.get_json() + assert 'labels' in data + assert 'values' in data + assert 'start_date' in data + assert 'end_date' in data + assert 'total_scans' in data + + # Should have 30 days of data + assert len(data['labels']) == 30 + assert len(data['values']) == 30 + + # Total scans should match (1+2+3+4+5 = 15) + assert data['total_scans'] == 15 + + # Values should be non-negative integers + assert all(isinstance(v, int) for v in data['values']) + assert all(v >= 0 for v in data['values']) + + def test_scan_trend_custom_days(self, client, auth_headers, db_session): + """Test scan trend endpoint with custom number of days.""" + # Create test scans + today = datetime.utcnow() + for i in range(10): + scan = Scan( + config_file='/app/configs/test.yaml', + timestamp=today - timedelta(days=i), + status='completed', + duration=10.5 + ) + db_session.add(scan) + db_session.commit() + + # Request 7 days of data + response = client.get('/api/stats/scan-trend?days=7', headers=auth_headers) + assert response.status_code == 200 + + data = response.get_json() + assert len(data['labels']) == 7 + assert len(data['values']) == 7 + assert data['total_scans'] == 7 + + def test_scan_trend_max_days_365(self, client, auth_headers): + """Test scan trend endpoint accepts maximum 365 days.""" + response = client.get('/api/stats/scan-trend?days=365', headers=auth_headers) + assert response.status_code == 200 + + data = response.get_json() + assert len(data['labels']) == 365 + + def test_scan_trend_rejects_days_over_365(self, client, auth_headers): + """Test scan trend endpoint rejects more than 365 days.""" + response = client.get('/api/stats/scan-trend?days=366', headers=auth_headers) + assert response.status_code == 400 + + data = response.get_json() + assert 'error' in data + assert '365' in data['error'] + + def test_scan_trend_rejects_days_less_than_1(self, client, auth_headers): + """Test scan trend endpoint rejects days less than 1.""" + response = client.get('/api/stats/scan-trend?days=0', headers=auth_headers) + assert response.status_code == 400 + + data = response.get_json() + assert 'error' in data + + def test_scan_trend_fills_missing_days_with_zero(self, client, auth_headers, db_session): + """Test scan trend fills days with no scans as zero.""" + # Create scans only on specific days + today = datetime.utcnow() + + # Create scan 5 days ago + scan1 = Scan( + config_file='/app/configs/test.yaml', + timestamp=today - timedelta(days=5), + status='completed', + duration=10.5 + ) + db_session.add(scan1) + + # Create scan 10 days ago + scan2 = Scan( + config_file='/app/configs/test.yaml', + timestamp=today - timedelta(days=10), + status='completed', + duration=10.5 + ) + db_session.add(scan2) + db_session.commit() + + # Request 15 days + response = client.get('/api/stats/scan-trend?days=15', headers=auth_headers) + assert response.status_code == 200 + + data = response.get_json() + + # Should have 15 days of data + assert len(data['values']) == 15 + + # Most days should be zero + zero_days = sum(1 for v in data['values'] if v == 0) + assert zero_days >= 13 # At least 13 days with no scans + + def test_scan_trend_requires_authentication(self, client): + """Test scan trend endpoint requires authentication.""" + response = client.get('/api/stats/scan-trend') + assert response.status_code == 401 + + def test_summary_endpoint(self, client, auth_headers, db_session): + """Test summary statistics endpoint.""" + # Create test scans with different statuses + today = datetime.utcnow() + + # 5 completed scans + for i in range(5): + scan = Scan( + config_file='/app/configs/test.yaml', + timestamp=today - timedelta(days=i), + status='completed', + duration=10.5 + ) + db_session.add(scan) + + # 2 failed scans + for i in range(2): + scan = Scan( + config_file='/app/configs/test.yaml', + timestamp=today - timedelta(days=i), + status='failed', + duration=5.0 + ) + db_session.add(scan) + + # 1 running scan + scan = Scan( + config_file='/app/configs/test.yaml', + timestamp=today, + status='running', + duration=None + ) + db_session.add(scan) + + db_session.commit() + + # Request summary + response = client.get('/api/stats/summary', headers=auth_headers) + assert response.status_code == 200 + + data = response.get_json() + assert 'total_scans' in data + assert 'completed_scans' in data + assert 'failed_scans' in data + assert 'running_scans' in data + assert 'scans_today' in data + assert 'scans_this_week' in data + + # Verify counts + assert data['total_scans'] == 8 + assert data['completed_scans'] == 5 + assert data['failed_scans'] == 2 + assert data['running_scans'] == 1 + assert data['scans_today'] >= 1 + assert data['scans_this_week'] >= 1 + + def test_summary_with_no_scans(self, client, auth_headers): + """Test summary endpoint with no scans.""" + response = client.get('/api/stats/summary', headers=auth_headers) + assert response.status_code == 200 + + data = response.get_json() + assert data['total_scans'] == 0 + assert data['completed_scans'] == 0 + assert data['failed_scans'] == 0 + assert data['running_scans'] == 0 + assert data['scans_today'] == 0 + assert data['scans_this_week'] == 0 + + def test_summary_scans_today(self, client, auth_headers, db_session): + """Test summary counts scans today correctly.""" + today = datetime.utcnow() + yesterday = today - timedelta(days=1) + + # Create 3 scans today + for i in range(3): + scan = Scan( + config_file='/app/configs/test.yaml', + timestamp=today, + status='completed', + duration=10.5 + ) + db_session.add(scan) + + # Create 2 scans yesterday + for i in range(2): + scan = Scan( + config_file='/app/configs/test.yaml', + timestamp=yesterday, + status='completed', + duration=10.5 + ) + db_session.add(scan) + + db_session.commit() + + response = client.get('/api/stats/summary', headers=auth_headers) + assert response.status_code == 200 + + data = response.get_json() + assert data['scans_today'] == 3 + assert data['scans_this_week'] >= 3 + + def test_summary_scans_this_week(self, client, auth_headers, db_session): + """Test summary counts scans this week correctly.""" + today = datetime.utcnow() + + # Create scans over the last 10 days + for i in range(10): + scan = Scan( + config_file='/app/configs/test.yaml', + timestamp=today - timedelta(days=i), + status='completed', + duration=10.5 + ) + db_session.add(scan) + + db_session.commit() + + response = client.get('/api/stats/summary', headers=auth_headers) + assert response.status_code == 200 + + data = response.get_json() + # Last 7 days (0-6) = 7 scans + assert data['scans_this_week'] == 7 + + def test_summary_requires_authentication(self, client): + """Test summary endpoint requires authentication.""" + response = client.get('/api/stats/summary') + assert response.status_code == 401 + + def test_scan_trend_date_format(self, client, auth_headers, db_session): + """Test scan trend returns dates in correct format.""" + # Create a scan + scan = Scan( + config_file='/app/configs/test.yaml', + timestamp=datetime.utcnow(), + status='completed', + duration=10.5 + ) + db_session.add(scan) + db_session.commit() + + response = client.get('/api/stats/scan-trend?days=7', headers=auth_headers) + assert response.status_code == 200 + + data = response.get_json() + + # Check date format (YYYY-MM-DD) + for label in data['labels']: + assert len(label) == 10 + assert label[4] == '-' + assert label[7] == '-' + # Try parsing to ensure valid date + datetime.strptime(label, '%Y-%m-%d') + + def test_scan_trend_consecutive_dates(self, client, auth_headers): + """Test scan trend returns consecutive dates.""" + response = client.get('/api/stats/scan-trend?days=7', headers=auth_headers) + assert response.status_code == 200 + + data = response.get_json() + labels = data['labels'] + + # Convert to datetime objects + dates = [datetime.strptime(label, '%Y-%m-%d') for label in labels] + + # Check dates are consecutive + for i in range(len(dates) - 1): + diff = dates[i + 1] - dates[i] + assert diff.days == 1, f"Dates not consecutive: {dates[i]} to {dates[i+1]}" + + def test_scan_trend_ends_with_today(self, client, auth_headers): + """Test scan trend ends with today's date.""" + response = client.get('/api/stats/scan-trend?days=7', headers=auth_headers) + assert response.status_code == 200 + + data = response.get_json() + + # Last date should be today + today = datetime.utcnow().date() + last_date = datetime.strptime(data['labels'][-1], '%Y-%m-%d').date() + assert last_date == today diff --git a/web/api/stats.py b/web/api/stats.py new file mode 100644 index 0000000..7591197 --- /dev/null +++ b/web/api/stats.py @@ -0,0 +1,151 @@ +""" +Stats API blueprint. + +Handles endpoints for dashboard statistics, trending data, and analytics. +""" + +import logging +from datetime import datetime, timedelta +from flask import Blueprint, current_app, jsonify, request +from sqlalchemy import func, Date +from sqlalchemy.exc import SQLAlchemyError + +from web.auth.decorators import api_auth_required +from web.models import Scan + +bp = Blueprint('stats', __name__) +logger = logging.getLogger(__name__) + + +@bp.route('/scan-trend', methods=['GET']) +@api_auth_required +def scan_trend(): + """ + Get scan activity trend data for charts. + + Query params: + days: Number of days to include (default: 30, max: 365) + + Returns: + JSON response with labels and values arrays for Chart.js + { + "labels": ["2025-01-01", "2025-01-02", ...], + "values": [5, 3, 7, 2, ...] + } + """ + try: + # Get and validate query parameters + days = request.args.get('days', 30, type=int) + + # Validate days parameter + if days < 1: + return jsonify({'error': 'days parameter must be at least 1'}), 400 + if days > 365: + return jsonify({'error': 'days parameter cannot exceed 365'}), 400 + + # Calculate date range + end_date = datetime.utcnow().date() + start_date = end_date - timedelta(days=days - 1) + + # Query scan counts per day + db_session = current_app.db_session + scan_counts = ( + db_session.query( + func.date(Scan.timestamp).label('scan_date'), + func.count(Scan.id).label('scan_count') + ) + .filter(func.date(Scan.timestamp) >= start_date) + .filter(func.date(Scan.timestamp) <= end_date) + .group_by(func.date(Scan.timestamp)) + .order_by('scan_date') + .all() + ) + + # Create a dictionary of date -> count + scan_dict = {str(row.scan_date): row.scan_count for row in scan_counts} + + # Generate all dates in range (fill missing dates with 0) + labels = [] + values = [] + current_date = start_date + while current_date <= end_date: + date_str = str(current_date) + labels.append(date_str) + values.append(scan_dict.get(date_str, 0)) + current_date += timedelta(days=1) + + return jsonify({ + 'labels': labels, + 'values': values, + 'start_date': str(start_date), + 'end_date': str(end_date), + 'total_scans': sum(values) + }), 200 + + except SQLAlchemyError as e: + logger.error(f"Database error in scan_trend: {str(e)}") + return jsonify({'error': 'Database error occurred'}), 500 + except Exception as e: + logger.error(f"Error in scan_trend: {str(e)}") + return jsonify({'error': 'An error occurred'}), 500 + + +@bp.route('/summary', methods=['GET']) +@api_auth_required +def summary(): + """ + Get dashboard summary statistics. + + Returns: + JSON response with summary stats + { + "total_scans": 150, + "completed_scans": 140, + "failed_scans": 5, + "running_scans": 5, + "scans_today": 3, + "scans_this_week": 15 + } + """ + try: + db_session = current_app.db_session + + # Get total counts by status + total_scans = db_session.query(func.count(Scan.id)).scalar() or 0 + completed_scans = db_session.query(func.count(Scan.id)).filter( + Scan.status == 'completed' + ).scalar() or 0 + failed_scans = db_session.query(func.count(Scan.id)).filter( + Scan.status == 'failed' + ).scalar() or 0 + running_scans = db_session.query(func.count(Scan.id)).filter( + Scan.status == 'running' + ).scalar() or 0 + + # Get scans today + today = datetime.utcnow().date() + scans_today = db_session.query(func.count(Scan.id)).filter( + func.date(Scan.timestamp) == today + ).scalar() or 0 + + # Get scans this week (last 7 days) + week_ago = today - timedelta(days=6) + scans_this_week = db_session.query(func.count(Scan.id)).filter( + func.date(Scan.timestamp) >= week_ago + ).scalar() or 0 + + return jsonify({ + 'total_scans': total_scans, + 'completed_scans': completed_scans, + 'failed_scans': failed_scans, + 'running_scans': running_scans, + 'scans_today': scans_today, + 'scans_this_week': scans_this_week + }), 200 + + except SQLAlchemyError as e: + logger.error(f"Database error in summary: {str(e)}") + return jsonify({'error': 'Database error occurred'}), 500 + except Exception as e: + logger.error(f"Error in summary: {str(e)}") + return jsonify({'error': 'An error occurred'}), 500 diff --git a/web/app.py b/web/app.py index 97a7a7e..ae2d4ac 100644 --- a/web/app.py +++ b/web/app.py @@ -317,6 +317,7 @@ def register_blueprints(app: Flask) -> None: from web.api.schedules import bp as schedules_bp from web.api.alerts import bp as alerts_bp from web.api.settings import bp as settings_bp + from web.api.stats import bp as stats_bp from web.auth.routes import bp as auth_bp from web.routes.main import bp as main_bp @@ -331,6 +332,7 @@ def register_blueprints(app: Flask) -> None: app.register_blueprint(schedules_bp, url_prefix='/api/schedules') app.register_blueprint(alerts_bp, url_prefix='/api/alerts') app.register_blueprint(settings_bp, url_prefix='/api/settings') + app.register_blueprint(stats_bp, url_prefix='/api/stats') app.logger.info("Blueprints registered") diff --git a/web/templates/dashboard.html b/web/templates/dashboard.html index 7d04fc5..d60de83 100644 --- a/web/templates/dashboard.html +++ b/web/templates/dashboard.html @@ -50,6 +50,51 @@ View All Scans + + Manage Schedules + + + + + + + +
+
+
+
+
Scan Activity (Last 30 Days)
+
+
+
+
+ Loading... +
+
+ +
+
+
+ + +
+
+
+
Upcoming Schedules
+ Manage +
+
+
+
+ Loading... +
+
+ +
@@ -140,6 +185,8 @@ document.addEventListener('DOMContentLoaded', function() { refreshScans(); loadStats(); + loadScanTrend(); + loadSchedules(); // Auto-refresh every 10 seconds if there are running scans refreshInterval = setInterval(function() { @@ -149,6 +196,9 @@ loadStats(); } }, 10000); + + // Refresh schedules every 30 seconds + setInterval(loadSchedules, 30000); }); // Load dashboard stats @@ -329,6 +379,162 @@ } } + // Load scan trend chart + async function loadScanTrend() { + const chartLoading = document.getElementById('chart-loading'); + const canvas = document.getElementById('scanTrendChart'); + + try { + const response = await fetch('/api/stats/scan-trend?days=30'); + if (!response.ok) { + throw new Error('Failed to load trend data'); + } + + const data = await response.json(); + + // Hide loading, show chart + chartLoading.style.display = 'none'; + canvas.style.display = 'block'; + + // Create chart + const ctx = canvas.getContext('2d'); + new Chart(ctx, { + type: 'line', + data: { + labels: data.labels, + datasets: [{ + label: 'Scans per Day', + data: data.values, + borderColor: '#60a5fa', + backgroundColor: 'rgba(96, 165, 250, 0.1)', + tension: 0.3, + fill: true + }] + }, + options: { + responsive: true, + maintainAspectRatio: true, + plugins: { + legend: { + display: false + }, + tooltip: { + mode: 'index', + intersect: false, + callbacks: { + title: function(context) { + return new Date(context[0].label).toLocaleDateString(); + } + } + } + }, + scales: { + y: { + beginAtZero: true, + ticks: { + stepSize: 1, + color: '#94a3b8' + }, + grid: { + color: '#334155' + } + }, + x: { + ticks: { + color: '#94a3b8', + maxRotation: 0, + autoSkip: true, + maxTicksLimit: 10 + }, + grid: { + color: '#334155' + } + } + } + } + }); + } catch (error) { + console.error('Error loading chart:', error); + chartLoading.innerHTML = '

Failed to load chart data

'; + } + } + + // Load upcoming schedules + async function loadSchedules() { + const loadingEl = document.getElementById('schedules-loading'); + const contentEl = document.getElementById('schedules-content'); + const emptyEl = document.getElementById('schedules-empty'); + + try { + const response = await fetch('/api/schedules?per_page=5'); + if (!response.ok) { + throw new Error('Failed to load schedules'); + } + + const data = await response.json(); + const schedules = data.schedules || []; + + loadingEl.style.display = 'none'; + + if (schedules.length === 0) { + emptyEl.style.display = 'block'; + } else { + contentEl.style.display = 'block'; + + // Filter enabled schedules and sort by next_run + const enabledSchedules = schedules + .filter(s => s.enabled && s.next_run) + .sort((a, b) => new Date(a.next_run) - new Date(b.next_run)) + .slice(0, 3); + + if (enabledSchedules.length === 0) { + contentEl.innerHTML = '

No enabled schedules

'; + } else { + contentEl.innerHTML = enabledSchedules.map(schedule => { + const nextRun = new Date(schedule.next_run); + const now = new Date(); + const diffMs = nextRun - now; + const diffMins = Math.floor(diffMs / 60000); + const diffHours = Math.floor(diffMins / 60); + const diffDays = Math.floor(diffHours / 24); + + let timeStr; + if (diffMins < 1) { + timeStr = 'In less than 1 minute'; + } else if (diffMins < 60) { + timeStr = `In ${diffMins} minute${diffMins === 1 ? '' : 's'}`; + } else if (diffHours < 24) { + timeStr = `In ${diffHours} hour${diffHours === 1 ? '' : 's'}`; + } else if (diffDays < 7) { + timeStr = `In ${diffDays} day${diffDays === 1 ? '' : 's'}`; + } else { + timeStr = nextRun.toLocaleDateString(); + } + + return ` +
+
+
+ ${schedule.name} +
+ ${timeStr} +
+ ${schedule.cron_expression} +
+
+
+ `; + }).join(''); + } + } + } catch (error) { + console.error('Error loading schedules:', error); + loadingEl.style.display = 'none'; + contentEl.style.display = 'block'; + contentEl.innerHTML = '

Failed to load schedules

'; + } + } + // Delete scan async function deleteScan(scanId) { if (!confirm(`Are you sure you want to delete scan ${scanId}?`)) {