phase3 #2

Merged
ptarrant merged 8 commits from phase3 into master 2025-11-17 18:06:57 +00:00
5 changed files with 688 additions and 4 deletions
Showing only changes of commit effce42f21 - Show all commits

View File

@@ -1,7 +1,7 @@
# Phase 3 Implementation Plan: Dashboard Enhancement & Scheduled Scans
**Status:** In Progress
**Progress:** 5/14 days complete (36%)
**Progress:** 9/14 days complete (64%)
**Estimated Duration:** 14 days (2 weeks)
**Dependencies:** Phase 2 Complete ✅
@@ -10,9 +10,9 @@
-**Step 1: Fix Styling Issues & CSS Refactor** (Day 1) - COMPLETE
-**Step 2: ScheduleService Implementation** (Days 2-3) - COMPLETE
-**Step 3: Schedules API Endpoints** (Days 4-5) - COMPLETE
- 📋 **Step 4: Schedule Management UI** (Days 6-7) - NEXT
- 📋 **Step 5: Enhanced Dashboard with Charts** (Days 8-9)
- 📋 **Step 6: Scheduler Integration** (Day 10)
- **Step 4: Schedule Management UI** (Days 6-7) - COMPLETE
- **Step 5: Enhanced Dashboard with Charts** (Days 8-9) - COMPLETE
- 📋 **Step 6: Scheduler Integration** (Day 10) - NEXT
- 📋 **Step 7: Scan Comparison Features** (Days 11-12)
- 📋 **Step 8: Testing & Documentation** (Days 13-14)

325
tests/test_stats_api.py Normal file
View File

@@ -0,0 +1,325 @@
"""
Tests for stats API endpoints.
Tests dashboard statistics and trending data endpoints.
"""
import pytest
from datetime import datetime, timedelta
from web.models import Scan
class TestStatsAPI:
"""Test suite for stats API endpoints."""
def test_scan_trend_default_30_days(self, client, auth_headers, db_session):
"""Test scan trend endpoint with default 30 days."""
# Create test scans over multiple days
today = datetime.utcnow()
for i in range(5):
scan_date = today - timedelta(days=i)
for j in range(i + 1): # Create 1, 2, 3, 4, 5 scans per day
scan = Scan(
config_file='/app/configs/test.yaml',
timestamp=scan_date,
status='completed',
duration=10.5
)
db_session.add(scan)
db_session.commit()
# Request trend data
response = client.get('/api/stats/scan-trend', headers=auth_headers)
assert response.status_code == 200
data = response.get_json()
assert 'labels' in data
assert 'values' in data
assert 'start_date' in data
assert 'end_date' in data
assert 'total_scans' in data
# Should have 30 days of data
assert len(data['labels']) == 30
assert len(data['values']) == 30
# Total scans should match (1+2+3+4+5 = 15)
assert data['total_scans'] == 15
# Values should be non-negative integers
assert all(isinstance(v, int) for v in data['values'])
assert all(v >= 0 for v in data['values'])
def test_scan_trend_custom_days(self, client, auth_headers, db_session):
"""Test scan trend endpoint with custom number of days."""
# Create test scans
today = datetime.utcnow()
for i in range(10):
scan = Scan(
config_file='/app/configs/test.yaml',
timestamp=today - timedelta(days=i),
status='completed',
duration=10.5
)
db_session.add(scan)
db_session.commit()
# Request 7 days of data
response = client.get('/api/stats/scan-trend?days=7', headers=auth_headers)
assert response.status_code == 200
data = response.get_json()
assert len(data['labels']) == 7
assert len(data['values']) == 7
assert data['total_scans'] == 7
def test_scan_trend_max_days_365(self, client, auth_headers):
"""Test scan trend endpoint accepts maximum 365 days."""
response = client.get('/api/stats/scan-trend?days=365', headers=auth_headers)
assert response.status_code == 200
data = response.get_json()
assert len(data['labels']) == 365
def test_scan_trend_rejects_days_over_365(self, client, auth_headers):
"""Test scan trend endpoint rejects more than 365 days."""
response = client.get('/api/stats/scan-trend?days=366', headers=auth_headers)
assert response.status_code == 400
data = response.get_json()
assert 'error' in data
assert '365' in data['error']
def test_scan_trend_rejects_days_less_than_1(self, client, auth_headers):
"""Test scan trend endpoint rejects days less than 1."""
response = client.get('/api/stats/scan-trend?days=0', headers=auth_headers)
assert response.status_code == 400
data = response.get_json()
assert 'error' in data
def test_scan_trend_fills_missing_days_with_zero(self, client, auth_headers, db_session):
"""Test scan trend fills days with no scans as zero."""
# Create scans only on specific days
today = datetime.utcnow()
# Create scan 5 days ago
scan1 = Scan(
config_file='/app/configs/test.yaml',
timestamp=today - timedelta(days=5),
status='completed',
duration=10.5
)
db_session.add(scan1)
# Create scan 10 days ago
scan2 = Scan(
config_file='/app/configs/test.yaml',
timestamp=today - timedelta(days=10),
status='completed',
duration=10.5
)
db_session.add(scan2)
db_session.commit()
# Request 15 days
response = client.get('/api/stats/scan-trend?days=15', headers=auth_headers)
assert response.status_code == 200
data = response.get_json()
# Should have 15 days of data
assert len(data['values']) == 15
# Most days should be zero
zero_days = sum(1 for v in data['values'] if v == 0)
assert zero_days >= 13 # At least 13 days with no scans
def test_scan_trend_requires_authentication(self, client):
"""Test scan trend endpoint requires authentication."""
response = client.get('/api/stats/scan-trend')
assert response.status_code == 401
def test_summary_endpoint(self, client, auth_headers, db_session):
"""Test summary statistics endpoint."""
# Create test scans with different statuses
today = datetime.utcnow()
# 5 completed scans
for i in range(5):
scan = Scan(
config_file='/app/configs/test.yaml',
timestamp=today - timedelta(days=i),
status='completed',
duration=10.5
)
db_session.add(scan)
# 2 failed scans
for i in range(2):
scan = Scan(
config_file='/app/configs/test.yaml',
timestamp=today - timedelta(days=i),
status='failed',
duration=5.0
)
db_session.add(scan)
# 1 running scan
scan = Scan(
config_file='/app/configs/test.yaml',
timestamp=today,
status='running',
duration=None
)
db_session.add(scan)
db_session.commit()
# Request summary
response = client.get('/api/stats/summary', headers=auth_headers)
assert response.status_code == 200
data = response.get_json()
assert 'total_scans' in data
assert 'completed_scans' in data
assert 'failed_scans' in data
assert 'running_scans' in data
assert 'scans_today' in data
assert 'scans_this_week' in data
# Verify counts
assert data['total_scans'] == 8
assert data['completed_scans'] == 5
assert data['failed_scans'] == 2
assert data['running_scans'] == 1
assert data['scans_today'] >= 1
assert data['scans_this_week'] >= 1
def test_summary_with_no_scans(self, client, auth_headers):
"""Test summary endpoint with no scans."""
response = client.get('/api/stats/summary', headers=auth_headers)
assert response.status_code == 200
data = response.get_json()
assert data['total_scans'] == 0
assert data['completed_scans'] == 0
assert data['failed_scans'] == 0
assert data['running_scans'] == 0
assert data['scans_today'] == 0
assert data['scans_this_week'] == 0
def test_summary_scans_today(self, client, auth_headers, db_session):
"""Test summary counts scans today correctly."""
today = datetime.utcnow()
yesterday = today - timedelta(days=1)
# Create 3 scans today
for i in range(3):
scan = Scan(
config_file='/app/configs/test.yaml',
timestamp=today,
status='completed',
duration=10.5
)
db_session.add(scan)
# Create 2 scans yesterday
for i in range(2):
scan = Scan(
config_file='/app/configs/test.yaml',
timestamp=yesterday,
status='completed',
duration=10.5
)
db_session.add(scan)
db_session.commit()
response = client.get('/api/stats/summary', headers=auth_headers)
assert response.status_code == 200
data = response.get_json()
assert data['scans_today'] == 3
assert data['scans_this_week'] >= 3
def test_summary_scans_this_week(self, client, auth_headers, db_session):
"""Test summary counts scans this week correctly."""
today = datetime.utcnow()
# Create scans over the last 10 days
for i in range(10):
scan = Scan(
config_file='/app/configs/test.yaml',
timestamp=today - timedelta(days=i),
status='completed',
duration=10.5
)
db_session.add(scan)
db_session.commit()
response = client.get('/api/stats/summary', headers=auth_headers)
assert response.status_code == 200
data = response.get_json()
# Last 7 days (0-6) = 7 scans
assert data['scans_this_week'] == 7
def test_summary_requires_authentication(self, client):
"""Test summary endpoint requires authentication."""
response = client.get('/api/stats/summary')
assert response.status_code == 401
def test_scan_trend_date_format(self, client, auth_headers, db_session):
"""Test scan trend returns dates in correct format."""
# Create a scan
scan = Scan(
config_file='/app/configs/test.yaml',
timestamp=datetime.utcnow(),
status='completed',
duration=10.5
)
db_session.add(scan)
db_session.commit()
response = client.get('/api/stats/scan-trend?days=7', headers=auth_headers)
assert response.status_code == 200
data = response.get_json()
# Check date format (YYYY-MM-DD)
for label in data['labels']:
assert len(label) == 10
assert label[4] == '-'
assert label[7] == '-'
# Try parsing to ensure valid date
datetime.strptime(label, '%Y-%m-%d')
def test_scan_trend_consecutive_dates(self, client, auth_headers):
"""Test scan trend returns consecutive dates."""
response = client.get('/api/stats/scan-trend?days=7', headers=auth_headers)
assert response.status_code == 200
data = response.get_json()
labels = data['labels']
# Convert to datetime objects
dates = [datetime.strptime(label, '%Y-%m-%d') for label in labels]
# Check dates are consecutive
for i in range(len(dates) - 1):
diff = dates[i + 1] - dates[i]
assert diff.days == 1, f"Dates not consecutive: {dates[i]} to {dates[i+1]}"
def test_scan_trend_ends_with_today(self, client, auth_headers):
"""Test scan trend ends with today's date."""
response = client.get('/api/stats/scan-trend?days=7', headers=auth_headers)
assert response.status_code == 200
data = response.get_json()
# Last date should be today
today = datetime.utcnow().date()
last_date = datetime.strptime(data['labels'][-1], '%Y-%m-%d').date()
assert last_date == today

151
web/api/stats.py Normal file
View File

@@ -0,0 +1,151 @@
"""
Stats API blueprint.
Handles endpoints for dashboard statistics, trending data, and analytics.
"""
import logging
from datetime import datetime, timedelta
from flask import Blueprint, current_app, jsonify, request
from sqlalchemy import func, Date
from sqlalchemy.exc import SQLAlchemyError
from web.auth.decorators import api_auth_required
from web.models import Scan
bp = Blueprint('stats', __name__)
logger = logging.getLogger(__name__)
@bp.route('/scan-trend', methods=['GET'])
@api_auth_required
def scan_trend():
"""
Get scan activity trend data for charts.
Query params:
days: Number of days to include (default: 30, max: 365)
Returns:
JSON response with labels and values arrays for Chart.js
{
"labels": ["2025-01-01", "2025-01-02", ...],
"values": [5, 3, 7, 2, ...]
}
"""
try:
# Get and validate query parameters
days = request.args.get('days', 30, type=int)
# Validate days parameter
if days < 1:
return jsonify({'error': 'days parameter must be at least 1'}), 400
if days > 365:
return jsonify({'error': 'days parameter cannot exceed 365'}), 400
# Calculate date range
end_date = datetime.utcnow().date()
start_date = end_date - timedelta(days=days - 1)
# Query scan counts per day
db_session = current_app.db_session
scan_counts = (
db_session.query(
func.date(Scan.timestamp).label('scan_date'),
func.count(Scan.id).label('scan_count')
)
.filter(func.date(Scan.timestamp) >= start_date)
.filter(func.date(Scan.timestamp) <= end_date)
.group_by(func.date(Scan.timestamp))
.order_by('scan_date')
.all()
)
# Create a dictionary of date -> count
scan_dict = {str(row.scan_date): row.scan_count for row in scan_counts}
# Generate all dates in range (fill missing dates with 0)
labels = []
values = []
current_date = start_date
while current_date <= end_date:
date_str = str(current_date)
labels.append(date_str)
values.append(scan_dict.get(date_str, 0))
current_date += timedelta(days=1)
return jsonify({
'labels': labels,
'values': values,
'start_date': str(start_date),
'end_date': str(end_date),
'total_scans': sum(values)
}), 200
except SQLAlchemyError as e:
logger.error(f"Database error in scan_trend: {str(e)}")
return jsonify({'error': 'Database error occurred'}), 500
except Exception as e:
logger.error(f"Error in scan_trend: {str(e)}")
return jsonify({'error': 'An error occurred'}), 500
@bp.route('/summary', methods=['GET'])
@api_auth_required
def summary():
"""
Get dashboard summary statistics.
Returns:
JSON response with summary stats
{
"total_scans": 150,
"completed_scans": 140,
"failed_scans": 5,
"running_scans": 5,
"scans_today": 3,
"scans_this_week": 15
}
"""
try:
db_session = current_app.db_session
# Get total counts by status
total_scans = db_session.query(func.count(Scan.id)).scalar() or 0
completed_scans = db_session.query(func.count(Scan.id)).filter(
Scan.status == 'completed'
).scalar() or 0
failed_scans = db_session.query(func.count(Scan.id)).filter(
Scan.status == 'failed'
).scalar() or 0
running_scans = db_session.query(func.count(Scan.id)).filter(
Scan.status == 'running'
).scalar() or 0
# Get scans today
today = datetime.utcnow().date()
scans_today = db_session.query(func.count(Scan.id)).filter(
func.date(Scan.timestamp) == today
).scalar() or 0
# Get scans this week (last 7 days)
week_ago = today - timedelta(days=6)
scans_this_week = db_session.query(func.count(Scan.id)).filter(
func.date(Scan.timestamp) >= week_ago
).scalar() or 0
return jsonify({
'total_scans': total_scans,
'completed_scans': completed_scans,
'failed_scans': failed_scans,
'running_scans': running_scans,
'scans_today': scans_today,
'scans_this_week': scans_this_week
}), 200
except SQLAlchemyError as e:
logger.error(f"Database error in summary: {str(e)}")
return jsonify({'error': 'Database error occurred'}), 500
except Exception as e:
logger.error(f"Error in summary: {str(e)}")
return jsonify({'error': 'An error occurred'}), 500

View File

@@ -317,6 +317,7 @@ def register_blueprints(app: Flask) -> None:
from web.api.schedules import bp as schedules_bp
from web.api.alerts import bp as alerts_bp
from web.api.settings import bp as settings_bp
from web.api.stats import bp as stats_bp
from web.auth.routes import bp as auth_bp
from web.routes.main import bp as main_bp
@@ -331,6 +332,7 @@ def register_blueprints(app: Flask) -> None:
app.register_blueprint(schedules_bp, url_prefix='/api/schedules')
app.register_blueprint(alerts_bp, url_prefix='/api/alerts')
app.register_blueprint(settings_bp, url_prefix='/api/settings')
app.register_blueprint(stats_bp, url_prefix='/api/stats')
app.logger.info("Blueprints registered")

View File

@@ -50,6 +50,51 @@
<span id="trigger-btn-spinner" class="spinner-border spinner-border-sm ms-2" style="display: none;"></span>
</button>
<a href="{{ url_for('main.scans') }}" class="btn btn-secondary btn-lg ms-2">View All Scans</a>
<a href="{{ url_for('main.schedules') }}" class="btn btn-secondary btn-lg ms-2">
<i class="bi bi-calendar-plus"></i> Manage Schedules
</a>
</div>
</div>
</div>
</div>
<!-- Scan Activity Chart -->
<div class="row mb-4">
<div class="col-md-8">
<div class="card">
<div class="card-header">
<h5 class="mb-0" style="color: #60a5fa;">Scan Activity (Last 30 Days)</h5>
</div>
<div class="card-body">
<div id="chart-loading" class="text-center py-4">
<div class="spinner-border" role="status">
<span class="visually-hidden">Loading...</span>
</div>
</div>
<canvas id="scanTrendChart" height="100" style="display: none;"></canvas>
</div>
</div>
</div>
<!-- Schedules Widget -->
<div class="col-md-4">
<div class="card h-100">
<div class="card-header d-flex justify-content-between align-items-center">
<h5 class="mb-0" style="color: #60a5fa;">Upcoming Schedules</h5>
<a href="{{ url_for('main.schedules') }}" class="btn btn-sm btn-secondary">Manage</a>
</div>
<div class="card-body">
<div id="schedules-loading" class="text-center py-4">
<div class="spinner-border spinner-border-sm" role="status">
<span class="visually-hidden">Loading...</span>
</div>
</div>
<div id="schedules-content" style="display: none;"></div>
<div id="schedules-empty" class="text-muted text-center py-4" style="display: none;">
No schedules configured yet.
<br><br>
<a href="{{ url_for('main.create_schedule') }}" class="btn btn-sm btn-primary">Create Schedule</a>
</div>
</div>
</div>
</div>
@@ -140,6 +185,8 @@
document.addEventListener('DOMContentLoaded', function() {
refreshScans();
loadStats();
loadScanTrend();
loadSchedules();
// Auto-refresh every 10 seconds if there are running scans
refreshInterval = setInterval(function() {
@@ -149,6 +196,9 @@
loadStats();
}
}, 10000);
// Refresh schedules every 30 seconds
setInterval(loadSchedules, 30000);
});
// Load dashboard stats
@@ -329,6 +379,162 @@
}
}
// Load scan trend chart
async function loadScanTrend() {
const chartLoading = document.getElementById('chart-loading');
const canvas = document.getElementById('scanTrendChart');
try {
const response = await fetch('/api/stats/scan-trend?days=30');
if (!response.ok) {
throw new Error('Failed to load trend data');
}
const data = await response.json();
// Hide loading, show chart
chartLoading.style.display = 'none';
canvas.style.display = 'block';
// Create chart
const ctx = canvas.getContext('2d');
new Chart(ctx, {
type: 'line',
data: {
labels: data.labels,
datasets: [{
label: 'Scans per Day',
data: data.values,
borderColor: '#60a5fa',
backgroundColor: 'rgba(96, 165, 250, 0.1)',
tension: 0.3,
fill: true
}]
},
options: {
responsive: true,
maintainAspectRatio: true,
plugins: {
legend: {
display: false
},
tooltip: {
mode: 'index',
intersect: false,
callbacks: {
title: function(context) {
return new Date(context[0].label).toLocaleDateString();
}
}
}
},
scales: {
y: {
beginAtZero: true,
ticks: {
stepSize: 1,
color: '#94a3b8'
},
grid: {
color: '#334155'
}
},
x: {
ticks: {
color: '#94a3b8',
maxRotation: 0,
autoSkip: true,
maxTicksLimit: 10
},
grid: {
color: '#334155'
}
}
}
}
});
} catch (error) {
console.error('Error loading chart:', error);
chartLoading.innerHTML = '<p class="text-muted">Failed to load chart data</p>';
}
}
// Load upcoming schedules
async function loadSchedules() {
const loadingEl = document.getElementById('schedules-loading');
const contentEl = document.getElementById('schedules-content');
const emptyEl = document.getElementById('schedules-empty');
try {
const response = await fetch('/api/schedules?per_page=5');
if (!response.ok) {
throw new Error('Failed to load schedules');
}
const data = await response.json();
const schedules = data.schedules || [];
loadingEl.style.display = 'none';
if (schedules.length === 0) {
emptyEl.style.display = 'block';
} else {
contentEl.style.display = 'block';
// Filter enabled schedules and sort by next_run
const enabledSchedules = schedules
.filter(s => s.enabled && s.next_run)
.sort((a, b) => new Date(a.next_run) - new Date(b.next_run))
.slice(0, 3);
if (enabledSchedules.length === 0) {
contentEl.innerHTML = '<p class="text-muted">No enabled schedules</p>';
} else {
contentEl.innerHTML = enabledSchedules.map(schedule => {
const nextRun = new Date(schedule.next_run);
const now = new Date();
const diffMs = nextRun - now;
const diffMins = Math.floor(diffMs / 60000);
const diffHours = Math.floor(diffMins / 60);
const diffDays = Math.floor(diffHours / 24);
let timeStr;
if (diffMins < 1) {
timeStr = 'In less than 1 minute';
} else if (diffMins < 60) {
timeStr = `In ${diffMins} minute${diffMins === 1 ? '' : 's'}`;
} else if (diffHours < 24) {
timeStr = `In ${diffHours} hour${diffHours === 1 ? '' : 's'}`;
} else if (diffDays < 7) {
timeStr = `In ${diffDays} day${diffDays === 1 ? '' : 's'}`;
} else {
timeStr = nextRun.toLocaleDateString();
}
return `
<div class="mb-3 pb-3 border-bottom">
<div class="d-flex justify-content-between align-items-start">
<div>
<strong>${schedule.name}</strong>
<br>
<small class="text-muted">${timeStr}</small>
<br>
<small class="text-muted mono">${schedule.cron_expression}</small>
</div>
</div>
</div>
`;
}).join('');
}
}
} catch (error) {
console.error('Error loading schedules:', error);
loadingEl.style.display = 'none';
contentEl.style.display = 'block';
contentEl.innerHTML = '<p class="text-muted">Failed to load schedules</p>';
}
}
// Delete scan
async function deleteScan(scanId) {
if (!confirm(`Are you sure you want to delete scan ${scanId}?`)) {