{{ it.title }}
+ {% if it.priority is not none %} + + P{{ it.priority }} + + {% endif %} +{{ it.goal }}
+ + +{{ it.id }}
+
+
+
+
+ diff --git a/app/__init__.py b/app/__init__.py index 4f447d7..a3d0c3b 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -7,8 +7,9 @@ from flask import Flask from .utils.settings import get_settings from .logging_setup import wire_logging_once, get_app_logger, get_engine_logger -from app.blueprints import ui # ui blueprint -from app.blueprints import api # api blueprint +from app.blueprints.ui import bp as main_bp # ui blueprint +from app.blueprints.api import api_bp as api_bp # api blueprint +from app.blueprints.roadmap import bp as roadmap_bp # roadmap def create_app() -> Flask: """ @@ -41,9 +42,14 @@ def create_app() -> Flask: app.config["APP_NAME"] = settings.app.name app.config["APP_VERSION"] = f"v{settings.app.version_major}.{settings.app.version_minor}" + # roadmap file + app.config["ROADMAP_FILE"] = str(Path(app.root_path) / "docs" / "roadmap.yaml") + + # Register blueprints - app.register_blueprint(ui.bp) - app.register_blueprint(api.api_bp) + app.register_blueprint(main_bp) + app.register_blueprint(api_bp) + app.register_blueprint(roadmap_bp) app_logger = get_app_logger() diff --git a/app/blueprints/roadmap.py b/app/blueprints/roadmap.py new file mode 100644 index 0000000..b87b521 --- /dev/null +++ b/app/blueprints/roadmap.py @@ -0,0 +1,173 @@ +""" +Roadmap view: loads data/roadmap.yaml, sorts and renders with filters. + +Query params (all optional): +- q=... (substring search over title/goal/id) +- tag=tag1&tag=tag2 (multi; include if item has ALL selected tags) +- min_priority=1..9 (int; keep items with priority >= this) +- milestone=v0.2 (string; exact match on milestone) +- section=roadmap|backlog|open_questions (default=roadmap) +""" + +from __future__ import annotations +from dataclasses import dataclass, field +from functools import lru_cache +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +import time +import yaml +from flask import Blueprint, render_template, request, abort, current_app + +from app.logging_setup import get_app_logger +logger = get_app_logger() + +bp = Blueprint("roadmap", __name__) + +@dataclass +class RoadmapItem: + id: str + title: str + goal: str + tags: List[str] = field(default_factory=list) + priority: Optional[int] = None + milestone: Optional[str] = None + details: List[str] = field(default_factory=list) + + +@dataclass +class RoadmapData: + updated: Optional[str] + roadmap: List[RoadmapItem] + backlog: List[RoadmapItem] + open_questions: List[RoadmapItem] + +def _normalize_details(val) -> List[str]: + # Accept string (block scalar) or list of strings; normalize to list[str] + if not val: + return [] + if isinstance(val, str): + # split on blank lines while preserving paragraphs + parts = [p.strip() for p in val.strip().split("\n\n")] + return [p for p in parts if p] + if isinstance(val, list): + return [str(x) for x in val if str(x).strip()] + # Fallback: stringify unknown types + return [str(val)] + +def _to_items(raw: List[Dict[str, Any]]) -> List[RoadmapItem]: + items: List[RoadmapItem] = [] + for obj in raw or []: + items.append( + RoadmapItem( + id=str(obj.get("id", "")), + title=str(obj.get("title", "")), + goal=str(obj.get("goal", "")), + tags=list(obj.get("tags", []) or []), + priority=obj.get("priority"), + milestone=obj.get("milestone"), + details=_normalize_details(obj.get("details")), + ) + ) + return items + + +def load_roadmap() -> RoadmapData: + """Load YAML and return structured RoadmapData (no caching).""" + path = Path(current_app.config.get("ROADMAP_FILE")) + with path.open("r", encoding="utf-8") as f: + raw = yaml.safe_load(f) or {} + + return RoadmapData( + updated=raw.get("updated"), + roadmap=_to_items(raw.get("roadmap", [])), + backlog=_to_items(raw.get("backlog", [])), + open_questions=_to_items(raw.get("open_questions", [])), + ) + + +def _apply_filters( + items: List[RoadmapItem], + query: str, + tags: List[str], + min_priority: Optional[int], + milestone: Optional[str], +) -> List[RoadmapItem]: + def matches(item: RoadmapItem) -> bool: + # text search over id/title/goal + if query: + hay = f"{item.id} {item.title} {item.goal}".lower() + if query not in hay: + return False + + # tag filter (AND) + if tags: + if not set(tags).issubset(set(item.tags)): + return False + + # min priority + if min_priority is not None and item.priority is not None: + if item.priority < min_priority: + return False + + # milestone + if milestone: + if (item.milestone or "").strip() != milestone.strip(): + return False + + return True + + # sort: priority asc (None last), then title + def sort_key(i: RoadmapItem) -> Tuple[int, str]: + pri = i.priority if i.priority is not None else 9999 + return (pri, i.title.lower()) + + return sorted([i for i in items if matches(i)], key=sort_key) + + +def _collect_all_tags(data: RoadmapData) -> List[str]: + seen = set() + for col in (data.roadmap, data.backlog, data.open_questions): + for i in col: + for t in i.tags: + seen.add(t) + return sorted(seen) + + +@bp.route("/roadmap") +def roadmap_view(): + data = load_roadmap() + + # which column? + section = request.args.get("section", "roadmap") + if section not in {"roadmap", "backlog", "open_questions"}: + abort(400, "invalid section") + + # filters + q = (request.args.get("q") or "").strip().lower() + tags = request.args.getlist("tag") + min_priority = request.args.get("min_priority") + milestone = request.args.get("milestone") or None + try: + min_priority_val = int(min_priority) if min_priority else None + except ValueError: + min_priority_val = None + + # pick list + filter + source = getattr(data, section) + items = _apply_filters(source, q, tags, min_priority_val, milestone) + + # tag universe for sidebar chips + all_tags = _collect_all_tags(data) + + return render_template( + "roadmap.html", + updated=data.updated, + section=section, + items=items, + all_tags=all_tags, + q=q, + selected_tags=tags, + min_priority=min_priority_val, + milestone=milestone, + ) diff --git a/app/docs/roadmap.yaml b/app/docs/roadmap.yaml new file mode 100644 index 0000000..24a2503 --- /dev/null +++ b/app/docs/roadmap.yaml @@ -0,0 +1,214 @@ +# roadmap.yaml +updated: "2025-08-22" + +roadmap: + - id: "p1-analysis-cloudflare" + priority: 1 + title: "Cloudflare Detection" + goal: "Detect Cloudflare usage and badge it, with explanation of dual-use (security vs. abuse)." + tags: ["analysis"] + milestone: null + details: + - "Detection signals: DNS (CNAME to Cloudflare, AS13335), HTTP headers (cf-ray, cf-cache-status), IP ranges, and challenge pages." + - "UI: add badge + tooltip with a short explainer about legitimate protection vs. abuse evasion." + - "Edge cases: 'grey-clouded' DNS entries, partial proxy (only some records), and CDN in front of non-HTTP services." + - "Acceptance: correctly identifies Cloudflare on known test hosts and avoids false positives on non-CF CDNs." + + - id: "p1-analysis-total-score" + priority: 1 + title: "Total Score" + goal: "Implement a generalized site “Total Score” (0–10 scale) to give analysts a quick risk snapshot." + tags: ["analysis"] + milestone: null + details: + - "Inputs: TLS posture, suspicious scripts/forms (severity-weighted), domain/IP reputation, server headers/misconfigs." + - "Method: weighted components with neutral defaults when data is unavailable; avoid over-penalizing partial signals." + - "Explainability: always show a breakdown and contribution per component; include a 'Why?' link in the UI." + - "Calibration: start with heuristic weights, then calibrate on a test set; store weights in settings.yaml." + + - id: "p2-ui-rules-lab" + priority: 2 + title: "Rules Lab" + goal: "Build a WYSIWYG Rules Lab (paste, validate, run against sample text)." + tags: ["ui"] + milestone: null + details: + - "Features: syntax-highlighted editor, rule validation, run against sample payloads, show matches/captures, timing." + - "Samples: ship a small library of example texts and rules; allow users to save their own samples (local storage)." + - "Safety: no external network calls; size/time limits to prevent runaway regex; clear error messages." + - "UX: one-click copy of rule JSON; link to docs on rule schema." + + - id: "p2-ui-usage-page" + priority: 2 + title: "Usage Page" + goal: "Create a “Usage” page to explain app functionality." + tags: ["ui","docs"] + milestone: null + details: + - "Content: quickstart, supported analyses, cache vs. re-run behavior, artifact locations." + - "Include: screenshots/GIFs, API curl examples, link to OpenAPI docs." + - "Notes: clarify privacy, what we store, and retention defaults." + + - id: "move-changelog-into-app" + priority: 2 + title: "Move Changelog into App" + goal: "Moves Changelog into App" + tags: ["ui","docs"] + milestone: null + details: + - "Notes:Makes it much easier for users to see what's happening" + - "Content: changelog.md already in docs." + + - id: "p2-ui-about-page" + priority: 2 + title: "About Page" + goal: "Create an “About” page with project context." + tags: ["ui","docs"] + milestone: null + details: + - "Content: project purpose, high-level architecture diagram, technology stack." + - "Meta: version, commit hash, build date; link to repo and roadmap." + - "Governance: disclaimer about intended use and limitations." + + - id: "p3-api-core-endpoints" + priority: 3 + title: "Core Endpoints" + goal: "Add `/screenshot`, `/source`, and `/analyse` endpoints." + tags: ["api"] + milestone: null + details: + - "Define request/response schemas; include run_id in responses to tie artifacts together." + - "Auth: simple token header; rate-limiting per token." + - "Errors: standardized JSON error body; consistent HTTP codes." + - "Docs: provide curl examples; note synchronous vs. long-running behavior." + + - id: "p3-api-analyze-script" + priority: 3 + title: "Analyze Script Endpoint" + goal: "Add POST /api/analyze_script in OpenAPI and serve /api/openapi.yaml." + tags: ["api"] + milestone: null + details: + - "Request: raw script text or URL; size cap; optional rule-set selection." + - "Processing: run rules engine; return matched rule names, severities, and excerpts." + - "Artifacts: store hashed script with metadata; include reference in response." + - "Validation: reject binary content; enforce content-type and max size." + + - id: "p3-api-docs-ui" + priority: 3 + title: "API Docs UI" + goal: "Provide interactive docs (Swagger UI or Redoc) at /docs." + tags: ["api"] + milestone: null + details: + - "Serve OpenAPI from /api/openapi.yaml; auto-refresh on rebuild." + - "Swagger UI 'try it out' toggle; disable in prod if needed." + - "Theming to match app; link to Usage page for context." + + - id: "p3-api-json-errors" + priority: 3 + title: "JSON Error Consistency" + goal: "Ensure JSON error consistency across 400–500 responses." + tags: ["api", "nice-to-have"] + milestone: null + details: + - "Schema: {\"error\": {\"code\": int, \"message\": str, \"details\": object, \"correlation_id\": str}}." + - "Implement Flask error handlers; return JSON for 400/403/404/405/500." + - "Log: include correlation_id in logs; surface it in responses for support." + + - id: "p4-ops-retention-policy" + priority: 4 + title: "Retention Policy" + goal: "Define retention thresholds for artifacts (age/size)." + tags: ["ops"] + milestone: null + details: + - "Policy: max age per artifact type; total size caps per workspace." + - "Configuration: settings.yaml-driven; per-type overrides." + - "Safety: dry-run mode and deletion preview; minimum free space guard." + + - id: "p4-ops-cleanup-scripts" + priority: 4 + title: "Cleanup Scripts" + goal: "Implement cleanup/maintenance scripts, driven by settings.yaml." + tags: ["ops"] + milestone: null + details: + - "CLI: list, simulate, prune; log summary of bytes reclaimed and items removed." + - "Scheduling: optional cron/apscheduler task; lock to prevent concurrent runs." + - "Observability: emit metrics (counts, durations) to logs." + + - id: "p4-ops-results-cache" + priority: 4 + title: "Results Cache" + goal: "Add UX toggle: “Re-run analysis” vs. “Load from cache.”" + tags: ["ops"] + milestone: null + details: + - "Cache key: normalized URL + analysis settings; include versioning to bust on rule changes." + - "UI: clearly label cached vs. fresh; provide 'Invalidate cache' action." + - "TTL: setting-driven; guard against stale security results." + + - id: "p5-intel-domain-reputation" + priority: 5 + title: "Domain Reputation" + goal: "Build consolidated reputation store (URLHaus, OpenPhish)." + tags: ["intel"] + milestone: null + details: + - "Ingestion: scheduled pulls; parse feeds; dedupe and normalize indicators." + - "Storage: compact on-disk DB (e.g., sqlite/duckdb) keyed by domain/URL with timestamps." + - "Use: query during analysis; add context to findings with source + first_seen/last_seen." + + - id: "p5-intel-threat-connectors" + priority: 5 + title: "Threat Intel Connectors" + goal: "Add connectors for VirusTotal, ThreatFox, and future providers (via settings.yaml)." + tags: ["intel"] + milestone: null + details: + - "Config: enable per-connector with API keys via settings.yaml or env." + - "Runtime: rate limiting and backoff; cache responses to reduce cost/latency." + - "Merge: normalize verdicts and confidence; avoid double-counting against Total Score." + +backlog: + - id: "backlog-scan-server-profile" + title: "Server Profile Scan" + goal: "Run lightweight nmap scan on web/alt ports, merge with headers for stack inference." + tags: ["scan"] + milestone: null + details: + - "Scope: common ports (80,443,8000,8080,8443,22); banner grab only; conservative timing." + - "Inference: combine banners + headers to guess stack (IIS vs. nginx/Apache)." + - "Controls: opt-in, with time and port limits to avoid noisy scans." + + - id: "backlog-intel-ip-reputation" + title: "IP Reputation Expansion" + goal: "Expand reputation checks to IP blocklists and datasets." + tags: ["intel"] + milestone: null + details: + - "Sources: community blocklists with permissive licenses; document any commercial sources separately." + - "Model: score IPs with decay over time; avoid permanent penalties for stale abuse." + - "Integration: surface as context; do not overrule domain-level signals." + +open_questions: + - id: "design-imports-unification" + title: "Imports Unification" + goal: "Decide if imports/utilities (e.g., decorators) should be centralized in state.py." + tags: ["design"] + milestone: null + details: + - "Pros: consistent imports, fewer circular references, easier testing." + - "Cons: can become a god-module; hidden dependencies." + - "Proposal: a small 'core/state.py' for app-wide state + 'utils/' packages for helpers." + + - id: "design-score-calibration" + title: "Score Calibration" + goal: "Define and calibrate methodology for the Total Score scale." + tags: ["design"] + milestone: null + details: + - "Dataset: assemble a labeled set of benign/suspicious sites for tuning." + - "Approach: start with manual weights, then fit via simple regression or grid search." + - "Outcome: publish thresholds for low/medium/high along with examples." diff --git a/app/templates/base.html b/app/templates/base.html index 2d760a9..eefba8f 100644 --- a/app/templates/base.html +++ b/app/templates/base.html @@ -30,6 +30,11 @@ Home +
Last updated: {{ updated }}
+ {% endif %} +{{ it.goal }}
+ + +{{ it.id }}
+
+
+
+
+