From cd30cde9462c0f3d63de6d885d23557307523ac2 Mon Sep 17 00:00:00 2001 From: Phillip Tarrant Date: Fri, 22 Aug 2025 15:05:09 -0500 Subject: [PATCH] feat(roadmap): YAML-driven roadmap + Tailwind UI w/ filters & details modal - Convert roadmap to YAML: - Add data structure: id, priority, title, goal, tags, milestone - Add `details` field (supports list or block string); populated initial content - Quote scalars and use explicit nulls to avoid YAML parse edge cases - Update `updated` date to 2025-08-22 - Flask blueprint + loader: - New /roadmap view with section switching (roadmap | backlog | open_questions) - Filters: q (search), tag (multi, AND), min_priority, milestone - Dataclasses: RoadmapData/RoadmapItem; include `details` - `_normalize_details()` to accept string or list, normalize to list[str] - Configurable path via `ROADMAP_FILE` (env or defaults) - Remove cache layer for simplicity - UI (Tailwind): - `templates/roadmap.html` with responsive cards, tag chips, and filter form - Details modal (larger max width, scrollable body) showing ID/goal/priority/tags/milestone - Safe JSON payload to modal via `|tojson|forceescape` - JS: - DOM-ready, event-delegated handler for `data-item` buttons - Populate modal fields and render multi-paragraph details - Fixes & polish: - Resolved YAML `ScannerError` by quoting strings with `:` and `#` - Ensured `details` is passed through route to template and included in button payload - Minor styling tweaks for consistency with Tailwind setup Usage: - Set `ROADMAP_FILE` if not using default path - Visit /roadmap and filter via q/tag/min_priority/milestone --- app/__init__.py | 14 +- app/blueprints/roadmap.py | 173 ++++++++++++++++++++++++ app/docs/roadmap.yaml | 214 ++++++++++++++++++++++++++++++ app/templates/base.html | 10 ++ app/templates/roadmap.html | 262 +++++++++++++++++++++++++++++++++++++ docs/changelog.md | 54 ++++++++ docs/roadmap.md | 34 ----- 7 files changed, 723 insertions(+), 38 deletions(-) create mode 100644 app/blueprints/roadmap.py create mode 100644 app/docs/roadmap.yaml create mode 100644 app/templates/roadmap.html create mode 100644 docs/changelog.md delete mode 100644 docs/roadmap.md diff --git a/app/__init__.py b/app/__init__.py index 4f447d7..a3d0c3b 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -7,8 +7,9 @@ from flask import Flask from .utils.settings import get_settings from .logging_setup import wire_logging_once, get_app_logger, get_engine_logger -from app.blueprints import ui # ui blueprint -from app.blueprints import api # api blueprint +from app.blueprints.ui import bp as main_bp # ui blueprint +from app.blueprints.api import api_bp as api_bp # api blueprint +from app.blueprints.roadmap import bp as roadmap_bp # roadmap def create_app() -> Flask: """ @@ -41,9 +42,14 @@ def create_app() -> Flask: app.config["APP_NAME"] = settings.app.name app.config["APP_VERSION"] = f"v{settings.app.version_major}.{settings.app.version_minor}" + # roadmap file + app.config["ROADMAP_FILE"] = str(Path(app.root_path) / "docs" / "roadmap.yaml") + + # Register blueprints - app.register_blueprint(ui.bp) - app.register_blueprint(api.api_bp) + app.register_blueprint(main_bp) + app.register_blueprint(api_bp) + app.register_blueprint(roadmap_bp) app_logger = get_app_logger() diff --git a/app/blueprints/roadmap.py b/app/blueprints/roadmap.py new file mode 100644 index 0000000..b87b521 --- /dev/null +++ b/app/blueprints/roadmap.py @@ -0,0 +1,173 @@ +""" +Roadmap view: loads data/roadmap.yaml, sorts and renders with filters. + +Query params (all optional): +- q=... (substring search over title/goal/id) +- tag=tag1&tag=tag2 (multi; include if item has ALL selected tags) +- min_priority=1..9 (int; keep items with priority >= this) +- milestone=v0.2 (string; exact match on milestone) +- section=roadmap|backlog|open_questions (default=roadmap) +""" + +from __future__ import annotations +from dataclasses import dataclass, field +from functools import lru_cache +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +import time +import yaml +from flask import Blueprint, render_template, request, abort, current_app + +from app.logging_setup import get_app_logger +logger = get_app_logger() + +bp = Blueprint("roadmap", __name__) + +@dataclass +class RoadmapItem: + id: str + title: str + goal: str + tags: List[str] = field(default_factory=list) + priority: Optional[int] = None + milestone: Optional[str] = None + details: List[str] = field(default_factory=list) + + +@dataclass +class RoadmapData: + updated: Optional[str] + roadmap: List[RoadmapItem] + backlog: List[RoadmapItem] + open_questions: List[RoadmapItem] + +def _normalize_details(val) -> List[str]: + # Accept string (block scalar) or list of strings; normalize to list[str] + if not val: + return [] + if isinstance(val, str): + # split on blank lines while preserving paragraphs + parts = [p.strip() for p in val.strip().split("\n\n")] + return [p for p in parts if p] + if isinstance(val, list): + return [str(x) for x in val if str(x).strip()] + # Fallback: stringify unknown types + return [str(val)] + +def _to_items(raw: List[Dict[str, Any]]) -> List[RoadmapItem]: + items: List[RoadmapItem] = [] + for obj in raw or []: + items.append( + RoadmapItem( + id=str(obj.get("id", "")), + title=str(obj.get("title", "")), + goal=str(obj.get("goal", "")), + tags=list(obj.get("tags", []) or []), + priority=obj.get("priority"), + milestone=obj.get("milestone"), + details=_normalize_details(obj.get("details")), + ) + ) + return items + + +def load_roadmap() -> RoadmapData: + """Load YAML and return structured RoadmapData (no caching).""" + path = Path(current_app.config.get("ROADMAP_FILE")) + with path.open("r", encoding="utf-8") as f: + raw = yaml.safe_load(f) or {} + + return RoadmapData( + updated=raw.get("updated"), + roadmap=_to_items(raw.get("roadmap", [])), + backlog=_to_items(raw.get("backlog", [])), + open_questions=_to_items(raw.get("open_questions", [])), + ) + + +def _apply_filters( + items: List[RoadmapItem], + query: str, + tags: List[str], + min_priority: Optional[int], + milestone: Optional[str], +) -> List[RoadmapItem]: + def matches(item: RoadmapItem) -> bool: + # text search over id/title/goal + if query: + hay = f"{item.id} {item.title} {item.goal}".lower() + if query not in hay: + return False + + # tag filter (AND) + if tags: + if not set(tags).issubset(set(item.tags)): + return False + + # min priority + if min_priority is not None and item.priority is not None: + if item.priority < min_priority: + return False + + # milestone + if milestone: + if (item.milestone or "").strip() != milestone.strip(): + return False + + return True + + # sort: priority asc (None last), then title + def sort_key(i: RoadmapItem) -> Tuple[int, str]: + pri = i.priority if i.priority is not None else 9999 + return (pri, i.title.lower()) + + return sorted([i for i in items if matches(i)], key=sort_key) + + +def _collect_all_tags(data: RoadmapData) -> List[str]: + seen = set() + for col in (data.roadmap, data.backlog, data.open_questions): + for i in col: + for t in i.tags: + seen.add(t) + return sorted(seen) + + +@bp.route("/roadmap") +def roadmap_view(): + data = load_roadmap() + + # which column? + section = request.args.get("section", "roadmap") + if section not in {"roadmap", "backlog", "open_questions"}: + abort(400, "invalid section") + + # filters + q = (request.args.get("q") or "").strip().lower() + tags = request.args.getlist("tag") + min_priority = request.args.get("min_priority") + milestone = request.args.get("milestone") or None + try: + min_priority_val = int(min_priority) if min_priority else None + except ValueError: + min_priority_val = None + + # pick list + filter + source = getattr(data, section) + items = _apply_filters(source, q, tags, min_priority_val, milestone) + + # tag universe for sidebar chips + all_tags = _collect_all_tags(data) + + return render_template( + "roadmap.html", + updated=data.updated, + section=section, + items=items, + all_tags=all_tags, + q=q, + selected_tags=tags, + min_priority=min_priority_val, + milestone=milestone, + ) diff --git a/app/docs/roadmap.yaml b/app/docs/roadmap.yaml new file mode 100644 index 0000000..24a2503 --- /dev/null +++ b/app/docs/roadmap.yaml @@ -0,0 +1,214 @@ +# roadmap.yaml +updated: "2025-08-22" + +roadmap: + - id: "p1-analysis-cloudflare" + priority: 1 + title: "Cloudflare Detection" + goal: "Detect Cloudflare usage and badge it, with explanation of dual-use (security vs. abuse)." + tags: ["analysis"] + milestone: null + details: + - "Detection signals: DNS (CNAME to Cloudflare, AS13335), HTTP headers (cf-ray, cf-cache-status), IP ranges, and challenge pages." + - "UI: add badge + tooltip with a short explainer about legitimate protection vs. abuse evasion." + - "Edge cases: 'grey-clouded' DNS entries, partial proxy (only some records), and CDN in front of non-HTTP services." + - "Acceptance: correctly identifies Cloudflare on known test hosts and avoids false positives on non-CF CDNs." + + - id: "p1-analysis-total-score" + priority: 1 + title: "Total Score" + goal: "Implement a generalized site “Total Score” (0–10 scale) to give analysts a quick risk snapshot." + tags: ["analysis"] + milestone: null + details: + - "Inputs: TLS posture, suspicious scripts/forms (severity-weighted), domain/IP reputation, server headers/misconfigs." + - "Method: weighted components with neutral defaults when data is unavailable; avoid over-penalizing partial signals." + - "Explainability: always show a breakdown and contribution per component; include a 'Why?' link in the UI." + - "Calibration: start with heuristic weights, then calibrate on a test set; store weights in settings.yaml." + + - id: "p2-ui-rules-lab" + priority: 2 + title: "Rules Lab" + goal: "Build a WYSIWYG Rules Lab (paste, validate, run against sample text)." + tags: ["ui"] + milestone: null + details: + - "Features: syntax-highlighted editor, rule validation, run against sample payloads, show matches/captures, timing." + - "Samples: ship a small library of example texts and rules; allow users to save their own samples (local storage)." + - "Safety: no external network calls; size/time limits to prevent runaway regex; clear error messages." + - "UX: one-click copy of rule JSON; link to docs on rule schema." + + - id: "p2-ui-usage-page" + priority: 2 + title: "Usage Page" + goal: "Create a “Usage” page to explain app functionality." + tags: ["ui","docs"] + milestone: null + details: + - "Content: quickstart, supported analyses, cache vs. re-run behavior, artifact locations." + - "Include: screenshots/GIFs, API curl examples, link to OpenAPI docs." + - "Notes: clarify privacy, what we store, and retention defaults." + + - id: "move-changelog-into-app" + priority: 2 + title: "Move Changelog into App" + goal: "Moves Changelog into App" + tags: ["ui","docs"] + milestone: null + details: + - "Notes:Makes it much easier for users to see what's happening" + - "Content: changelog.md already in docs." + + - id: "p2-ui-about-page" + priority: 2 + title: "About Page" + goal: "Create an “About” page with project context." + tags: ["ui","docs"] + milestone: null + details: + - "Content: project purpose, high-level architecture diagram, technology stack." + - "Meta: version, commit hash, build date; link to repo and roadmap." + - "Governance: disclaimer about intended use and limitations." + + - id: "p3-api-core-endpoints" + priority: 3 + title: "Core Endpoints" + goal: "Add `/screenshot`, `/source`, and `/analyse` endpoints." + tags: ["api"] + milestone: null + details: + - "Define request/response schemas; include run_id in responses to tie artifacts together." + - "Auth: simple token header; rate-limiting per token." + - "Errors: standardized JSON error body; consistent HTTP codes." + - "Docs: provide curl examples; note synchronous vs. long-running behavior." + + - id: "p3-api-analyze-script" + priority: 3 + title: "Analyze Script Endpoint" + goal: "Add POST /api/analyze_script in OpenAPI and serve /api/openapi.yaml." + tags: ["api"] + milestone: null + details: + - "Request: raw script text or URL; size cap; optional rule-set selection." + - "Processing: run rules engine; return matched rule names, severities, and excerpts." + - "Artifacts: store hashed script with metadata; include reference in response." + - "Validation: reject binary content; enforce content-type and max size." + + - id: "p3-api-docs-ui" + priority: 3 + title: "API Docs UI" + goal: "Provide interactive docs (Swagger UI or Redoc) at /docs." + tags: ["api"] + milestone: null + details: + - "Serve OpenAPI from /api/openapi.yaml; auto-refresh on rebuild." + - "Swagger UI 'try it out' toggle; disable in prod if needed." + - "Theming to match app; link to Usage page for context." + + - id: "p3-api-json-errors" + priority: 3 + title: "JSON Error Consistency" + goal: "Ensure JSON error consistency across 400–500 responses." + tags: ["api", "nice-to-have"] + milestone: null + details: + - "Schema: {\"error\": {\"code\": int, \"message\": str, \"details\": object, \"correlation_id\": str}}." + - "Implement Flask error handlers; return JSON for 400/403/404/405/500." + - "Log: include correlation_id in logs; surface it in responses for support." + + - id: "p4-ops-retention-policy" + priority: 4 + title: "Retention Policy" + goal: "Define retention thresholds for artifacts (age/size)." + tags: ["ops"] + milestone: null + details: + - "Policy: max age per artifact type; total size caps per workspace." + - "Configuration: settings.yaml-driven; per-type overrides." + - "Safety: dry-run mode and deletion preview; minimum free space guard." + + - id: "p4-ops-cleanup-scripts" + priority: 4 + title: "Cleanup Scripts" + goal: "Implement cleanup/maintenance scripts, driven by settings.yaml." + tags: ["ops"] + milestone: null + details: + - "CLI: list, simulate, prune; log summary of bytes reclaimed and items removed." + - "Scheduling: optional cron/apscheduler task; lock to prevent concurrent runs." + - "Observability: emit metrics (counts, durations) to logs." + + - id: "p4-ops-results-cache" + priority: 4 + title: "Results Cache" + goal: "Add UX toggle: “Re-run analysis” vs. “Load from cache.”" + tags: ["ops"] + milestone: null + details: + - "Cache key: normalized URL + analysis settings; include versioning to bust on rule changes." + - "UI: clearly label cached vs. fresh; provide 'Invalidate cache' action." + - "TTL: setting-driven; guard against stale security results." + + - id: "p5-intel-domain-reputation" + priority: 5 + title: "Domain Reputation" + goal: "Build consolidated reputation store (URLHaus, OpenPhish)." + tags: ["intel"] + milestone: null + details: + - "Ingestion: scheduled pulls; parse feeds; dedupe and normalize indicators." + - "Storage: compact on-disk DB (e.g., sqlite/duckdb) keyed by domain/URL with timestamps." + - "Use: query during analysis; add context to findings with source + first_seen/last_seen." + + - id: "p5-intel-threat-connectors" + priority: 5 + title: "Threat Intel Connectors" + goal: "Add connectors for VirusTotal, ThreatFox, and future providers (via settings.yaml)." + tags: ["intel"] + milestone: null + details: + - "Config: enable per-connector with API keys via settings.yaml or env." + - "Runtime: rate limiting and backoff; cache responses to reduce cost/latency." + - "Merge: normalize verdicts and confidence; avoid double-counting against Total Score." + +backlog: + - id: "backlog-scan-server-profile" + title: "Server Profile Scan" + goal: "Run lightweight nmap scan on web/alt ports, merge with headers for stack inference." + tags: ["scan"] + milestone: null + details: + - "Scope: common ports (80,443,8000,8080,8443,22); banner grab only; conservative timing." + - "Inference: combine banners + headers to guess stack (IIS vs. nginx/Apache)." + - "Controls: opt-in, with time and port limits to avoid noisy scans." + + - id: "backlog-intel-ip-reputation" + title: "IP Reputation Expansion" + goal: "Expand reputation checks to IP blocklists and datasets." + tags: ["intel"] + milestone: null + details: + - "Sources: community blocklists with permissive licenses; document any commercial sources separately." + - "Model: score IPs with decay over time; avoid permanent penalties for stale abuse." + - "Integration: surface as context; do not overrule domain-level signals." + +open_questions: + - id: "design-imports-unification" + title: "Imports Unification" + goal: "Decide if imports/utilities (e.g., decorators) should be centralized in state.py." + tags: ["design"] + milestone: null + details: + - "Pros: consistent imports, fewer circular references, easier testing." + - "Cons: can become a god-module; hidden dependencies." + - "Proposal: a small 'core/state.py' for app-wide state + 'utils/' packages for helpers." + + - id: "design-score-calibration" + title: "Score Calibration" + goal: "Define and calibrate methodology for the Total Score scale." + tags: ["design"] + milestone: null + details: + - "Dataset: assemble a labeled set of benign/suspicious sites for tuning." + - "Approach: start with manual weights, then fit via simple regression or grid search." + - "Outcome: publish thresholds for low/medium/high along with examples." diff --git a/app/templates/base.html b/app/templates/base.html index 2d760a9..eefba8f 100644 --- a/app/templates/base.html +++ b/app/templates/base.html @@ -30,6 +30,11 @@ Home +
  • + + Roadmap + +
    • {# Mobile toggle #} @@ -52,6 +57,11 @@ Home +
  • + + Roadmap + +
    • diff --git a/app/templates/roadmap.html b/app/templates/roadmap.html new file mode 100644 index 0000000..fed88da --- /dev/null +++ b/app/templates/roadmap.html @@ -0,0 +1,262 @@ +{% extends "base.html" %} +{% block title %}Roadmap{% endblock %} + +{% block content %} +
    + +
    +
    +

    SneakyScope Roadmap

    + {% if updated %} +

    Last updated: {{ updated }}

    + {% endif %} +
    + + +
    + + {% for t in selected_tags %} + + {% endfor %} + {% if min_priority is not none %} + + {% endif %} + {% if milestone %} + + {% endif %} + + + +
    +
    + + +
    + + + +
    + + +
    + + +
    + + +
    + + +
    + + +
    + + +
    + +
    + + +
    + +
    + {% for t in all_tags %} + + {% endfor %} +
    +
    +
    + + + {% if not items %} +
    + No items match your filters. +
    + {% endif %} + + +
    + {% for it in items %} +
    +
    +

    {{ it.title }}

    + {% if it.priority is not none %} + + P{{ it.priority }} + + {% endif %} +
    +

    {{ it.goal }}

    + + +
    + {% for tag in it.tags %} + + {{ tag }} + + {% endfor %} + {% if it.milestone %} + + milestone: {{ it.milestone }} + + {% endif %} +
    + +
    + {{ it.id }} + + + + +
    +
    + {% endfor %} +
    +
    + + + + + + +{% endblock %} + +{% block scripts %} + + + + +{% endblock %} \ No newline at end of file diff --git a/docs/changelog.md b/docs/changelog.md new file mode 100644 index 0000000..cae89e8 --- /dev/null +++ b/docs/changelog.md @@ -0,0 +1,54 @@ +# Changelog + +All notable changes to this project will be documented in this file. +This project follows [Semantic Versioning](https://semver.org/). + +--- + +## [Unreleased] + +### ✨ Features +- _Nothing yet — add upcoming features here._ + +### 🛠️ Refactors +- _Nothing yet — add upcoming refactors here._ + +### 🐛 Fixes +- _Nothing yet — add upcoming fixes here._ + +--- + + +## [v0.2] – 2025-08-22 + +### ✨ Features +- **UI Modernization** + Migrated the entire front-end to **Tailwind CSS (compiled)** with **Flowbite JS** components for better responsiveness, consistency, and developer productivity. + Introduced a **new navbar and layout system**, improving navigation and making future expansion easier. + Added **Docker-based CSS build** to keep builds reproducible and lightweight. + +- **Reusable CSS Components** + Added custom utility classes (`badge`, `badge-ok`, `badge-warn`, `badge-danger`, `chip`, `card`, etc.) to replace long Tailwind strings. + This reduces repetition and ensures a consistent look across the app. + +### 🛠️ Refactors +- **Template Includes** + Extracted shared UI sections (headers, footers, layout chunks) into separate **Jinja includes**, improving maintainability and readability of templates. + +### 🐛 Fixes +- **Table Rendering** + Locked table column widths and fixed snippet scaling issues to prevent column misalignment and content reflow. + This ensures analysis results (like script and form findings) remain readable and properly aligned. + +- **Rules Engine State** + Fixed a bug where the **rules engine** was not being pulled correctly from the application state after the previous refactor. + This restores proper detection of suspicious scripts/forms and ensures rule definitions (with `name` and `description`) are honored. + +--- + +## [v0.1] – Initial Work + +- Implemented initial **Flask-based web UI** for URL submission and analysis. +- Added **domain & IP enrichment** (WHOIS, GeoIP, ASN/ISP lookups). +- Built first version of the **Suspicious Rules Engine** for script and form detection. +- Basic Docker setup for sandboxed deployment. diff --git a/docs/roadmap.md b/docs/roadmap.md deleted file mode 100644 index 8acb3c7..0000000 --- a/docs/roadmap.md +++ /dev/null @@ -1,34 +0,0 @@ -# SneakyScope — Roadmap (Updated 8-21-25) - -## Priority 1 – Core Analysis / Stability -* if cloudflare, we notate and badge it, along with a blurp that explains how cloudflare is both used for good and evil. -* need a generalized "total score" for the site. something that is a quick 0/10 (guessing on the number), so new analyst don't have to think on the details. -* make a "dectorators" file to unify imports - -## Priority 2 – UI / UX -* Rules Lab (WYSIWYG tester): paste a rule, validate/compile, run against sample text; lightweight nav entry. -* Build reusable util classes in tailwind and replace the long class strings. Classes to build: badge, badge-ok, badge-warn, badge-danger, chip, card. - -## Priority 3 – API Layer - -* API endpoints: `/screenshot`, `/source`, `/analyse`. -* **OpenAPI**: add `POST /api/analyze_script` (request/response schemas, examples) to `openapi/openapi.yaml`; serve at `/api/openapi.yaml`. -* Docs UI: Swagger UI or Redoc at `/docs`. -* (Nice-to-have) API JSON error consistency: handlers for 400/403/404/405/500 that always return JSON. - -## Priority 4 – Artifact Management & Ops - -* Retention/cleanup policy for old artifacts (age/size thresholds). -* Make periodic maintenance scripts for storage; cleanup options set in `settings.yaml`. -* Results caching UX: add “Re-run analysis” vs. “Load from cache” controls in the results UI. - -## Priority 5 – Extras / Integrations - -* Domain reputation (local feeds): build and refresh a consolidated domain/URL reputation store from URLHaus database dump and OpenPhish community dataset (scheduled pulls with dedup/normalize). -* Threat intel connectors (settings-driven): add `settings.yaml` entries for VirusTotal and ThreatFox API keys (plus future providers); when present, enrich lookups and merge results into the unified reputation checks during analysis. - -## Backlog / Far-Off Plans - -* Server profile scan: run a lightweight nmap service/banner scan on common web/alt ports (80, 443, 8000, 8080, 8443, etc.) and SSH; combine with server headers to infer stack (e.g., IIS vs. Linux/\*nix). - -* IP Lookups 0 if we are successful on domain replutation / ip reputation \ No newline at end of file