diff --git a/app/templates/partials/result_text.html b/app/templates/partials/result_text.html
index a1e54f4..f5fae21 100644
--- a/app/templates/partials/result_text.html
+++ b/app/templates/partials/result_text.html
@@ -1,6 +1,6 @@
- Text
+ Suspicious Text
{% if suspicious_text and suspicious_text|length > 0 %}
diff --git a/app/utils/enrichment.py b/app/utils/enrichment.py
index 70b0f4d..ad57c67 100644
--- a/app/utils/enrichment.py
+++ b/app/utils/enrichment.py
@@ -30,6 +30,8 @@ days = 24 * 60
GEOIP_DEFAULT_TTL = settings.cache.geoip_cache_days * days
WHOIS_DEFAULT_TTL = settings.cache.whois_cache_days * days
+CRT_DEFAULT_TTL = settings.cache.crt_cache_days * days
+
logger = get_app_logger()
@@ -137,6 +139,20 @@ def search_certs(domain, wildcard=True, expired=True, deduplicate=True):
"not_before": "2018-02-08T15:47:39"
}
"""
+
+ cache_key = f"crt_cert:{domain}"
+
+ # log if caching is turned on or not
+ logger.info(f"CRT Cache is set to: {settings.cache.crt_cache_enabled}")
+
+ if settings.cache.crt_cache_enabled:
+ cached = cache.read(cache_key)
+ if cached:
+ logger.info(f"[CACHE HIT] for CRT Cert: {domain}")
+ return cached
+ else:
+ logger.info(f"[CACHE MISS] for CRT Cert: {domain} - {cache_key}")
+
base_url = "https://crt.sh/?q={}&output=json"
if not expired:
base_url = base_url + "&exclude=expired"
@@ -153,11 +169,21 @@ def search_certs(domain, wildcard=True, expired=True, deduplicate=True):
try:
content = req.content.decode('utf-8')
data = json.loads(content)
+ # if caching
+ if settings.cache.crt_cache_enabled:
+ logger.info(f"Setting Cache for {cache_key}")
+ cache.create(cache_key, data, CRT_DEFAULT_TTL)
return data
except ValueError:
# crt.sh fixed their JSON response. This shouldn't be necessary anymore
# https://github.com/crtsh/certwatch_db/commit/f4f46ea37c23543c4cdf1a3c8867d68967641807
data = json.loads("[{}]".format(content.replace('}{', '},{')))
+
+ # if caching
+ if settings.cache.crt_cache_enabled:
+ logger.info(f"Setting Cache for {cache_key}")
+ cache.create(cache_key, data, CRT_DEFAULT_TTL)
+
return data
except Exception as err:
logger.error("Error retrieving cert information from CRT.sh.")
@@ -200,6 +226,7 @@ def gather_crtsh_certs_for_target(target):
hostname = parse_target_to_host(target)
result["hostname"] = hostname
+ # return fake return if no hostname was able to be parsed
if hostname is None:
return result
@@ -209,6 +236,7 @@ def gather_crtsh_certs_for_target(target):
# Always query crt.sh for the specific hostname
# (expired=False means we filter expired)
+
host_certs = search_certs(hostname, wildcard=False, expired=False)
result["crtsh"]["host_certs"] = host_certs
diff --git a/app/utils/settings.py b/app/utils/settings.py
index 1f5acd7..7c05476 100644
--- a/app/utils/settings.py
+++ b/app/utils/settings.py
@@ -53,27 +53,34 @@ class UIConfig:
@dataclass
class Cache_Config:
+ recent_runs_count: int = 10
+
whois_cache_days: int = 7
geoip_cache_days: int = 7
- recent_runs_count: int = 10
+ crt_cache_enabled: bool = True
+ crt_cache_days: int = 7
+
+
@dataclass
-class AppConfig:
- name: str = "MyApp"
- version_major: int = 1
- version_minor: int = 0
+class Logging_Config:
log_rule_loads: bool = False
log_rule_dispatch: bool = False
log_rule_debug: bool = False
+@dataclass
+class BrandingConfig:
+ name: str = "MyApp"
+
@dataclass
class Settings:
cache: Cache_Config = field(default_factory=Cache_Config)
ui: UIConfig = field(default_factory=UIConfig)
external_fetch: External_FetchConfig = field(default_factory=External_FetchConfig)
- app: AppConfig = field(default_factory=AppConfig)
+ branding: BrandingConfig = field(default_factory=BrandingConfig)
+ logconfig: Logging_Config = field(default_factory=Logging_Config)
@classmethod
def from_yaml(cls, path: str | Path) -> "Settings":
diff --git a/app/wsgi.py b/app/wsgi.py
index 35f38ba..f0a0733 100644
--- a/app/wsgi.py
+++ b/app/wsgi.py
@@ -9,7 +9,7 @@ from . import create_app
# Gunicorn will look for "app"
app = create_app()
-from app.state import set_rules_engine, get_rules_engine
+from app.state import set_rules_engine
from app.logging_setup import get_app_logger
from app.rules.factory import build_rules_engine
diff --git a/docs/rule_processing_notes.md b/docs/rule_processing_notes.md
new file mode 100644
index 0000000..d0081f0
--- /dev/null
+++ b/docs/rule_processing_notes.md
@@ -0,0 +1,24 @@
+Blueprint -> Analyse calls browser.fetch_page_artifacts
+
+Browser.py -> fetch_page_artifacts
+grabs html_content
+ run analyse_forms on html_content
+ run analyse_scripts on html_content
+ enrich_url
+ build_rule_checks_overview
+
+builds the structured dict for return
+dumps data to disk
+
+
+text flag mapping for rules
+# --- Helper: map human-friendly flags to re.* constants ---
+ FLAG_MAP = {
+ "i": _re.IGNORECASE, "ignorecase": _re.IGNORECASE,
+ "m": _re.MULTILINE, "multiline": _re.MULTILINE,
+ "s": _re.DOTALL, "dotall": _re.DOTALL, "singleline": _re.DOTALL,
+ "x": _re.VERBOSE, "verbose": _re.VERBOSE,
+ "a": _re.ASCII, "ascii": _re.ASCII,
+ "u": _re.UNICODE, "unicode": _re.UNICODE,
+ "l": _re.LOCALE, "locale": _re.LOCALE,
+ }
\ No newline at end of file
diff --git a/docs/rules-rebuild-remember.md b/docs/rules-rebuild-remember.md
new file mode 100644
index 0000000..5349412
--- /dev/null
+++ b/docs/rules-rebuild-remember.md
@@ -0,0 +1,113 @@
+Got it — here’s a structured outline you can keep as a “memory primer” for when you want to resume this work (whether continuing the current engine or pivoting to a SaaS build). Think of it as a **checkpoint map** of what we’ve covered and what directions you flagged.
+
+---
+
+# SneakyScope – Rules Engine Refactor & SaaS Considerations (Outline)
+
+## 1. Current State
+
+* Engine: `app/rules/rules_engine.py` — mixed YAML + code rules, `(bool, str)` return shape expected.
+* Issue: function-based rules returning `(bool, dict)` → caused invalid type warnings.
+* Stopgap: `FunctionRuleAdapter` converted returns.
+* Desire: Remove YAML entirely; rules defined in code only.
+
+---
+
+## 2. Agreed Direction
+
+* **Rule definition approach**:
+
+ * Option B chosen → decorator-based registration.
+ * Every rule defined in `app/rules/` as Python functions.
+ * Rules register with metadata (`name`, `category`, `severity`, etc.).
+
+* **Return shape**:
+
+ * Always return a **Result dict** (no adapter needed).
+ * Engine enforces schema and fills in defaults.
+
+* **Engine relocation**:
+
+ * Move to `app/utils/rules_engine/`.
+ * Responsibilities: load, validate, freeze registry, run rules, aggregate results, log/report.
+
+---
+
+## 3. Result Schema (concept)
+
+* **Per RuleResult**
+
+ * Required: `ok: bool`, `message: str`.
+ * Identity: `name`, `category`, `severity`, `tags`, `rule_version`.
+ * Detail: `data: object|null`.
+ * Timing: `duration_ms`.
+ * Errors: structured `error` object if exceptions occur.
+ * Provenance: `source_module`, optional `policy` snapshot.
+
+* **Per AnalysisResult (run-level envelope)**
+
+ * Input scope: target URL, category, content hash, facts profile.
+ * Provenance: run\_id, engine\_version, ruleset\_checksum, timestamp, duration.
+ * Results: array of RuleResults.
+ * Summary: counts by severity, match count, errors, first match, top severity.
+ * Artifacts: references (screenshot, DOM snapshot, etc.).
+ * Policy snapshot: optional central policy/overrides.
+
+---
+
+## 4. Operational Standards
+
+* **Determinism**: identical inputs + ruleset\_checksum → identical results.
+* **Message stability**: avoid wording churn; expand via `data`.
+* **Size limits**: `message ≤ 256 chars`; `data ≤ 8–16 KB`.
+* **Errors**: `ok=false` if error present; always emit `message`.
+* **Severity**: rule sets default; policy may override.
+* **Tags**: controlled vocabulary; additive.
+
+---
+
+## 5. Migration Plan
+
+1. Create new `rules_engine` package in `app/utils/`.
+2. Add decorator/registry for rules.
+3. Port all rules from YAML → Python modules grouped by category.
+4. Delete YAML loader + adapters.
+5. Update call sites to build `facts` and call `engine.run(...)`.
+6. Add CI tests:
+
+ * Schema compliance.
+ * No duplicates.
+ * Ruleset checksum snapshot.
+7. Integration tests with real fixtures.
+8. Benchmark & harden (caps on input size, rule runtime).
+
+---
+
+## 6. SaaS Expansion (future)
+
+* **Multi-tenancy**: separate org/user scopes for data and rule runs.
+* **RBAC**: roles (admin, analyst, viewer).
+* **Compliance**: logging, retention, export, audit trails.
+* **Rules**: centrally maintained, not user-editable.
+* **APIs**: authenticated endpoints, per-user quotas.
+* **Observability**: per-tenant metrics, alerts.
+* **Security**: sandboxing, strict module allowlists, compliance with SOC2/ISO.
+* **Data controls**: PII redaction, encryption, retention policies.
+
+---
+
+## 7. Future-Proofing Hooks
+
+* Versioning: ruleset checksum + per-rule versions.
+* Extensibility: support `actions`, `links`, `evidence` in Result.
+* Policy: central config for thresholds/overrides.
+* Hot reload (optional, dev-only).
+* Rule provenance tracking (source\_module, commit SHA).
+
+---
+
+✅ This outline is enough to “re-hydrate” the context later — you won’t need to dig back into old logs to remember why `(bool, str)` didn’t fit, why YAML was removed, or what schema we were converging on.
+
+---
+
+Do you want me to also save this in a **short “README-spec” style** (like `RESULTS.md`) so it can live in your repo as the contract doc for rules, or should I keep this as just your personal checkpoint outline?