diff --git a/api/app.py b/api/app.py index 24b1289..be229c1 100644 --- a/api/app.py +++ b/api/app.py @@ -1410,5 +1410,120 @@ def api_sitrep_generate(): except Exception as e: return jsonify({'status': 'error', 'message': str(e)}), 500 +# ─── .GOV Domain Tracker ────────────────────────────── + +def _load_govdomains(): + p = DATA_DIR / 'govdomains.json' + if p.exists(): + try: + with open(p) as f: + return json.load(f) + except Exception: + pass + return {'last_sync': None, 'total': 0, 'domains': []} + +@app.route('/api/govdomains') +def api_govdomains(): + """Return .gov domains with optional filters: range, search, type, new_only.""" + data = _load_govdomains() + domains = data.get('domains', []) + + # Range filter (by first_seen) + range_param = request.args.get('range', 'all').lower() + if range_param != 'all': + range_map = { + '24h': 1, '3d': 3, '7d': 7, '14d': 14, '30d': 30 + } + days = range_map.get(range_param, None) + if days: + from datetime import datetime as dt, timedelta + cutoff = (dt.utcnow() - timedelta(days=days)).strftime('%Y-%m-%d') + domains = [d for d in domains if d.get('first_seen', '') >= cutoff] + + # Type filter + type_param = request.args.get('type', '').strip() + if type_param: + type_lower = type_param.lower() + domains = [d for d in domains if d.get('type', '').lower() == type_lower] + + # New only filter + if request.args.get('new_only', '').lower() == 'true': + domains = [d for d in domains if d.get('is_new', False)] + + # Search filter + search = request.args.get('search', '').strip().lower() + if search: + domains = [d for d in domains if ( + search in d.get('domain', '').lower() or + search in d.get('agency', '').lower() or + search in d.get('organization', '').lower() or + search in d.get('city', '').lower() or + search in d.get('state', '').lower() + )] + + # Pagination + limit = min(int(request.args.get('limit', 500)), 2000) + offset = int(request.args.get('offset', 0)) + + return jsonify({ + 'last_sync': data.get('last_sync'), + 'total_all': data.get('total', 0), + 'total_filtered': len(domains), + 'offset': offset, + 'limit': limit, + 'domains': domains[offset:offset + limit] + }) + +@app.route('/api/govdomains/stats') +def api_govdomains_stats(): + """Return summary stats for .gov domains.""" + data = _load_govdomains() + domains = data.get('domains', []) + + # Count by type + type_counts = {} + for d in domains: + t = d.get('type', 'Unknown') or 'Unknown' + type_counts[t] = type_counts.get(t, 0) + 1 + + # Count new (24h) + from datetime import datetime as dt, timedelta + cutoff_24h = (dt.utcnow() - timedelta(days=1)).strftime('%Y-%m-%d') + cutoff_7d = (dt.utcnow() - timedelta(days=7)).strftime('%Y-%m-%d') + cutoff_30d = (dt.utcnow() - timedelta(days=30)).strftime('%Y-%m-%d') + + new_24h = sum(1 for d in domains if d.get('first_seen', '') >= cutoff_24h) + new_7d = sum(1 for d in domains if d.get('first_seen', '') >= cutoff_7d) + new_30d = sum(1 for d in domains if d.get('first_seen', '') >= cutoff_30d) + + # Load history for recent additions timeline + hist_path = DATA_DIR / 'govdomains_history.json' + history = {} + if hist_path.exists(): + try: + with open(hist_path) as f: + history = json.load(f) + except Exception: + pass + + # Recent history (last 30 days) + recent_history = {} + for date_key, entries in history.items(): + if date_key >= cutoff_30d: + count = len([e for e in entries if not e.startswith('__baseline__')]) + if count > 0: + recent_history[date_key] = count + + return jsonify({ + 'total': data.get('total', 0), + 'last_sync': data.get('last_sync'), + 'new_24h': new_24h, + 'new_7d': new_7d, + 'new_30d': new_30d, + 'by_type': type_counts, + 'recent_additions': recent_history, + 'types_list': sorted(type_counts.keys()) + }) + if __name__ == '__main__': app.run(host='0.0.0.0', port=5000, debug=False) diff --git a/api/govdomains_sync.py b/api/govdomains_sync.py new file mode 100644 index 0000000..625a76d --- /dev/null +++ b/api/govdomains_sync.py @@ -0,0 +1,228 @@ +#!/usr/bin/env python3 +"""JAESWIFT .GOV Domain Tracker — Sync Script +Clones/pulls cisagov/dotgov-data, parses current-full.csv, +tracks first_seen dates and new domain additions. +Designed to run via cron every 12 hours. +""" +import csv +import json +import os +import subprocess +import sys +from datetime import datetime, date +from pathlib import Path + +# ─── Configuration ───────────────────────────────────── +BASE_DIR = Path(__file__).parent +SOURCE_DIR = BASE_DIR / 'govdomains-source' +DATA_DIR = BASE_DIR / 'data' +DOMAINS_FILE = DATA_DIR / 'govdomains.json' +HISTORY_FILE = DATA_DIR / 'govdomains_history.json' +REPO_URL = 'https://github.com/cisagov/dotgov-data.git' +CSV_FILE = SOURCE_DIR / 'current-full.csv' + + +def log(msg): + ts = datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S UTC') + print(f'[{ts}] {msg}', flush=True) + + +def clone_or_pull(): + """Clone the CISA dotgov-data repo or pull latest changes.""" + if (SOURCE_DIR / '.git').exists(): + log('Pulling latest dotgov-data...') + result = subprocess.run( + ['git', '-C', str(SOURCE_DIR), 'pull', '--ff-only'], + capture_output=True, text=True, timeout=120 + ) + if result.returncode != 0: + log(f'Git pull failed: {result.stderr.strip()}') + # Force reset + subprocess.run( + ['git', '-C', str(SOURCE_DIR), 'fetch', 'origin'], + capture_output=True, text=True, timeout=120 + ) + subprocess.run( + ['git', '-C', str(SOURCE_DIR), 'reset', '--hard', 'origin/main'], + capture_output=True, text=True, timeout=60 + ) + log('Force-reset to origin/main') + else: + log(f'Pull OK: {result.stdout.strip()}') + else: + log('Cloning dotgov-data repo...') + SOURCE_DIR.mkdir(parents=True, exist_ok=True) + result = subprocess.run( + ['git', 'clone', '--depth', '1', REPO_URL, str(SOURCE_DIR)], + capture_output=True, text=True, timeout=300 + ) + if result.returncode != 0: + log(f'Clone failed: {result.stderr.strip()}') + sys.exit(1) + log('Clone complete') + + +def parse_csv(): + """Parse current-full.csv and return list of domain dicts.""" + if not CSV_FILE.exists(): + log(f'CSV not found: {CSV_FILE}') + sys.exit(1) + + domains = [] + with open(CSV_FILE, 'r', encoding='utf-8-sig') as f: + reader = csv.DictReader(f) + for row in reader: + domain = (row.get('Domain Name') or row.get('Domain name') or '').strip().lower() + if not domain: + continue + # Ensure .gov suffix + if not domain.endswith('.gov'): + domain += '.gov' + domains.append({ + 'domain': domain, + 'type': (row.get('Domain Type') or row.get('Domain type') or '').strip(), + 'agency': (row.get('Agency') or '').strip(), + 'organization': (row.get('Organization') or row.get('Organization name') or '').strip(), + 'city': (row.get('City') or '').strip(), + 'state': (row.get('State') or '').strip(), + 'security_contact': (row.get('Security Contact Email') or row.get('Security contact email') or '').strip(), + }) + + log(f'Parsed {len(domains)} domains from CSV') + return domains + + +def load_existing(): + """Load existing govdomains.json if it exists.""" + if DOMAINS_FILE.exists(): + try: + with open(DOMAINS_FILE, 'r') as f: + return json.load(f) + except (json.JSONDecodeError, IOError) as e: + log(f'Error loading existing data: {e}') + return None + + +def load_history(): + """Load domain addition history.""" + if HISTORY_FILE.exists(): + try: + with open(HISTORY_FILE, 'r') as f: + return json.load(f) + except (json.JSONDecodeError, IOError): + pass + return {} + + +def sync(): + """Main sync logic.""" + today = date.today().isoformat() + now_iso = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ') + + # Ensure data directory exists + DATA_DIR.mkdir(parents=True, exist_ok=True) + + # Clone or pull the repo + clone_or_pull() + + # Parse fresh CSV data + csv_domains = parse_csv() + if not csv_domains: + log('No domains parsed — aborting') + sys.exit(1) + + # Build lookup from CSV: domain -> record + csv_lookup = {} + for d in csv_domains: + csv_lookup[d['domain']] = d + + # Load existing data + existing = load_existing() + history = load_history() + + # Build lookup of existing domains: domain -> record (with first_seen) + existing_lookup = {} + if existing and 'domains' in existing: + for d in existing['domains']: + existing_lookup[d['domain']] = d + + # Merge: preserve first_seen for known domains, mark new ones + new_domains_today = [] + merged = [] + + for domain_name, csv_record in csv_lookup.items(): + if domain_name in existing_lookup: + # Existing domain — preserve first_seen, update other fields + entry = { + **csv_record, + 'first_seen': existing_lookup[domain_name].get('first_seen', today), + 'is_new': False, + } + else: + # New domain + entry = { + **csv_record, + 'first_seen': today, + 'is_new': True, + } + new_domains_today.append(domain_name) + + merged.append(entry) + + # Sort by first_seen descending then domain name + merged.sort(key=lambda x: (x['first_seen'], x['domain']), reverse=True) + + # On first run, don't mark everything as "new" in history + # (all domains get first_seen=today but that's expected) + is_first_run = existing is None + + # Build output + output = { + 'last_sync': now_iso, + 'total': len(merged), + 'new_today': len(new_domains_today), + 'is_first_run': is_first_run, + 'domains': merged, + } + + # Save domains file + with open(DOMAINS_FILE, 'w') as f: + json.dump(output, f, indent=2) + log(f'Saved {len(merged)} domains to {DOMAINS_FILE}') + + # Update history + if new_domains_today and not is_first_run: + if today not in history: + history[today] = [] + # Append without duplicating + existing_in_day = set(history[today]) + for d in new_domains_today: + if d not in existing_in_day: + history[today].append(d) + log(f'Recorded {len(new_domains_today)} new domains for {today}') + elif is_first_run: + # First run — record total as baseline, not individual domains + history[today] = [f'__baseline__:{len(merged)}_domains'] + log(f'First run — baseline of {len(merged)} domains established') + + with open(HISTORY_FILE, 'w') as f: + json.dump(history, f, indent=2) + log(f'History updated: {HISTORY_FILE}') + + # Summary + log(f'Sync complete: {len(merged)} total domains, {len(new_domains_today)} new today') + if new_domains_today and len(new_domains_today) <= 20: + for d in new_domains_today: + log(f' NEW: {d}') + elif new_domains_today: + log(f' (too many to list individually)') + + +if __name__ == '__main__': + try: + sync() + except Exception as e: + log(f'FATAL ERROR: {e}') + import traceback + traceback.print_exc() + sys.exit(1) diff --git a/css/radar.css b/css/radar.css index 32717d3..952ff8e 100644 --- a/css/radar.css +++ b/css/radar.css @@ -1,4 +1,4 @@ -/* ─── RADAR: Live Intelligence Feed ─────────────── */ +/* ─── RADAR: Live Intelligence Feed + .GOV Domain Tracker ─── */ .radar-container { max-width: clamp(1200px, 90vw, 2400px); @@ -6,6 +6,61 @@ padding: 0 2rem 3rem; } +/* ─── View Toggle ──────────────────────────────── */ +.radar-view-toggle { + display: flex; + gap: 0; + margin-bottom: 1.5rem; + border: 1px solid rgba(255, 255, 255, 0.08); + overflow: hidden; + width: fit-content; +} + +.radar-view-btn { + font-family: 'Orbitron', 'JetBrains Mono', monospace; + font-size: 0.75rem; + font-weight: 700; + letter-spacing: 2px; + padding: 0.7rem 1.5rem; + background: rgba(255, 255, 255, 0.02); + border: none; + border-right: 1px solid rgba(255, 255, 255, 0.08); + color: rgba(255, 255, 255, 0.25); + cursor: pointer; + transition: all 0.25s ease; + display: flex; + align-items: center; + gap: 0.5rem; +} + +.radar-view-btn:last-child { + border-right: none; +} + +.radar-view-btn:hover { + background: rgba(255, 170, 0, 0.04); + color: rgba(255, 170, 0, 0.5); +} + +.radar-view-btn.active { + background: rgba(255, 170, 0, 0.08); + color: rgba(255, 170, 0, 0.95); + box-shadow: inset 0 -2px 0 rgba(255, 170, 0, 0.6); +} + +.view-icon { + font-size: 0.9rem; +} + +/* ─── View Containers ──────────────────────────── */ +.radar-view { + transition: opacity 0.2s ease; +} + +.radar-view.hidden { + display: none; +} + /* ─── Controls Bar ─────────────────────────────── */ .radar-controls { display: flex; @@ -22,7 +77,8 @@ flex-wrap: wrap; } -.radar-filter { +.radar-filter, +.gov-range { font-family: 'JetBrains Mono', monospace; font-size: 0.75rem; letter-spacing: 1.5px; @@ -34,13 +90,15 @@ transition: all 0.2s ease; } -.radar-filter:hover { +.radar-filter:hover, +.gov-range:hover { background: rgba(255, 170, 0, 0.06); border-color: rgba(255, 170, 0, 0.2); color: rgba(255, 170, 0, 0.7); } -.radar-filter.active { +.radar-filter.active, +.gov-range.active { background: rgba(255, 170, 0, 0.08); border-color: rgba(255, 170, 0, 0.4); color: rgba(255, 170, 0, 0.9); @@ -100,6 +158,36 @@ to { transform: rotate(360deg); } } +/* ─── Type Filter Dropdown ─────────────────────── */ +.gov-type-filter { + font-family: 'JetBrains Mono', monospace; + font-size: 0.75rem; + letter-spacing: 1px; + padding: 0.4rem 0.8rem; + background: rgba(255, 255, 255, 0.03); + border: 1px solid rgba(255, 255, 255, 0.08); + color: rgba(255, 255, 255, 0.6); + outline: none; + cursor: pointer; + min-width: 160px; + -webkit-appearance: none; + -moz-appearance: none; + appearance: none; + background-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='10' height='6'%3E%3Cpath d='M0 0l5 6 5-6z' fill='rgba(255,170,0,0.5)'/%3E%3C/svg%3E"); + background-repeat: no-repeat; + background-position: right 0.6rem center; + padding-right: 1.8rem; +} + +.gov-type-filter:focus { + border-color: rgba(255, 170, 0, 0.4); +} + +.gov-type-filter option { + background: #0a0a0a; + color: rgba(255, 255, 255, 0.7); +} + /* ─── Stats Bar ────────────────────────────────── */ .radar-stats { display: flex; @@ -126,6 +214,11 @@ color: rgba(0, 204, 68, 0.7); } +.stat-new { + color: rgba(0, 204, 68, 0.9); + font-weight: 700; +} + .radar-live { display: flex; align-items: center; @@ -142,12 +235,27 @@ animation: pulse-dot 1.5s ease infinite; } +.gov-dot { + background: #00cc44; +} + @keyframes pulse-dot { 0%, 100% { opacity: 1; box-shadow: 0 0 4px #00cc44; } 50% { opacity: 0.4; box-shadow: 0 0 1px #00cc44; } } -/* ─── Feed Items ───────────────────────────────── */ +/* ─── Gov Results Bar ──────────────────────────── */ +.gov-results-bar { + font-family: 'JetBrains Mono', monospace; + font-size: 0.7rem; + letter-spacing: 1.5px; + color: rgba(255, 255, 255, 0.2); + padding: 0.4rem 1rem; + margin-bottom: 0.5rem; + border-bottom: 1px solid rgba(255, 255, 255, 0.04); +} + +/* ─── Feed Items (News) ────────────────────────── */ .radar-feed { display: flex; flex-direction: column; @@ -276,7 +384,162 @@ letter-spacing: 2px; } +/* ═══════════════════════════════════════════════════ */ +/* .GOV DOMAIN TRACKER STYLES */ +/* ═══════════════════════════════════════════════════ */ + +/* ─── Domain Item ──────────────────────────────── */ +.gov-domain-item { + display: grid; + grid-template-columns: 1fr auto auto auto; + gap: 1rem; + align-items: center; + padding: 0.7rem 1rem; + border-bottom: 1px solid rgba(255, 255, 255, 0.03); + transition: background 0.15s, border-color 0.3s; + border-left: 2px solid transparent; +} + +.gov-domain-item:hover { + background: rgba(255, 255, 255, 0.02); +} + +/* ─── New Domain Highlight ─────────────────────── */ +.gov-domain-item.gov-new { + border-left: 2px solid rgba(0, 204, 68, 0.6); + background: rgba(0, 204, 68, 0.02); + animation: gov-new-pulse 3s ease infinite; +} + +@keyframes gov-new-pulse { + 0%, 100% { background: rgba(0, 204, 68, 0.02); } + 50% { background: rgba(0, 204, 68, 0.05); } +} + +.gov-domain-item.gov-new:hover { + background: rgba(0, 204, 68, 0.06); +} + +/* ─── Domain Name ──────────────────────────────── */ +.gov-domain-main { + display: flex; + align-items: center; + gap: 0.5rem; + min-width: 0; +} + +.gov-domain-name { + font-family: 'JetBrains Mono', monospace; + font-size: 0.8rem; + font-weight: 500; + color: rgba(255, 255, 255, 0.8); + text-decoration: none; + letter-spacing: 0.5px; + transition: color 0.15s; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +} + +.gov-domain-name:hover { + color: rgba(255, 170, 0, 0.95); +} + +/* ─── NEW Badge ────────────────────────────────── */ +.gov-new-badge { + font-family: 'Orbitron', monospace; + font-size: 0.55rem; + font-weight: 900; + letter-spacing: 2px; + padding: 0.1rem 0.4rem; + background: rgba(0, 204, 68, 0.15); + border: 1px solid rgba(0, 204, 68, 0.4); + color: #00cc44; + animation: badge-glow 2s ease infinite; + flex-shrink: 0; +} + +@keyframes badge-glow { + 0%, 100% { box-shadow: 0 0 4px rgba(0, 204, 68, 0.2); } + 50% { box-shadow: 0 0 10px rgba(0, 204, 68, 0.4); } +} + +/* ─── Type Badge ───────────────────────────────── */ +.gov-domain-type { + font-family: 'JetBrains Mono', monospace; + font-size: 0.6rem; + letter-spacing: 1.5px; + padding: 0.15rem 0.6rem; + border: 1px solid; + text-align: center; + white-space: nowrap; + flex-shrink: 0; + min-width: 80px; + text-transform: uppercase; +} + +/* ─── Domain Details ───────────────────────────── */ +.gov-domain-details { + display: flex; + gap: 1.2rem; + flex-wrap: wrap; + min-width: 0; +} + +.gov-detail { + font-family: 'JetBrains Mono', monospace; + font-size: 0.65rem; + color: rgba(255, 255, 255, 0.35); + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + max-width: 250px; +} + +.gov-detail-label { + color: rgba(255, 255, 255, 0.15); + font-size: 0.6rem; + letter-spacing: 1px; +} + +/* ─── First Seen Date ──────────────────────────── */ +.gov-domain-date { + font-family: 'JetBrains Mono', monospace; + font-size: 0.65rem; + color: rgba(255, 255, 255, 0.2); + letter-spacing: 0.5px; + white-space: nowrap; + flex-shrink: 0; + min-width: 80px; + text-align: right; +} + +/* ─── Truncation Notice ────────────────────────── */ +.gov-truncated { + font-family: 'JetBrains Mono', monospace; + font-size: 0.7rem; + color: rgba(255, 170, 0, 0.4); + text-align: center; + padding: 1.5rem; + letter-spacing: 1.5px; + border-top: 1px solid rgba(255, 170, 0, 0.1); +} + /* ─── Responsive ───────────────────────────────── */ +@media (max-width: 1024px) { + .gov-domain-item { + grid-template-columns: 1fr auto; + grid-template-rows: auto auto; + gap: 0.4rem 1rem; + } + .gov-domain-details { + grid-column: 1 / -1; + } + .gov-domain-date { + text-align: left; + } +} + @media (max-width: 768px) { .radar-controls { flex-direction: column; @@ -300,17 +563,46 @@ .radar-stats { gap: 0.8rem; } + .radar-view-toggle { + width: 100%; + } + .radar-view-btn { + flex: 1; + justify-content: center; + padding: 0.6rem 1rem; + font-size: 0.65rem; + } + .gov-domain-item { + grid-template-columns: 1fr; + gap: 0.3rem; + padding: 0.6rem 0.8rem; + } + .gov-domain-type { + justify-self: start; + } + .gov-domain-date { + text-align: left; + } + .gov-type-filter { + min-width: 0; + flex: 1; + } } @media (max-width: 480px) { .radar-container { padding: 0 1rem 2rem; } - .radar-filters { + .radar-filters, + .gov-range-filters { gap: 0.3rem; } - .radar-filter { + .radar-filter, + .gov-range { font-size: 0.65rem; padding: 0.3rem 0.5rem; } + .gov-detail { + max-width: 180px; + } } diff --git a/js/radar.js b/js/radar.js index 993682c..d91836b 100644 --- a/js/radar.js +++ b/js/radar.js @@ -1,14 +1,17 @@ -/* ─── RADAR: Live Intelligence Feed ─────────────── */ +/* ─── RADAR: Live Intelligence Feed + .GOV Domain Tracker ─── */ (function() { 'use strict'; - const API = '/api/radar'; - const REFRESH_INTERVAL = 15 * 60 * 1000; // 15 minutes - let currentSource = 'all'; - let allItems = []; - let refreshTimer = null; + // ═══════════════════════════════════════════════════ + // SHARED UTILITIES + // ═══════════════════════════════════════════════════ + + function esc(s) { + const d = document.createElement('div'); + d.textContent = s || ''; + return d.innerHTML; + } - // ─── Time Ago ────────────────────────────────── function timeAgo(dateStr) { if (!dateStr) return ''; const now = new Date(); @@ -21,146 +24,368 @@ return Math.floor(diff / 604800) + 'w ago'; } - // ─── Extract Domain ──────────────────────────── function extractDomain(url) { - try { - const u = new URL(url); - return u.hostname.replace('www.', ''); - } catch(e) { - return ''; - } + try { return new URL(url).hostname.replace('www.', ''); } + catch(e) { return ''; } } - // ─── Escape HTML ─────────────────────────────── - function esc(s) { - const d = document.createElement('div'); - d.textContent = s; - return d.innerHTML; - } + // ═══════════════════════════════════════════════════ + // NEWS FEED MODULE + // ═══════════════════════════════════════════════════ - // ─── Render Feed ─────────────────────────────── - function renderFeed(items) { - const feed = document.getElementById('radarFeed'); - if (!items || items.length === 0) { - feed.innerHTML = '
> Live intelligence feed — intercepted from Hacker News, Reddit, and Lobste.rs
+> Live intelligence feed — intercepted from Hacker News, Reddit, and Lobste.rs