#!/usr/bin/env python3
"""
Proactive Research Monitor

Checks topics due for monitoring, scores findings, and sends alerts.
Run via cron for automated monitoring.
"""

import os
import sys
import json
import hashlib
import argparse
import re
from datetime import datetime, timedelta, timezone
from email.utils import parsedate_to_datetime
from html.parser import HTMLParser
from pathlib import Path
from typing import Dict, List, Optional
from urllib.parse import urljoin
from urllib.request import Request, urlopen

SCRIPT_DIR = Path(__file__).parent
VENDOR_DIR = SCRIPT_DIR / "_vendor"
sys.path.insert(0, str(SCRIPT_DIR))
if VENDOR_DIR.exists():
    sys.path.insert(0, str(VENDOR_DIR))

from config import (
    load_config, load_state, save_state, get_settings,
    save_finding, queue_alert
)
from importance_scorer import score_result

try:
    import feedparser
except ImportError:  # pragma: no cover - handled at runtime
    feedparser = None


class FeedDiscoveryParser(HTMLParser):
    """Extract RSS/Atom alternate links from HTML."""

    def __init__(self):
        super().__init__()
        self.feed_links = []

    def handle_starttag(self, tag, attrs):
        if tag.lower() != "link":
            return
        attr_map = {k.lower(): v for k, v in attrs}
        rel = (attr_map.get("rel") or "").lower()
        href = attr_map.get("href")
        content_type = (attr_map.get("type") or "").lower()
        if not href:
            return
        if "alternate" in rel and content_type in (
            "application/rss+xml",
            "application/atom+xml",
            "application/xml",
            "text/xml",
        ):
            self.feed_links.append(href)


def hash_url(url: str) -> str:
    return hashlib.md5(url.encode()).hexdigest()


def normalize_text_list(values: Optional[List[str]]) -> List[str]:
    if not values:
        return []
    out = []
    for value in values:
        if value is None:
            continue
        text = str(value).strip()
        if text:
            out.append(text)
    return out


def normalize_feed_url(url: str) -> str:
    url = (url or "").strip()
    if not url:
        return ""
    if "github.com" in url and "/releases" not in url and url.count("/") >= 4:
        parts = url.rstrip("/").split("/")
        if len(parts) >= 5:
            owner = parts[-2]
            repo = parts[-1]
            return f"https://github.com/{owner}/{repo}/releases.atom"
    return url


def github_release_feed_url(repo: str) -> str:
    repo = repo.strip().strip("/")
    return f"https://github.com/{repo}/releases.atom"


def parse_http_date(value: Optional[str]) -> Optional[str]:
    if not value:
        return None
    try:
        return parsedate_to_datetime(value).astimezone(timezone.utc).isoformat()
    except Exception:
        return None


def discover_feed_urls(url: str, timeout: int = 15) -> List[str]:
    """Try to discover RSS/Atom feeds from a regular webpage URL."""
    url = (url or "").strip()
    if not url:
        return []

    if url.endswith((".rss", ".xml", ".atom")) or url.endswith("/feed"):
        return [url]

    request = Request(
        url,
        headers={
            "User-Agent": "topic-monitor/1.5 (+feed-discovery)",
            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
        },
    )
    try:
        with urlopen(request, timeout=timeout) as response:
            content_type = response.headers.get("Content-Type", "")
            final_url = response.geturl()
            body = response.read(250_000)
    except Exception:
        return []

    lowered_type = content_type.lower()
    if "rss" in lowered_type or "atom" in lowered_type or "xml" in lowered_type:
        return [final_url]

    try:
        html = body.decode("utf-8", errors="ignore")
    except Exception:
        return []

    parser = FeedDiscoveryParser()
    parser.feed(html)
    discovered = [urljoin(final_url, href) for href in parser.feed_links]

    common_guesses = [
        urljoin(final_url, "/feed"),
        urljoin(final_url, "/rss"),
        urljoin(final_url, "/rss.xml"),
        urljoin(final_url, "/atom.xml"),
        urljoin(final_url, "/feeds/posts/default"),
    ]

    seen = set()
    ordered = []
    for candidate in discovered + common_guesses:
        candidate = candidate.strip()
        if candidate and candidate not in seen:
            seen.add(candidate)
            ordered.append(candidate)
    return ordered


def is_duplicate(url: str, state: Dict, dedup_hours: int = 72) -> bool:
    url_hash = hash_url(url)
    dedup_map = state.get("deduplication", {}).get("url_hash_map", {})
    if url_hash not in dedup_map:
        return False
    last_seen = datetime.fromisoformat(dedup_map[url_hash])
    return (datetime.now() - last_seen) < timedelta(hours=dedup_hours)


def mark_as_seen(url: str, state: Dict):
    if "deduplication" not in state:
        state["deduplication"] = {"url_hash_map": {}}
    state["deduplication"]["url_hash_map"][hash_url(url)] = datetime.now().isoformat()


def get_topic_state(state: Dict, topic_id: str) -> Dict:
    if "topics" not in state:
        state["topics"] = {}
    if topic_id not in state["topics"]:
        state["topics"][topic_id] = {}
    return state["topics"][topic_id]


def get_feed_state(state: Dict, topic_id: str) -> Dict:
    topic_state = get_topic_state(state, topic_id)
    if "feeds" not in topic_state:
        topic_state["feeds"] = {}
    return topic_state["feeds"]


def feed_cache_headers(state: Dict, topic_id: str, feed_url: str) -> Dict[str, str]:
    feed_state = get_feed_state(state, topic_id).get(feed_url, {})
    headers = {"User-Agent": "topic-monitor/1.5 (+feedparser)"}
    if feed_state.get("etag"):
        headers["If-None-Match"] = feed_state["etag"]
    if feed_state.get("modified"):
        headers["If-Modified-Since"] = feed_state["modified"]
    return headers


def update_feed_cache(state: Dict, topic_id: str, feed_url: str, parsed_feed):
    cache = get_feed_state(state, topic_id).setdefault(feed_url, {})
    if getattr(parsed_feed, "etag", None):
        cache["etag"] = parsed_feed.etag
    response_headers = getattr(parsed_feed, "headers", {}) or {}
    if response_headers.get("etag"):
        cache["etag"] = response_headers.get("etag")
    if response_headers.get("last-modified"):
        cache["modified"] = response_headers.get("last-modified")
    cache["last_checked"] = datetime.now().isoformat()
    if getattr(parsed_feed, "status", None) is not None:
        cache["last_status"] = parsed_feed.status
    href = getattr(parsed_feed.feed, "link", None) if getattr(parsed_feed, "feed", None) else None
    if href:
        cache["site_url"] = href
    title = getattr(parsed_feed.feed, "title", None) if getattr(parsed_feed, "feed", None) else None
    if title:
        cache["feed_title"] = title


def iso_from_struct_time(struct_time_value) -> Optional[str]:
    if not struct_time_value:
        return None
    try:
        return datetime(*struct_time_value[:6], tzinfo=timezone.utc).isoformat()
    except Exception:
        return None


def entry_to_result(entry: Dict, feed_url: str, topic: Dict, feed_title: str = "") -> Dict:
    title = entry.get("title", "Untitled")
    summary = entry.get("summary", "") or entry.get("description", "")
    link = entry.get("link", feed_url)
    published = (
        iso_from_struct_time(entry.get("published_parsed"))
        or iso_from_struct_time(entry.get("updated_parsed"))
        or entry.get("published")
        or entry.get("updated")
        or ""
    )
    tags = ", ".join(tag.get("term", "") for tag in entry.get("tags", []) if tag.get("term"))
    source_label = feed_title or topic.get("name", "Feed")

    result = {
        "title": title,
        "url": link,
        "snippet": re.sub(r"\s+", " ", summary).strip(),
        "published_date": published,
        "source": "feed",
        "feed_url": feed_url,
        "feed_title": feed_title,
        "source_label": source_label,
        "tags": tags,
    }

    if "github.com" in feed_url and "/releases.atom" in feed_url:
        repo = feed_url.split("github.com/")[-1].split("/releases.atom")[0]
        result["source"] = "github_release"
        result["github_repo"] = repo
        result["title"] = f"{repo} release: {title}"
        if not result["snippet"]:
            result["snippet"] = f"New GitHub release published for {repo}."
    return result


def search_topic(topic: Dict, dry_run: bool = False, verbose: bool = False) -> List[Dict]:
    query = topic.get("query", "")
    if not query:
        return []

    web_search_plus = Path(os.environ.get(
        "WEB_SEARCH_PLUS_PATH",
        os.path.join(
            os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))),
            "web-search-plus",
            "scripts",
            "search.py",
        ),
    ))

    if web_search_plus.exists():
        import subprocess
        try:
            safe_query = re.sub(r'[\x00-\x1f\x7f]', '', query)[:500]
            if verbose:
                print(f"   🔍 Searching via web-search-plus: {safe_query}")
            result = subprocess.run(
                ["python3", str(web_search_plus), "--query", safe_query, "--max-results", "5"],
                capture_output=True,
                text=True,
                timeout=45,
                env={k: v for k, v in os.environ.items() if k in (
                    "PATH", "HOME", "LANG", "TERM",
                    "SERPER_API_KEY", "TAVILY_API_KEY", "EXA_API_KEY",
                    "YOU_API_KEY", "SEARXNG_INSTANCE_URL", "WSP_CACHE_DIR",
                )},
            )
            if result.returncode == 0:
                stdout = result.stdout.strip()
                json_start = stdout.find('{')
                if json_start >= 0:
                    data = json.loads(stdout[json_start:])
                    results = data.get("results", [])
                    if verbose:
                        print(f"   ✅ Got {len(results)} search results from {data.get('provider', 'unknown')}")
                    for item in results:
                        item.setdefault("source", "web_search")
                    return results
            elif verbose and result.stderr:
                print(f"   ⚠️ web-search-plus error: {result.stderr[:200]}", file=sys.stderr)
        except subprocess.TimeoutExpired:
            print(f"⚠️ web-search-plus timed out for query: {query}", file=sys.stderr)
        except Exception as e:
            print(f"⚠️ web-search-plus failed: {e}", file=sys.stderr)
    elif verbose:
        print(f"   ⚠️ web-search-plus not found at {web_search_plus}", file=sys.stderr)

    if dry_run:
        return [{
            "title": f"[Mock] Result for: {query}",
            "url": f"https://example.com/mock-{hashlib.md5(query.encode()).hexdigest()[:8]}",
            "snippet": f"This is a mock/test result for query: {query}. Run without --dry-run to use real search.",
            "published_date": datetime.now().isoformat(),
            "source": "web_search",
        }]
    return []


def fetch_feed_results(topic: Dict, state: Dict, dry_run: bool = False, verbose: bool = False) -> List[Dict]:
    if feedparser is None:
        if verbose:
            print("   ⚠️ feedparser is not installed; skipping feeds", file=sys.stderr)
        return []

    topic_id = topic.get("id")
    explicit_feeds = [normalize_feed_url(url) for url in normalize_text_list(topic.get("feeds", []))]
    github_feeds = [github_release_feed_url(repo) for repo in normalize_text_list(topic.get("github_repos", []))]
    all_feeds = []
    for feed in explicit_feeds + github_feeds:
        if feed and feed not in all_feeds:
            all_feeds.append(feed)

    if not all_feeds:
        return []

    results = []
    for feed_url in all_feeds:
        headers = feed_cache_headers(state, topic_id, feed_url)
        if verbose:
            print(f"   📰 Fetching feed: {feed_url}")
        parsed = feedparser.parse(feed_url, request_headers=headers)
        update_feed_cache(state, topic_id, feed_url, parsed)

        status = getattr(parsed, "status", None)
        if status == 304:
            if verbose:
                print("   ⏭️  Feed not modified (304)")
            continue

        if getattr(parsed, "bozo", False) and not getattr(parsed, "entries", []):
            if verbose:
                print(f"   ⚠️ feed parse issue for {feed_url}: {getattr(parsed, 'bozo_exception', 'unknown')}", file=sys.stderr)
            continue

        feed_title = parsed.feed.get("title", "") if getattr(parsed, "feed", None) else ""
        entries = getattr(parsed, "entries", [])[:10]
        if verbose:
            print(f"   ✅ Got {len(entries)} feed entries")
        for entry in entries:
            results.append(entry_to_result(entry, feed_url, topic, feed_title=feed_title))
    return results


def collect_results(topic: Dict, state: Dict, dry_run: bool = False, verbose: bool = False) -> List[Dict]:
    results = []
    results.extend(search_topic(topic, dry_run=dry_run, verbose=verbose))
    results.extend(fetch_feed_results(topic, state, dry_run=dry_run, verbose=verbose))
    return results


def passes_topic_filters(result: Dict, topic: Dict) -> (bool, str):
    title = result.get("title", "")
    snippet = result.get("snippet", "")
    content = f"{title}\n{snippet}".lower()

    exclude_keywords = [k.lower() for k in normalize_text_list(topic.get("exclude_keywords", []))]
    for keyword in exclude_keywords:
        if keyword in content:
            return False, f"excluded_by_{keyword}"

    required_keywords = [k.lower() for k in normalize_text_list(topic.get("required_keywords", []))]
    missing = [keyword for keyword in required_keywords if keyword not in content]
    if missing:
        return False, f"missing_required_{','.join(missing)}"

    return True, ""


def should_check_topic(topic: Dict, state: Dict, force: bool = False) -> bool:
    if force:
        return True
    topic_state = state.get("topics", {}).get(topic.get("id"), {})
    last_check_str = topic_state.get("last_check")
    if not last_check_str:
        return True
    last_check = datetime.fromisoformat(last_check_str)
    now = datetime.now()
    frequency = topic.get("frequency", "daily")
    if frequency == "hourly":
        return (now - last_check) >= timedelta(hours=1)
    if frequency == "daily":
        return (now - last_check) >= timedelta(days=1)
    if frequency == "weekly":
        return (now - last_check) >= timedelta(weeks=1)
    return False


def check_rate_limits(topic: Dict, state: Dict, settings: Dict) -> bool:
    topic_id = topic.get("id")
    max_per_day = settings.get("max_alerts_per_day", 5)
    max_per_topic_per_day = settings.get("max_alerts_per_topic_per_day", 2)
    topic_state = state.get("topics", {}).get(topic_id, {})
    alerts_today = topic_state.get("alerts_today", 0)
    if alerts_today >= max_per_topic_per_day:
        return False
    total_alerts_today = sum(s.get("alerts_today", 0) for s in state.get("topics", {}).values())
    return total_alerts_today < max_per_day


def sentiment_shifted(topic: Dict, state: Dict, new_sentiment: str) -> bool:
    if not topic.get("alert_on_sentiment_shift"):
        return False
    history = get_topic_state(state, topic.get("id")).get("sentiment_history", [])
    if not history:
        return False
    previous = history[-1].get("sentiment")
    return bool(previous and previous != new_sentiment)


def record_sentiment(topic: Dict, state: Dict, result: Dict, sentiment: str):
    topic_state = get_topic_state(state, topic.get("id"))
    history = topic_state.setdefault("sentiment_history", [])
    history.append({
        "timestamp": datetime.now().isoformat(),
        "sentiment": sentiment,
        "url": result.get("url", ""),
        "title": result.get("title", ""),
    })
    if len(history) > 50:
        del history[:-50]
    topic_state["last_sentiment"] = sentiment


def build_alert_message(topic: Dict, result: Dict, priority: str, score: float, reason: str, sentiment: str, sentiment_shift: bool) -> str:
    emoji_map = {"high": "🔥", "medium": "📌", "low": "📝"}
    source = result.get("source", "web_search")
    source_label = {
        "web_search": "🌐 Web",
        "feed": "📰 Feed",
        "github_release": "🚀 GitHub Release",
    }.get(source, "🌐 Web")
    lines = [f"{emoji_map.get(priority, '📌')} **{topic.get('name', 'Research Alert')}** {topic.get('emoji', '🔍')}", ""]
    lines.append(f"**{result.get('title', 'Untitled')}**")
    lines.append("")
    snippet = result.get("snippet", "")
    if snippet:
        if len(snippet) > 320:
            snippet = snippet[:317] + "..."
        lines.append(snippet)
        lines.append("")
    context = topic.get("context", "")
    if context:
        lines.append(f"💡 _Context: {context}_")
        lines.append("")
    lines.append(f"{source_label}: {result.get('feed_title') or result.get('source_label') or result.get('url', '')}")
    if result.get("url"):
        lines.append(f"🔗 {result['url']}")
    lines.append(f"📊 _Score: {score:.2f} | {reason}_")
    lines.append(f"🙂 _Sentiment: {sentiment}_")
    if sentiment_shift:
        lines.append("🔄 _Sentiment shift detected_")
    return "\n".join(lines)


def send_alert(topic: Dict, result: Dict, priority: str, score: float, reason: str, sentiment: str, sentiment_shift: bool = False, dry_run: bool = False):
    channels = topic.get("channels", [])
    message = build_alert_message(topic, result, priority, score, reason, sentiment, sentiment_shift)

    if dry_run:
        print(f"\n{'='*60}")
        print("DRY RUN - Would send alert:")
        print(f"Channels: {', '.join(channels)}")
        print(f"Priority: {priority.upper()}")
        print()
        print(message)
        print(f"{'='*60}\n")
        return None

    alert_ids = []
    for channel in channels:
        alert_data = {
            "timestamp": datetime.now().isoformat(),
            "priority": priority,
            "channel": channel,
            "topic_id": topic.get("id"),
            "topic_name": topic.get("name"),
            "title": result.get("title", ""),
            "snippet": result.get("snippet", ""),
            "url": result.get("url", ""),
            "score": score,
            "reason": reason,
            "message": message,
            "context": topic.get("context", ""),
            "sentiment": sentiment,
            "sentiment_shift": sentiment_shift,
            "source": result.get("source", "web_search"),
            "feed_url": result.get("feed_url", ""),
            "github_repo": result.get("github_repo", ""),
        }
        alert_id = queue_alert(alert_data)
        alert_ids.append(alert_id)
        print(f"📢 ALERT_QUEUED: {json.dumps({'id': alert_id, 'channel': channel, 'priority': priority, 'topic': topic.get('name'), 'sentiment': sentiment})}")
    return alert_ids


def monitor_topic(topic: Dict, state: Dict, settings: Dict, dry_run: bool = False, verbose: bool = False):
    topic_id = topic.get("id")
    topic_name = topic.get("name")
    if verbose:
        print(f"\n🔍 Checking topic: {topic_name} ({topic_id})")

    results = collect_results(topic, state, dry_run=dry_run, verbose=verbose)
    if verbose:
        print(f"   Found {len(results)} total results across all sources")

    dedup_hours = settings.get("deduplication_window_hours", 72)
    high_priority = []
    medium_priority = []

    for result in results:
        url = result.get("url", "") or result.get("feed_url", "")
        if not url:
            continue

        if is_duplicate(url, state, dedup_hours):
            if verbose:
                print(f"   ⏭️  Skipping duplicate: {url}")
            continue

        passes, filter_reason = passes_topic_filters(result, topic)
        if not passes:
            if verbose:
                print(f"   🚫 Filtered out: {filter_reason} - {result.get('title', '')[:60]}")
            mark_as_seen(url, state)
            continue

        priority, score, reason, sentiment = score_result(result, topic, settings)
        sentiment_shift = sentiment_shifted(topic, state, sentiment)

        if sentiment_shift and priority == "medium":
            priority = "high"
            reason = f"{reason} + sentiment_shift"
        elif sentiment_shift and priority == "low":
            priority = "medium"
            reason = f"{reason} + sentiment_shift"

        if verbose:
            print(f"   {priority.upper():6} ({score:.2f}) [{sentiment}] - {result.get('title', '')[:55]}...")

        if priority == "high":
            high_priority.append((result, score, reason, sentiment, sentiment_shift))
        elif priority == "medium":
            medium_priority.append((result, score, reason, sentiment, sentiment_shift))

        mark_as_seen(url, state)
        if not dry_run:
            record_sentiment(topic, state, result, sentiment)

    for result, score, reason, sentiment, sentiment_shift in high_priority:
        if check_rate_limits(topic, state, settings):
            send_alert(topic, result, "high", score, reason, sentiment, sentiment_shift, dry_run=dry_run)
            if not dry_run:
                topic_state = get_topic_state(state, topic_id)
                topic_state["alerts_today"] = topic_state.get("alerts_today", 0) + 1
        elif verbose:
            print("   ⚠️ Rate limit reached, skipping alert")

    date_str = datetime.now().strftime("%Y-%m-%d")
    for result, score, reason, sentiment, sentiment_shift in medium_priority:
        if not dry_run:
            save_finding(topic_id, date_str, {
                "result": result,
                "score": score,
                "reason": reason,
                "timestamp": datetime.now().isoformat(),
                "sentiment": sentiment,
                "sentiment_shift": sentiment_shift,
            })
        if verbose:
            print(f"   💾 Saved to digest: {result.get('title', '')[:50]}...")

    if not dry_run:
        topic_state = get_topic_state(state, topic_id)
        topic_state["last_check"] = datetime.now().isoformat()
        topic_state["last_results_count"] = len(results)
        topic_state["findings_count"] = topic_state.get("findings_count", 0) + len(medium_priority)


def main():
    parser = argparse.ArgumentParser(description="Monitor research topics")
    parser.add_argument("--dry-run", action="store_true", help="Don't send alerts or save state")
    parser.add_argument("--verbose", "-v", action="store_true", help="Verbose output")
    parser.add_argument("--topic", help="Check specific topic by ID")
    parser.add_argument("--force", action="store_true", help="Force check even if not due")
    parser.add_argument("--frequency", choices=["hourly", "daily", "weekly"], help="Only check topics with this frequency")
    parser.add_argument("--discover-feed", metavar="URL", help="Discover RSS/Atom feed links for a URL and exit")
    args = parser.parse_args()

    if args.discover_feed:
        for item in discover_feed_urls(args.discover_feed):
            print(item)
        return

    try:
        config = load_config()
    except FileNotFoundError as e:
        print(f"❌ {e}", file=sys.stderr)
        sys.exit(1)

    state = load_state()
    settings = get_settings()
    topics = config.get("topics", [])
    if not topics:
        print("⚠️ No topics configured", file=sys.stderr)
        sys.exit(0)

    topics_to_check = []
    for topic in topics:
        if args.topic and topic.get("id") != args.topic:
            continue
        if args.frequency and topic.get("frequency") != args.frequency:
            continue
        if should_check_topic(topic, state, force=args.force):
            topics_to_check.append(topic)

    if not topics_to_check:
        if args.verbose:
            print("✅ No topics due for checking")
        sys.exit(0)

    print(f"🔍 Monitoring {len(topics_to_check)} topic(s)...")
    for topic in topics_to_check:
        try:
            monitor_topic(topic, state, settings, dry_run=args.dry_run, verbose=args.verbose)
        except Exception as e:
            print(f"❌ Error monitoring {topic.get('name')}: {e}", file=sys.stderr)
            if args.verbose:
                import traceback
                traceback.print_exc()

    if not args.dry_run:
        save_state(state)
        print("✅ State saved")
    print("✅ Monitoring complete")


if __name__ == "__main__":
    main()