aggregator.py

"""Result aggregator - combines and formats scanner results"""
import asyncio
from typing import Any, Dict

from loguru import logger
from tabulate import tabulate

import config
from performance import get_cached_result, set_cached_result, MemoryOptimizer
from scanners import (
    AdvertisingChecker,
    APIChecker,
    AuthChecker,
    AvailabilityChecker,
    BotProtectionChecker,
    CORSChecker,
    DatabaseChecker,
    DNSChecker,
    GeolocationChecker,
    HostingChecker,
    MCPChecker,
    MobileChecker,
    PaymentChecker,
    PaywallChecker,
    PerformanceChecker,
    PrivacyChecker,
    ProtocolChecker,
    RedirectChecker,
    SecurityChecker,
    SEOChecker,
    TechStackChecker,
)


class ResultAggregator:
    """Aggregate and format results from all scanners"""
    
    def __init__(self):
        self.availability_checker = AvailabilityChecker()
        self.dns_checker = DNSChecker()
        self.protocol_checker = ProtocolChecker()
        self.paywall_checker = PaywallChecker()
        self.security_checker = SecurityChecker()
        self.hosting_checker = HostingChecker()
        self.api_checker = APIChecker()
        self.advertising_checker = AdvertisingChecker()
        self.bot_protection_checker = BotProtectionChecker()
        self.mcp_checker = MCPChecker()
        self.performance_checker = PerformanceChecker()
        self.seo_checker = SEOChecker()
        self.tech_stack_checker = TechStackChecker()
        self.privacy_checker = PrivacyChecker()
        self.auth_checker = AuthChecker()
        self.database_checker = DatabaseChecker()
        self.cors_checker = CORSChecker()
        self.mobile_checker = MobileChecker()
        self.payment_checker = PaymentChecker()
        self.redirect_checker = RedirectChecker()
        self.geolocation_checker = GeolocationChecker()
    
    async def scan_url(self, url: str) -> Dict[str, Any]:
        """
        Run all scanners in parallel and aggregate results
        Uses caching to improve performance on Render
        
        Returns:
            dict: Complete scan results from all modules
        """
        logger.info(f"Starting parallel scan for: {url}")
        
        # Check cache if enabled
        if config.ENABLE_CACHING:
            cached = get_cached_result(url, 'full_scan')
            if cached:
                logger.info(f"Using cached results for: {url}")
                return cached
        
        # Run all checks in parallel with timeout
        try:
            results = await asyncio.wait_for(
                asyncio.gather(
                    self.availability_checker.check(url),
                    self.dns_checker.check(url),
                    self.protocol_checker.check(url),
                    self.paywall_checker.check(url),
                    self.security_checker.check(url),
                    self.hosting_checker.check(url),
                    self.api_checker.check(url),
                    self.advertising_checker.check(url),
                    self.bot_protection_checker.check(url),
                    self.mcp_checker.check(url),
                    self.performance_checker.check(url),
                    self.seo_checker.check(url),
                    self.tech_stack_checker.check(url),
                    self.privacy_checker.check(url),
                    self.auth_checker.check(url),
                    self.database_checker.check(url),
                    self.cors_checker.check(url),
                    self.mobile_checker.check(url),
                    self.payment_checker.check(url),
                    self.redirect_checker.check(url),
                    self.geolocation_checker.check(url),
                    return_exceptions=True
                ),
                timeout=45  # Global timeout for all scanners
            )
            
            # Unpack results
            (availability, dns, protocol, paywall, security, hosting, api, 
             advertising, bot_protection, mcp, performance, seo, tech_stack, 
             privacy, auth, database, cors, mobile, payment, redirect, geolocation) = results
            
            # Handle any exceptions
            for i, result in enumerate(results):
                if isinstance(result, Exception):
                    logger.error(f"Scanner {i} failed: {result}")
                    results[i] = {'status': 'ERROR', 'error': str(result)}
            
            # Combine results
            aggregated = {
                'url': url,
                'timestamp': self._get_timestamp(),
                'availability': availability if not isinstance(availability, Exception) else {'status': 'ERROR'},
                'dns': dns if not isinstance(dns, Exception) else {'status': 'ERROR'},
                'protocol': protocol if not isinstance(protocol, Exception) else {'status': 'ERROR'},
                'paywall': paywall if not isinstance(paywall, Exception) else {'status': 'ERROR'},
                'security': security if not isinstance(security, Exception) else {'status': 'ERROR'},
                'hosting': hosting if not isinstance(hosting, Exception) else {'provider': 'ERROR'},
                'api': api if not isinstance(api, Exception) else {'has_api': False},
                'advertising': advertising if not isinstance(advertising, Exception) else {'has_ads': False},
                'bot_protection': bot_protection if not isinstance(bot_protection, Exception) else {'has_protection': False},
                'mcp': mcp if not isinstance(mcp, Exception) else {'has_mcp': False},
                'performance': performance if not isinstance(performance, Exception) else {'load_time': 0},
                'seo': seo if not isinstance(seo, Exception) else {'score': 0},
                'tech_stack': tech_stack if not isinstance(tech_stack, Exception) else {'detected': []},
                'privacy': privacy if not isinstance(privacy, Exception) else {'score': 0},
                'auth': auth if not isinstance(auth, Exception) else {'providers': []},
                'database': database if not isinstance(database, Exception) else {'detected': []},
                'cors': cors if not isinstance(cors, Exception) else {'enabled': False},
                'mobile': mobile if not isinstance(mobile, Exception) else {'score': 0},
                'payment': payment if not isinstance(payment, Exception) else {'providers': []},
                'redirect': redirect if not isinstance(redirect, Exception) else {'has_redirects': False},
                'geolocation': geolocation if not isinstance(geolocation, Exception) else {'ip_addresses': {}},
            }
            
            # Add summary
            aggregated['summary'] = self._create_summary(aggregated)
            
            # Optimize memory usage
            aggregated = MemoryOptimizer.compress_result(aggregated)
            
            # Cache result if enabled
            if config.ENABLE_CACHING:
                set_cached_result(url, 'full_scan', aggregated)
            
            logger.info(f"Scan completed for: {url}")
            return aggregated
            
        except asyncio.TimeoutError:
            logger.error(f"Scan timeout for: {url}")
            return {
                'url': url,
                'timestamp': self._get_timestamp(),
                'error': 'Scan timed out after 45 seconds',
                'status': 'TIMEOUT'
            }
        except Exception as e:
            logger.error(f"Error during scan: {e}")
            return {
                'url': url,
                'timestamp': self._get_timestamp(),
                'error': str(e),
                'status': 'FAILED'
            }
    
    def _create_summary(self, results: Dict[str, Any]) -> Dict[str, Any]:
        """Create a summary of the scan results"""
        availability = results.get('availability', {})
        dns = results.get('dns', {})
        protocol = results.get('protocol', {})
        paywall = results.get('paywall', {})
        security = results.get('security', {})
        
        # Determine overall accessibility
        is_accessible = availability.get('accessible', False)
        
        # Get primary IP
        primary_ip = dns.get('ipv4', [None])[0] if dns.get('ipv4') else None
        if not primary_ip:
            primary_ip = dns.get('ipv6', [None])[0] if dns.get('ipv6') else None
        
        # Determine protocol
        detected_protocol = protocol.get('protocol', 'unknown')
        
        # Paywall status
        has_paywall = paywall.get('detected', False)
        
        # Security status
        security_status = security.get('status', 'NO_INFO')
        
        return {
            'accessible': is_accessible,
            'status': availability.get('status', 'UNKNOWN'),
            'primary_ip': primary_ip,
            'protocol': detected_protocol,
            'has_paywall': has_paywall,
            'security_status': security_status,
            'response_time': availability.get('response_time'),
            'ssl_enabled': protocol.get('ssl_enabled', False)
        }
    
    def format_as_table(self, results: Dict[str, Any]) -> str:
        """Format results as a table"""
        if 'error' in results:
            return f"❌ Scan failed: {results['error']}"
        
        availability = results.get('availability', {})
        dns = results.get('dns', {})
        protocol = results.get('protocol', {})
        paywall = results.get('paywall', {})
        security = results.get('security', {})
        hosting = results.get('hosting', {})
        api = results.get('api', {})
        advertising = results.get('advertising', {})
        bot_protection = results.get('bot_protection', {})
        mcp = results.get('mcp', {})
        performance = results.get('performance', {})
        seo = results.get('seo', {})
        tech_stack = results.get('tech_stack', {})
        privacy = results.get('privacy', {})
        auth = results.get('auth', {})
        database = results.get('database', {})
        cors = results.get('cors', {})
        mobile = results.get('mobile', {})
        payment = results.get('payment', {})
        
        # Build table data
        table_data = [
            ['🎯 Target', results.get('url', 'N/A')],
            ['📊 Status', config.AVAILABILITY_STATUSES.get(
                availability.get('status', 'UNKNOWN'),
                availability.get('status', 'Unknown')
            )],
            ['⏱️ Response Time', 
             f"{availability.get('response_time', 0):.3f}s" if availability.get('response_time') is not None else 'N/A'],
            ['⚡ Page Load', 
             f"{performance.get('load_time') or 0:.3f}s ({performance.get('rating') or 'N/A'})"],
            ['🌐 Protocol', protocol.get('protocol', 'unknown').upper()],
            ['🔒 SSL/TLS', '✅ Enabled' if protocol.get('ssl_enabled') else '❌ Disabled'],
            ['📍 Primary IP', dns.get('ipv4', ['N/A'])[0] if dns.get('ipv4') else 'N/A'],
            ['🌍 IPv6', dns.get('ipv6', ['N/A'])[0] if dns.get('ipv6') else 'N/A'],
            ['🔍 DNS Status', dns.get('status', 'N/A')],
            ['☁️ Hosting', f"{hosting.get('provider', 'Unknown')} ({hosting.get('confidence', 'low')})"],
            ['🗄️ Database', ', '.join(database.get('detected', ['None']))],
            ['🔧 Tech Stack', ', '.join(tech_stack.get('detected', ['Unknown']))[:50]],
            ['🔌 API Available', '✅ Yes' if api.get('has_api') else '❌ No'],
            ['🔐 Auth Providers', ', '.join(auth.get('providers', ['None']))[:50]],
            ['🌐 CORS', f"{'✅' if cors.get('enabled') else '❌'} ({cors.get('security_level') or 'N/A'})"],
            ['📱 Mobile', f"{mobile.get('score') or 0}/100 ({mobile.get('responsive') or 'Unknown'})"],
            ['🔍 SEO Score', f"{seo.get('score') or 0}/100"],
            ['🔒 Privacy Score', f"{privacy.get('score') or 0}/100"],
            ['💳 Payment', ', '.join(payment.get('providers') or ['None'])[:50]],
            ['📺 Advertising', f"{'🔴 Yes' if advertising.get('has_ads') else '🟢 No'} ({advertising.get('ad_count') or 0} ads)"],
            ['🤖 Bot Protection', '✅ Active' if bot_protection.get('has_protection') else '❌ None'],
            ['🤖 MCP Support', '✅ Yes' if mcp.get('has_mcp') else '❌ No'],
            ['💰 Paywall', 
             f"{'🔴 Detected' if paywall.get('detected') else '🟢 None'} "
             f"({(paywall.get('confidence') or 0):.0%} confidence)"],
            ['🛡️ Security', config.SECURITY_STATUSES.get(
                security.get('status', 'NO_INFO'),
                security.get('status', 'Unknown')
            )],
            ['📈 Security Score', f"{security.get('overall_score') or 0}/100"],
        ]
        
        # Add HTTP status code if available
        if availability.get('status_code'):
            table_data.insert(2, ['📋 HTTP Status', availability.get('status_code')])
        
        # Add redirects info
        if availability.get('redirects', 0) > 0:
            table_data.append(['🔄 Redirects', availability.get('redirects', 0)])
        
        # Create table
        table = tabulate(table_data, tablefmt='simple', colalign=('left', 'left'))
        
        return f"```\n{table}\n```"
    
    def format_detailed_report(self, results: Dict[str, Any]) -> str:
        """Format detailed report with all information"""
        if 'error' in results:
            return f"❌ Scan failed: {results['error']}"
        
        report_sections = []
        
        # Header
        report_sections.append("📊 *Detailed Scan Report*")
        report_sections.append(f"🎯 Target: `{results.get('url')}`")
        report_sections.append(f"🕐 Time: {results.get('timestamp')}\n")
        
        # Availability
        availability = results.get('availability', {})
        report_sections.append("*🌐 Availability:*")
        report_sections.append(f"  Status: {availability.get('status')}")
        if availability.get('message'):
            report_sections.append(f"  Message: {availability.get('message')}")
        if availability.get('response_time') is not None:
            report_sections.append(f"  Response Time: {availability.get('response_time'):.3f}s")
        
        # DNS
        dns = results.get('dns', {})
        report_sections.append("\n*🔍 DNS Information:*")
        if dns.get('ipv4'):
            report_sections.append(f"  IPv4: {', '.join(dns['ipv4'])}")
        if dns.get('ipv6'):
            report_sections.append(f"  IPv6: {', '.join(dns['ipv6'][:2])}")  # Show first 2
        if dns.get('cname'):
            report_sections.append(f"  CNAME: {dns['cname']}")
        
        # Protocol
        protocol = results.get('protocol', {})
        report_sections.append("\n*🔒 Protocol & Security:*")
        report_sections.append(f"  Protocol: {protocol.get('protocol', 'unknown').upper()}")
        report_sections.append(f"  SSL/TLS: {'✅ Enabled' if protocol.get('ssl_enabled') else '❌ Disabled'}")
        if protocol.get('ssl_version'):
            report_sections.append(f"  SSL Version: {protocol.get('ssl_version')}")
        if protocol.get('http_version'):
            report_sections.append(f"  HTTP Version: {protocol.get('http_version')}")
        
        # Hosting
        hosting = results.get('hosting', {})
        report_sections.append("\n*☁️ Hosting Provider:*")
        report_sections.append(f"  Provider: {hosting.get('provider', 'Unknown')}")
        report_sections.append(f"  Confidence: {hosting.get('confidence', 'low')}")
        if hosting.get('server'):
            report_sections.append(f"  Server: {hosting.get('server')}")
        if hosting.get('cdn'):
            report_sections.append(f"  CDN: {hosting.get('cdn')}")
        if hosting.get('ip_address'):
            report_sections.append(f"  IP: {hosting.get('ip_address')}")
        
        # API
        api = results.get('api', {})
        report_sections.append("\n*🔌 API Availability:*")
        report_sections.append(f"  Has API: {'✅ Yes' if api.get('has_api') else '❌ No'}")
        if api.get('has_api'):
            report_sections.append(f"  Type: {api.get('api_type', 'Unknown')}")
            report_sections.append(f"  Confidence: {api.get('confidence', 'low')}")
            if api.get('api_docs_url'):
                report_sections.append(f"  Docs: {api.get('api_docs_url')}")
            if api.get('api_endpoints'):
                report_sections.append(f"  Endpoints: {len(api.get('api_endpoints'))} found")
        
        # Advertising
        advertising = results.get('advertising', {})
        report_sections.append("\n*📺 Advertising:*")
        report_sections.append(f"  Has Ads: {'🔴 Yes' if advertising.get('has_ads') else '🟢 No'}")
        if advertising.get('has_ads'):
            report_sections.append(f"  Ad Count: {advertising.get('ad_count', 0)}")
            report_sections.append(f"  Confidence: {advertising.get('confidence', 'low')}")
            if advertising.get('ad_networks'):
                networks = ', '.join(advertising['ad_networks'][:3])
                report_sections.append(f"  Networks: {networks}")
            if advertising.get('ad_types'):
                types = ', '.join(advertising['ad_types'])
                report_sections.append(f"  Types: {types}")
        
        # Bot Protection
        bot_protection = results.get('bot_protection', {})
        report_sections.append("\n*🤖 Bot Protection:*")
        report_sections.append(f"  Protected: {'✅ Yes' if bot_protection.get('has_protection') else '❌ No'}")
        if bot_protection.get('has_protection'):
            report_sections.append(f"  Confidence: {bot_protection.get('confidence', 'low')}")
            if bot_protection.get('captcha_detected'):
                report_sections.append("  CAPTCHA: ✅ Detected")
            if bot_protection.get('providers'):
                providers = ', '.join(bot_protection['providers'][:3])
                report_sections.append(f"  Providers: {providers}")
            if bot_protection.get('protection_type'):
                types = ', '.join(bot_protection['protection_type'])
                report_sections.append(f"  Types: {types}")
        
        # MCP
        mcp = results.get('mcp', {})
        report_sections.append("\n*🤖 MCP Support:*")
        report_sections.append(f"  Available: {'✅ Yes' if mcp.get('has_mcp') else '❌ No'}")
        if mcp.get('has_mcp'):
            report_sections.append(f"  Confidence: {mcp.get('confidence', 'low')}")
            if mcp.get('mcp_endpoint'):
                report_sections.append(f"  Endpoint: {mcp.get('mcp_endpoint')}")
            if mcp.get('mcp_version'):
                report_sections.append(f"  Version: {mcp.get('mcp_version')}")
            if mcp.get('capabilities'):
                caps = ', '.join(mcp['capabilities'])
                report_sections.append(f"  Capabilities: {caps}")
        
        # Performance
        performance = results.get('performance', {})
        report_sections.append("\n*⚡ Performance:*")
        report_sections.append(f"  Load Time: {performance.get('load_time') or 0:.3f}s")
        report_sections.append(f"  TTFB: {performance.get('ttfb') or 0:.3f}s")
        report_sections.append(f"  Rating: {performance.get('rating') or 'N/A'}")
        report_sections.append(f"  Response Size: {performance.get('response_size') or 0} bytes")
        
        # SEO
        seo = results.get('seo', {})
        report_sections.append("\n*🔍 SEO Analysis:*")
        report_sections.append(f"  Score: {seo.get('score') or 0}/100")
        if seo.get('title'):
            report_sections.append(f"  Title: {seo.get('title')[:60]}")
        if seo.get('description'):
            report_sections.append(f"  Description: {seo.get('description')[:80]}")
        report_sections.append(f"  Has Sitemap: {'✅' if seo.get('has_sitemap') else '❌'}")
        report_sections.append(f"  Has Robots.txt: {'✅' if seo.get('has_robots') else '❌'}")
        if seo.get('headings'):
            h_count = seo['headings']
            report_sections.append(f"  Headings: H1({h_count.get('h1') or 0}) H2({h_count.get('h2') or 0})")
        
        # Tech Stack
        tech_stack = results.get('tech_stack', {})
        report_sections.append("\n*🔧 Technology Stack:*")
        detected = tech_stack.get('detected', [])
        if detected:
            report_sections.append(f"  Technologies: {', '.join(detected[:5])}")
        else:
            report_sections.append("  Technologies: None detected")
        if tech_stack.get('cms'):
            report_sections.append(f"  CMS: {tech_stack['cms']}")
        if tech_stack.get('server'):
            report_sections.append(f"  Server: {tech_stack['server']}")
        if tech_stack.get('frameworks'):
            report_sections.append(f"  Frameworks: {', '.join(tech_stack['frameworks'])}")
        
        # Privacy
        privacy = results.get('privacy', {})
        report_sections.append("\n*🔒 Privacy:*")
        report_sections.append(f"  Score: {privacy.get('score') or 0}/100")
        report_sections.append(f"  Privacy Policy: {'✅' if privacy.get('has_privacy_policy') else '❌'}")
        report_sections.append(f"  Cookie Consent: {'✅' if privacy.get('has_cookie_consent') else '❌'}")
        report_sections.append(f"  GDPR Compliant: {'✅' if privacy.get('gdpr_compliant') else '❌'}")
        report_sections.append(f"  Tracking Scripts: {privacy.get('tracking_count') or 0}")
        
        # Auth
        auth = results.get('auth', {})
        report_sections.append("\n*🔐 Authentication:*")
        providers = auth.get('providers', [])
        if providers:
            report_sections.append(f"  Providers: {', '.join(providers)}")
        else:
            report_sections.append("  Providers: None detected")
        report_sections.append(f"  Has OAuth: {'✅' if auth.get('has_oauth') else '❌'}")
        report_sections.append(f"  Has SSO: {'✅' if auth.get('has_sso') else '❌'}")
        
        # Database
        database = results.get('database', {})
        report_sections.append("\n*🗄️ Database:*")
        db_detected = database.get('detected', [])
        if db_detected:
            report_sections.append(f"  Detected: {', '.join(db_detected)}")
        else:
            report_sections.append("  Detected: None")
        
        # CORS
        cors = results.get('cors', {})
        report_sections.append("\n*🌐 CORS:*")
        report_sections.append(f"  Enabled: {'✅' if cors.get('enabled') else '❌'}")
        if cors.get('enabled'):
            report_sections.append(f"  Allow Origin: {cors.get('allow_origin', 'N/A')}")
            report_sections.append(f"  Security Level: {cors.get('security_level', 'N/A')}")
            if cors.get('methods'):
                report_sections.append(f"  Methods: {', '.join(cors['methods'])}")
        
        # Mobile
        mobile = results.get('mobile', {})
        report_sections.append("\n*📱 Mobile Optimization:*")
        report_sections.append(f"  Score: {mobile.get('score') or 0}/100")
        report_sections.append(f"  Responsive: {mobile.get('responsive') or 'Unknown'}")
        report_sections.append(f"  Has Viewport: {'✅' if mobile.get('has_viewport') else '❌'}")
        report_sections.append(f"  AMP Support: {'✅' if mobile.get('has_amp') else '❌'}")
        
        # Payment
        payment = results.get('payment', {})
        report_sections.append("\n*💳 Payment Providers:*")
        pay_providers = payment.get('providers', [])
        if pay_providers:
            report_sections.append(f"  Providers: {', '.join(pay_providers)}")
        else:
            report_sections.append("  Providers: None detected")
        report_sections.append(f"  Has Cart: {'✅' if payment.get('has_cart') else '❌'}")
        report_sections.append(f"  Has Checkout: {'✅' if payment.get('has_checkout') else '❌'}")
        
        # Paywall
        paywall = results.get('paywall', {})
        report_sections.append("\n*💰 Paywall Detection:*")
        report_sections.append(
            f"  Status: {'🔴 Detected' if paywall.get('detected') else '🟢 None'}"
        )
        if paywall.get('detected'):
            report_sections.append(f"  Type: {paywall.get('type')}")
            report_sections.append(f"  Confidence: {paywall.get('confidence', 0):.0%}")
        
        # Security
        security = results.get('security', {})
        report_sections.append("\n*🛡️ Security Status:*")
        report_sections.append(f"  Status: {security.get('status')}")
        report_sections.append(f"  Overall Score: {security.get('overall_score', 0)}/100")
        if security.get('vulnerabilities'):
            report_sections.append(f"  Vulnerabilities: {len(security['vulnerabilities'])}")
        if security.get('leaked'):
            report_sections.append("  ⚠️ Domain found in breach databases!")
        
        return "\n".join(report_sections)
    
    def _get_timestamp(self) -> str:
        """Get current timestamp"""
        from datetime import datetime
        return datetime.now().strftime('%Y-%m-%d %H:%M:%S')
    
    def get_simple_status(self, results: Dict[str, Any]) -> str:
        """Get simple emoji status for bookmarks list"""
        if 'error' in results:
            return '❌'
        
        summary = results.get('summary', {})
        
        # If accessible and no major issues, return green
        if summary.get('accessible') and not summary.get('has_paywall'):
            if summary.get('security_status') in ['SECURE', 'NO_INFO']:
                return '🟢'
        
        # Otherwise red
        return '🔴'