bot.py

"""Main Telegram Bot - handles all user interactions"""
import asyncio
from pathlib import Path

from loguru import logger
from telegram import (
    InlineKeyboardButton,
    InlineKeyboardMarkup,
    KeyboardButton,
    ReplyKeyboardMarkup,
    Update,
    WebAppInfo,
)
from telegram.constants import ParseMode
from telegram.ext import (
    Application,
    CallbackQueryHandler,
    CommandHandler,
    ContextTypes,
    MessageHandler,
    PersistenceInput,
    PicklePersistence,
    filters,
)

import config
from aggregator import ResultAggregator
from database import Database
from utils import is_valid_url, normalize_url, setup_logging, truncate_text
from log_server import start_log_server
from batch_scanner import BatchScanner, DeepScanner, ScanQueue
from export_handler import ExportHandler
from visual_enhancer import VisualEnhancer


class APIBot:
    """Telegram Bot for API Resource Availability Testing"""
    
    def __init__(self):
        """Initialize bot"""
        self.aggregator = ResultAggregator()
        self.db = Database()  # Only for bot-wide stats
        self.active_scans = {}  # Track active scans per user
        self.start_time = None  # Track bot start time
        self.is_running = False  # Track bot status
        self.last_status_update = None  # Track last status update time
        self.update_task = None  # Background task for periodic status updates
        
        # New features
        self.batch_scanner = BatchScanner(self.aggregator, max_parallel=3)
        self.deep_scanner = DeepScanner(self.aggregator, max_depth=2, max_pages=10)
        self.scan_queue = ScanQueue(max_concurrent=2)
        self.visual = VisualEnhancer()
        self.cancel_flags = {}  # Track cancel requests per user
        
        # Setup logging
        setup_logging(config.LOG_LEVEL, config.LOG_FILE)
        
        # Ensure user data directory exists
        self.user_data_dir = Path("telegram_user_data")
        self.user_data_dir.mkdir(exist_ok=True)
    
    def _get_main_menu_keyboard(self) -> ReplyKeyboardMarkup:
        """Get persistent reply keyboard for main menu"""
        # Web App hosted on same Render instance
        webapp_url = "https://api-bot-in6q.onrender.com/webapp"
        
        keyboard = [
            [KeyboardButton("🌐 Open Web Scanner", web_app=WebAppInfo(url=webapp_url))],
            [KeyboardButton("📚 Bookmarks"), KeyboardButton("📊 Stats")],
            [KeyboardButton("📦 Batch Scan"), KeyboardButton("🔬 Deep Scan")],
            [KeyboardButton("📤 Export"), KeyboardButton("🔴 Status")],
            [KeyboardButton("❓ Help")]
        ]
        return ReplyKeyboardMarkup(
            keyboard, 
            resize_keyboard=True, 
            one_time_keyboard=False
        )
    
    async def _update_bot_status(self, bot, force=False):
        """Update bot's Telegram status/description"""
        try:
            from datetime import datetime, timedelta
            
            # Rate limit: only update every 30 seconds (unless forced)
            if not force and self.last_status_update:
                time_since_update = datetime.now() - self.last_status_update
                if time_since_update < timedelta(seconds=30):
                    logger.debug("Skipping status update (rate limited)")
                    return
            
            if self.is_running and self.start_time:
                uptime = datetime.now() - self.start_time
                days = uptime.days
                hours, remainder = divmod(uptime.seconds, 3600)
                minutes, _ = divmod(remainder, 60)
                
                # Format uptime for status
                if days > 0:
                    uptime_str = f"{days}d {hours}h"
                elif hours > 0:
                    uptime_str = f"{hours}h {minutes}m"
                else:
                    uptime_str = f"{minutes}m"
                
                status_text = f"🟢 Active | Uptime: {uptime_str} | 21 Scanners Running"
                status_emoji = "🟢"
            else:
                status_text = "🔴 Initializing..."
                status_emoji = "🔴"
            
            # Update bot's display name with status indicator (visible everywhere)
            try:
                await bot.set_my_name(
                    name=f"{status_emoji} API URL Bot"
                )
            except Exception as e:
                logger.warning(f"Could not update bot name: {e}")
            
            # Set bot's short description (visible in bot info)
            await bot.set_my_short_description(
                short_description=status_text
            )
            
            # Set bot's description (visible when starting chat)
            full_description = (
                "🤖 API & Resource Testing Bot\n\n"
                f"Status: {status_text}\n\n"
                "✅ 19 Advanced Scanners\n"
                "⚡ Parallel Processing\n"
                "🔍 Comprehensive Analysis\n\n"
                "Send any URL to start scanning!"
            )
            await bot.set_my_description(
                description=full_description
            )
            
            self.last_status_update = datetime.now()
            logger.info(f"Bot status updated: {status_text}")
        except Exception as e:
            logger.error(f"Failed to update bot status: {e}")
    
    async def _periodic_status_update(self, bot):
        """Background task to update bot status every 5 minutes"""
        import httpx
        try:
            while self.is_running:
                await asyncio.sleep(300)  # Wait 5 minutes
                if self.is_running:  # Check again after sleep
                    logger.info("Periodic status update triggered")
                    await self._update_bot_status(bot, force=False)
                    
                    # Keep-alive: ping own log server to prevent Render sleep
                    try:
                        async with httpx.AsyncClient(timeout=10) as client:
                            await client.get("http://localhost:10000/")
                            logger.info("✓ Keep-alive ping sent")
                    except Exception as ping_error:
                        logger.debug(f"Keep-alive ping failed (non-critical): {ping_error}")
        except asyncio.CancelledError:
            logger.info("Periodic status update task cancelled")
        except Exception as e:
            logger.error(f"Error in periodic status update: {e}")
        
    async def post_init(self, application: Application):
        """Post initialization - setup database"""
        from datetime import datetime
        from performance import warm_up, clear_old_cache
        
        await self.db.initialize()
        self.is_running = True
        self.start_time = datetime.now()
        
        # Start log server FIRST (Render needs open port immediately)
        logger.info("Starting web log server...")
        await start_log_server()
        
        # Warm up for Render performance
        logger.info("Warming up services...")
        await warm_up()
        
        # Start cache cleanup task
        asyncio.create_task(self._periodic_cache_cleanup())
        
        # Initialize bot-wide data if not exists
        if 'total_users' not in application.bot_data:
            application.bot_data['total_users'] = 0
        
        # Set bot status in Telegram
        await self._update_bot_status(application.bot, force=True)
        
        # Start periodic status update task
        self.update_task = asyncio.create_task(
            self._periodic_status_update(application.bot)
        )
        logger.info("Periodic status update task started")
        
        logger.info("Bot initialized successfully")
    
    async def _periodic_cache_cleanup(self):
        """Background task to clean expired cache every 10 minutes"""
        from performance import clear_old_cache
        try:
            while self.is_running:
                await asyncio.sleep(600)  # 10 minutes
                clear_old_cache()
                logger.debug("Cache cleanup completed")
        except asyncio.CancelledError:
            pass
    
    async def post_shutdown(self, application: Application):
        """Post shutdown - update status to offline with delay"""
        import asyncio
        
        try:
            logger.info("Starting shutdown sequence...")
            self.is_running = False
            
            # Cancel periodic update task
            if self.update_task and not self.update_task.done():
                self.update_task.cancel()
                try:
                    await self.update_task
                except asyncio.CancelledError:
                    pass
                logger.info("Periodic update task cancelled")
            
            # Update bot status to offline (with retry and delay)
            update_success = False
            try:
                logger.info("Updating bot status to offline...")
                
                # Try to update bot name in chat/profile
                try:
                    await application.bot.set_my_name(
                        name="🔴 API URL Bot"
                    )
                    logger.info("✓ Bot name updated to offline")
                except Exception as e:
                    logger.warning(f"Could not update bot name: {e}")
                
                # Update short description (visible in bot info)
                try:
                    await application.bot.set_my_short_description(
                        short_description="🔴 Offline"
                    )
                    logger.info("✓ Short description updated")
                except Exception as e:
                    logger.warning(f"Could not update short description: {e}")
                
                # Update full description (visible when starting chat)
                try:
                    await application.bot.set_my_description(
                        description=(
                            "🤖 API & Resource Testing Bot\n\n"
                            "Status: 🔴 Offline\n\n"
                            "✅ 19 Advanced Scanners\n"
                            "⚡ Parallel Processing\n"
                            "🔍 Comprehensive Analysis\n\n"
                            "Bot is currently not running."
                        )
                    )
                    logger.info("✓ Full description updated")
                except Exception as e:
                    logger.warning(f"Could not update description: {e}")
                
                update_success = True
                logger.info("Bot status successfully updated to offline")
                
                # Add delay to ensure updates are sent
                logger.info("Waiting 2 seconds for status updates to propagate...")
                await asyncio.sleep(2)
                
            except Exception as e:
                logger.warning(f"Could not update Telegram status: {e}")
                
        except Exception as e:
            logger.error(f"Error during shutdown: {e}")
        finally:
            # Close database
            try:
                await self.db.close()
            except Exception as e:
                logger.error(f"Error closing database: {e}")
            
            logger.info("Bot shutdown complete")
    
    async def start_command(self, update: Update, context: ContextTypes.DEFAULT_TYPE):
        """Handle /start command"""
        from datetime import datetime
        
        # Initialize user data if first time
        if 'initialized' not in context.user_data:
            context.user_data['initialized'] = True
            context.user_data['bookmarks'] = []
            context.user_data['scan_history'] = []
            context.user_data['preferences'] = {
                'show_detailed': True,
                'auto_bookmark': False,
                'notification_enabled': True
            }
            context.user_data['first_seen'] = datetime.now().isoformat()
            
            # Increment total users count
            context.bot_data['total_users'] = context.bot_data.get('total_users', 0) + 1
            logger.info(f"New user registered: {update.effective_user.id}")
        
        # Update last seen
        context.user_data['last_seen'] = datetime.now().isoformat()
        
        # Get total user count from bot_data
        total_users = context.bot_data.get('total_users', 0)
        
        # Calculate real-time uptime
        if self.is_running and self.start_time:
            uptime = datetime.now() - self.start_time
            days = uptime.days
            hours, remainder = divmod(uptime.seconds, 3600)
            minutes, _ = divmod(remainder, 60)
            
            # Format uptime
            uptime_parts = []
            if days > 0:
                uptime_parts.append(f"{days}d")
            if hours > 0:
                uptime_parts.append(f"{hours}h")
            if minutes > 0:
                uptime_parts.append(f"{minutes}m")
            uptime_str = " ".join(uptime_parts) if uptime_parts else "< 1m"
            
            status_text = f"🟢 *Active* | Uptime: `{uptime_str}` | 👥 {total_users} Users | 21 Scanners"
        else:
            status_text = f"🔴 *Initializing...* | 👥 {total_users} Users"
        
        # Dynamic welcome message with real-time status
        welcome_message = (
            f"🤖 *API Resource Availability Bot*\n\n"
            f"*Status:* {status_text}\n\n"
            f"🔍 *21 Advanced Scanners - Comprehensive Website Analysis*\n\n"
            f"*Core Scanners:*\n"
            f"• 🌐 Availability & Latency Detection\n"
            f"• 🔍 DNS & IP Information Lookup\n"
            f"• 🔒 Protocol & Security Analysis\n"
            f"• 💰 Paywall & Authentication Detection\n"
            f"• ⚠️ Security Breach Status Check\n\n"
            f"*Performance & SEO:*\n"
            f"• ⚡ Performance Metrics (Load Time, TTFB)\n"
            f"• 📊 SEO Analysis (Meta Tags, Sitemap)\n"
            f"• 📱 Mobile Optimization Check\n\n"
            f"*Security & Privacy:*\n"
            f"• 🔐 Authentication Methods (OAuth, SSO)\n"
            f"• 🍪 Privacy & Cookie Compliance\n"
            f"• 🌍 CORS Configuration\n"
            f"• 🔓 Insecure Content Detection\n\n"
            f"*Advanced Analysis:*\n"
            f"• 🛠️ Tech Stack Detection (CMS, Frameworks, Hidden Tech)\n"
            f"• 💾 Database Detection\n"
            f"• 💳 Payment Provider Detection\n"
            f"• 🔄 Redirect Chain Analysis\n"
            f"• 🌍 Geolocation & Network Detection\n"
            f"• 📡 CDN & Hosting Provider Detection\n"
            f"• 🤖 Robots.txt & Sitemap Checker\n"
            f"• 📜 Certificate & SSL Analysis\n\n"
            f"⚡ *Parallel Processing* - All scanners run simultaneously!\n"
            f"📊 Get detailed reports in seconds!\n"
            f"🌐 *NEW:* WebApp interface available!\n\n"
            f"Send me any URL to start comprehensive scanning!"
        )
        
        # WebApp button - Uncomment and update URL after deploying webapp/index.html
        # from telegram import InlineKeyboardButton, InlineKeyboardMarkup, WebAppInfo
        # 
        # Deploy webapp/index.html to GitHub Pages, Netlify, or Vercel first, then:
        # keyboard = [
        #     [InlineKeyboardButton(
        #         "🌐 Open Web Scanner",
        #         web_app=WebAppInfo(url="https://YOUR-USERNAME.github.io/bot-webapp/index.html")
        #     )],
        #     [InlineKeyboardButton("📖 Help", callback_data="show_help")],
        # ]
        # reply_markup = InlineKeyboardMarkup(keyboard)
        
        await update.message.reply_text(
            welcome_message,
            parse_mode=ParseMode.MARKDOWN,
            reply_markup=self._get_main_menu_keyboard()
        )
    
    async def help_command(self, update: Update, context: ContextTypes.DEFAULT_TYPE):
        """Handle /help command"""
        help_text = (
            "*Available Commands:*\n\n"
            "/start - Start the bot\n"
            "/start - Start the bot\n"
            "/help - Show this help message\n"
            "/bookmarks - View your bookmarks\n"
            "/stats - View your statistics\n"
            "/status - Check bot status\n"
            "/batch - Scan multiple URLs at once\n"
            "/deepscan - Deep scan with link crawling\n"
            "/export - Export last scan results\n"
            "/cancel - Cancel current mode\n"
            "/cleardata - Clear all your data\n\n"
            "*New Features:*\n"
            "📦 **Batch Scan** - Scan up to 10 URLs simultaneously\n"
            "🔬 **Deep Scan** - Crawl and scan linked pages (up to 2 levels)\n"
            "📤 **Export** - Export results in JSON, CSV, Markdown, HTML, or Text\n"
            "✨ **Enhanced Visuals** - Beautiful progress bars and animations\n\n"
            "*Usage:*\n"
            "Simply send me any URL to scan!"
        )
        await update.message.reply_text(
            help_text,
            parse_mode=ParseMode.MARKDOWN,
            reply_markup=self._get_main_menu_keyboard()
        )
    
    async def status_command(self, update: Update, context: ContextTypes.DEFAULT_TYPE):
        """Handle /status command - show bot status"""
        from datetime import datetime
        
        # Update bot's Telegram status
        await self._update_bot_status(context.bot)
        
        if not self.is_running or not self.start_time:
            status_text = "🔴 *Bot Status: Unknown*\n\nCannot determine bot status."
        else:
            # Calculate uptime
            uptime = datetime.now() - self.start_time
            days = uptime.days
            hours, remainder = divmod(uptime.seconds, 3600)
            minutes, seconds = divmod(remainder, 60)
            
            # Format uptime
            uptime_parts = []
            if days > 0:
                uptime_parts.append(f"{days}d")
            if hours > 0:
                uptime_parts.append(f"{hours}h")
            if minutes > 0:
                uptime_parts.append(f"{minutes}m")
            uptime_parts.append(f"{seconds}s")
            uptime_str = " ".join(uptime_parts)
            
            # Get stats
            user_id = update.effective_user.id
            
            # Get stats from context.user_data
            total_scans = len(context.user_data.get('scan_history', []))
            bookmarks_count = len(context.user_data.get('bookmarks', []))
            
            # Check active scans
            active_scan_count = len(self.active_scans)
            
            # Get performance stats
            from performance import get_performance_stats
            perf_stats = get_performance_stats()
            cache_entries = perf_stats['cache']['total_entries']
            cache_mb = perf_stats['cache']['memory_mb']
            
            status_text = (
                f"🟢 *Bot Status: Active*\n\n"
                f"⏱️ Uptime: `{uptime_str}`\n"
                f"📅 Started: `{self.start_time.strftime('%Y-%m-%d %H:%M:%S')}`\n"
                f"🔄 Active Scans: {active_scan_count}\n\n"
                f"*Your Activity:*\n"
                f"📊 Total Scans: {total_scans}\n"
                f"📚 Bookmarks: {bookmarks_count}\n\n"
                f"*Performance:*\n"
                f"💾 Cache: {cache_entries} entries ({cache_mb:.2f} MB)\n"
                f"⚡ Fast mode: {'ON' if config.ENABLE_CACHING else 'OFF'}\n\n"
                f"*Scanner Modules:*\n"
                f"✅ 21 scanners active\n"
                f"⚡ Parallel execution enabled\n"
                f"🔍 Comprehensive analysis"
            )
        
        await update.message.reply_text(
            status_text,
            parse_mode=ParseMode.MARKDOWN,
            reply_markup=self._get_main_menu_keyboard()
        )
    
    async def handle_keyboard_button(self, update: Update, context: ContextTypes.DEFAULT_TYPE):
        """Handle keyboard button presses"""
        text = update.message.text
        
        if text == "📚 Bookmarks":
            await self.bookmarks_command(update, context)
        elif text == "📊 Stats":
            await self.stats_command(update, context)
        elif text == "📦 Batch Scan":
            await self.batch_scan_command(update, context)
        elif text == "🔬 Deep Scan":
            await self.deep_scan_command(update, context)
        elif text == "📤 Export":
            await self.export_command(update, context)
        elif text == "🔴 Status":
            await self.status_command(update, context)
        elif text == "❓ Help":
            await self.help_command(update, context)
    
    async def bookmarks_command(self, update: Update, context: ContextTypes.DEFAULT_TYPE):
        """Handle /bookmarks command"""
        bookmarks = context.user_data.get('bookmarks', [])
        
        if not bookmarks:
            await update.message.reply_text(
                config.MESSAGES['no_bookmarks'],
                reply_markup=self._get_main_menu_keyboard()
            )
            return
        
        # Create keyboard with bookmarks
        keyboard = []
        for bookmark in bookmarks[:20]:  # Limit to 20
            # Get status emoji
            status_emoji = '🟢' if bookmark.get('last_status') == 'AVAILABLE' else '🔴'
            button_text = f"{status_emoji} {truncate_text(bookmark.get('title', bookmark['url']), 30)}"
            keyboard.append([
                InlineKeyboardButton(
                    button_text,
                    callback_data=f"scan:{bookmark['url']}"
                )
            ])
        
        # Add refresh all button
        keyboard.append([
            InlineKeyboardButton("🔄 Refresh All", callback_data="refresh_bookmarks")
        ])
        
        reply_markup = InlineKeyboardMarkup(keyboard)
        await update.message.reply_text(
            f"📚 *Your Bookmarks* ({len(bookmarks)})\n"
            "Click to scan again:",
            parse_mode=ParseMode.MARKDOWN,
            reply_markup=reply_markup
        )
    
    async def stats_command(self, update: Update, context: ContextTypes.DEFAULT_TYPE):
        """Handle /stats command"""
        bookmarks = context.user_data.get('bookmarks', [])
        scan_history = context.user_data.get('scan_history', [])
        
        # Calculate most scanned URL
        from collections import Counter
        if scan_history:
            url_counter = Counter([scan['url'] for scan in scan_history if 'url' in scan])
            most_common = url_counter.most_common(1)
            most_scanned_url = most_common[0][0] if most_common else None
            most_scanned_count = most_common[0][1] if most_common else 0
        else:
            most_scanned_url = None
            most_scanned_count = 0
        
        stats_text = (
            "*📊 Your Statistics*\n\n"
            f"📚 Total Bookmarks: {len(bookmarks)}\n"
            f"🔍 Total Scans: {len(scan_history)}\n"
        )
        
        if most_scanned_url:
            stats_text += f"\n🎯 Most Scanned: {truncate_text(most_scanned_url, 40)}\n"
            stats_text += f"   ({most_scanned_count} times)"
        
        await update.message.reply_text(
            stats_text,
            parse_mode=ParseMode.MARKDOWN,
            reply_markup=self._get_main_menu_keyboard()
        )
    
    async def settings_command(self, update: Update, context: ContextTypes.DEFAULT_TYPE):
        """Handle /settings command"""
        prefs = context.user_data.get('preferences', {
            'show_detailed': True,
            'auto_bookmark': False,
            'notification_enabled': True
        })
        
        keyboard = [
            [InlineKeyboardButton(
                f"{'✅' if prefs.get('show_detailed') else '❌'} Detailed Reports",
                callback_data="toggle:show_detailed"
            )],
            [InlineKeyboardButton(
                f"{'✅' if prefs.get('auto_bookmark') else '❌'} Auto Bookmark",
                callback_data="toggle:auto_bookmark"
            )],
            [InlineKeyboardButton(
                f"{'✅' if prefs.get('notification_enabled') else '❌'} Notifications",
                callback_data="toggle:notification_enabled"
            )],
        ]
        
        reply_markup = InlineKeyboardMarkup(keyboard)
        await update.message.reply_text(
            "*⚙️ Settings*\n\nClick to toggle options:",
            parse_mode=ParseMode.MARKDOWN,
            reply_markup=reply_markup
        )
    
    async def cleardata_command(self, update: Update, context: ContextTypes.DEFAULT_TYPE):
        """Handle /cleardata command - clear all user data"""
        from telegram import InlineKeyboardButton, InlineKeyboardMarkup
        
        # Ask for confirmation
        keyboard = [
            [
                InlineKeyboardButton("✅ Yes, Clear All Data", callback_data="confirm_cleardata"),
                InlineKeyboardButton("❌ Cancel", callback_data="cancel_cleardata")
            ]
        ]
        
        await update.message.reply_text(
            "⚠️ *Warning*\n\n"
            "This will permanently delete:\n"
            "• All your bookmarks\n"
            "• All scan history\n"
            "• All your preferences\n\n"
            "This action cannot be undone. Are you sure?",
            parse_mode=ParseMode.MARKDOWN,
            reply_markup=InlineKeyboardMarkup(keyboard)
        )
    
    async def handle_url(self, update: Update, context: ContextTypes.DEFAULT_TYPE):
        """Handle URL messages with security validation"""
        user_id = update.effective_user.id
        text = update.message.text.strip()
        
        # Check if it's a keyboard button (fallback if handler didn't catch it)
        if text in ['📚 Bookmarks', '📊 Stats', '📦 Batch Scan', '🔬 Deep Scan', '📤 Export', '🔴 Status', '❓ Help']:
            await self.handle_keyboard_button(update, context)
            return
        
        # Check for batch mode
        if context.user_data.get('expecting_batch_urls'):
            urls = [line.strip() for line in text.split('\n') if line.strip()]
            urls = [u for u in urls if is_valid_url(u)]
            
            if not urls:
                await update.message.reply_text(
                    f"{self.visual.EMOJIS['error']} No valid URLs found!\n\n"
                    f"Please send URLs, one per line.",
                    reply_markup=self._get_main_menu_keyboard()
                )
                return
            
            if len(urls) > 10:
                await update.message.reply_text(
                    f"{self.visual.EMOJIS['warning']} Too many URLs! Maximum is 10.\n\n"
                    f"Please send fewer URLs.",
                    reply_markup=self._get_main_menu_keyboard()
                )
                return
            
            # Clear batch mode
            context.user_data.pop('expecting_batch_urls', None)
            
            # Process batch
            await self.process_batch_scan(update, urls)
            return
        
        # Check for deep scan mode
        if context.user_data.get('expecting_deep_scan_url'):
            if not is_valid_url(text):
                await update.message.reply_text(
                    f"{self.visual.EMOJIS['error']} Invalid URL!\n\n"
                    f"Please send a valid URL.",
                    reply_markup=self._get_main_menu_keyboard()
                )
                return
            
            # Clear deep scan mode
            context.user_data.pop('expecting_deep_scan_url', None)
            
            # Process deep scan
            await self.process_deep_scan(update, text, context)
            return
        
        # Regular URL handling
        url = text
        
        # SECURITY: Rate limiting check
        from security_utils import check_rate_limit, validate_url as secure_validate_url, sanitize_url
        
        is_allowed, error_msg = check_rate_limit(user_id)
        if not is_allowed:
            await update.message.reply_text(
                f"⚠️ {error_msg}",
                reply_markup=self._get_main_menu_keyboard()
            )
            return
        
        # SECURITY: Validate URL with advanced checks
        is_valid, error_msg = secure_validate_url(url)
        if not is_valid:
            await update.message.reply_text(
                f"❌ {error_msg}",
                reply_markup=self._get_main_menu_keyboard()
            )
            logger.warning(f"Invalid URL from user {user_id}: {url}")
            return
        
        # Fallback to legacy validation
        if not is_valid_url(url):
            await update.message.reply_text(
                config.MESSAGES['invalid_url'],
                reply_markup=self._get_main_menu_keyboard()
            )
            return
        
        # SECURITY: Sanitize URL
        url = sanitize_url(url)
        
        # Normalize URL
        url = normalize_url(url)
        
        # Check if already scanning
        if user_id in self.active_scans:
            await update.message.reply_text(
                "⏳ Please wait for the current scan to complete.",
                reply_markup=self._get_main_menu_keyboard()
            )
            return
        
        # Start scan
        self.active_scans[user_id] = True
        
        # Show animated scanning message
        status_message = await update.message.reply_text(
            f"{self.visual.EMOJIS['loading'][0]} **Initializing scan...**\n\n"
            f"URL: {url[:50]}{'...' if len(url) > 50 else ''}",
            parse_mode=ParseMode.MARKDOWN,
            reply_markup=self._get_main_menu_keyboard()
        )
        
        # Start animation task
        animation_task = asyncio.create_task(
            self.visual.animate_scanning(status_message, url[:40])
        )
        
        try:
            from datetime import datetime
            
            # Check for cancel before scanning
            if self.cancel_flags.get(user_id):
                self.cancel_flags.pop(user_id, None)  # Clear flag
                animation_task.cancel()
                try:
                    await animation_task
                except asyncio.CancelledError:
                    pass
                await status_message.edit_text(
                    f"{self.visual.EMOJIS['cross']} Scan cancelled by user.",
                    reply_markup=self._get_main_menu_keyboard()
                )
                self.active_scans.pop(user_id, None)
                return
            
            # Perform scan with timeout (60 seconds max)
            try:
                results = await asyncio.wait_for(
                    self.aggregator.scan_url(url),
                    timeout=60.0
                )
            except asyncio.TimeoutError:
                animation_task.cancel()
                try:
                    await animation_task
                except asyncio.CancelledError:
                    pass
                await status_message.edit_text(
                    f"{self.visual.EMOJIS['cross']} Scan timeout! The URL took too long to respond.\n\n"
                    f"Try again later.",
                    reply_markup=self._get_main_menu_keyboard()
                )
                self.active_scans.pop(user_id, None)
                return
            
            # Cancel animation
            animation_task.cancel()
            try:
                await animation_task
            except asyncio.CancelledError:
                pass
            
            # Save to user's scan history in context.user_data
            if 'scan_history' not in context.user_data:
                context.user_data['scan_history'] = []
            
            context.user_data['scan_history'].append({
                'url': url,
                'results': results,
                'timestamp': datetime.now().isoformat()
            })
            
            # Save last scan result for export
            context.user_data['last_scan_result'] = results
            
            # Keep only last 100 scans to avoid memory issues
            if len(context.user_data['scan_history']) > 100:
                context.user_data['scan_history'] = context.user_data['scan_history'][-100:]
            
            # Check if bookmarked and update status
            bookmarks = context.user_data.get('bookmarks', [])
            for bookmark in bookmarks:
                if bookmark['url'] == url:
                    bookmark['last_status'] = results.get('summary', {}).get('status', 'UNKNOWN')
                    bookmark['last_scan'] = datetime.now().isoformat()
                    break
            
            # Format results with enhanced visuals
            result_text = self.visual.format_scan_result(results)
            
            # Add detailed report section
            prefs = context.user_data.get('preferences', {'show_detailed': True})
            
            if prefs.get('show_detailed', True):
                result_text += "\n\n" + self.aggregator.format_detailed_report(results)
            else:
                result_text += "\n\n" + self.aggregator.format_as_table(results)
            
            # Delete status message
            await status_message.delete()
            
            # Send results with action buttons AND persistent menu
            keyboard = self._get_result_keyboard_with_bookmark(url, bookmarks)
            
            await update.message.reply_text(
                result_text,
                parse_mode=None,  # Disable markdown to prevent parse errors
                reply_markup=keyboard
            )
            
            # Send persistent menu in separate message to keep it visible
            await update.message.reply_text(
                "Use buttons below for quick actions:",
                reply_markup=self._get_main_menu_keyboard()
            )
            
        except Exception as e:
            logger.error(f"Error scanning {url}: {e}")
            await status_message.edit_text(
                config.MESSAGES['error'].format(str(e))
            )
        finally:
            # Remove from active scans
            self.active_scans.pop(user_id, None)
    
    async def handle_callback(self, update: Update, context: ContextTypes.DEFAULT_TYPE):
        """Handle callback queries from inline buttons"""
        from telegram.error import BadRequest
        
        query = update.callback_query
        
        # Try to answer the callback query
        try:
            await query.answer()
        except BadRequest as e:
            if "Query is too old" in str(e) or "query id is invalid" in str(e):
                logger.warning(f"Ignoring expired callback query: {e}")
                return
            raise
        
        user_id = update.effective_user.id
        data = query.data
        
        try:
            if data.startswith('scan:'):
                # Rescan URL
                url = data.split(':', 1)[1]
                await self._rescan_url(query, user_id, url, context)
                
            elif data.startswith('bookmark:'):
                # Add to bookmarks
                url = data.split(':', 1)[1]
                await self._add_bookmark(query, user_id, url, context)
                
            elif data.startswith('unbookmark:'):
                # Remove from bookmarks
                url = data.split(':', 1)[1]
                await self._remove_bookmark(query, user_id, url, context)
                
            elif data.startswith('logs:'):
                # Show logs
                url = data.split(':', 1)[1]
                await self._show_logs(query, user_id, url, context)
                
            elif data.startswith('toggle:'):
                # Toggle setting
                setting = data.split(':', 1)[1]
                await self._toggle_setting(query, user_id, setting, context)
                
            elif data == 'refresh_bookmarks':
                await self._refresh_bookmarks(query, user_id, context)
            
            elif data.startswith('export_'):
                # Export results
                export_format = data.split('_', 1)[1]
                await self._export_results(query, user_id, export_format, context)
                
            elif data == 'confirm_cleardata':
                # Clear all user data
                context.user_data.clear()
                context.user_data['initialized'] = True
                context.user_data['bookmarks'] = []
                context.user_data['scan_history'] = []
                context.user_data['preferences'] = {
                    'show_detailed': True,
                    'auto_bookmark': False,
                    'notification_enabled': True
                }
                from datetime import datetime
                context.user_data['first_seen'] = datetime.now().isoformat()
                
                await query.message.edit_text(
                    "✅ *Data Cleared*\n\n"
                    "All your data has been permanently deleted.",
                    parse_mode=ParseMode.MARKDOWN
                )
                
            elif data == 'cancel_cleardata':
                await query.message.edit_text(
                    "❌ Cancelled. Your data is safe.",
                    parse_mode=ParseMode.MARKDOWN
                )
                
        except Exception as e:
            logger.error(f"Error handling callback {data}: {e}")
            await query.message.reply_text(
                config.MESSAGES['error'].format(str(e))
            )
    
    async def _rescan_url(self, query, user_id: int, url: str, context: ContextTypes.DEFAULT_TYPE):
        """Rescan a URL"""
        await query.message.edit_text(config.MESSAGES['scanning'])
        
        try:
            from datetime import datetime
            
            results = await self.aggregator.scan_url(url)
            
            # Save to scan history
            if 'scan_history' not in context.user_data:
                context.user_data['scan_history'] = []
            
            context.user_data['scan_history'].append({
                'url': url,
                'results': results,
                'timestamp': datetime.now().isoformat()
            })
            
            # Keep only last 100 scans
            if len(context.user_data['scan_history']) > 100:
                context.user_data['scan_history'] = context.user_data['scan_history'][-100:]
            
            # Update bookmark status if bookmarked
            bookmarks = context.user_data.get('bookmarks', [])
            for bookmark in bookmarks:
                if bookmark['url'] == url:
                    bookmark['last_status'] = results.get('summary', {}).get('status', 'UNKNOWN')
                    bookmark['last_scan'] = datetime.now().isoformat()
                    break
            
            prefs = context.user_data.get('preferences', {'show_detailed': True})
            
            if prefs.get('show_detailed', True):
                result_text = self.aggregator.format_detailed_report(results)
            else:
                result_text = self.aggregator.format_as_table(results)
            
            keyboard = self._get_result_keyboard_with_bookmark(url, bookmarks)
            
            await query.message.edit_text(
                result_text,
                parse_mode=ParseMode.MARKDOWN,
                reply_markup=keyboard
            )
        except Exception as e:
            logger.error(f"Error rescanning {url}: {e}")
            await query.message.edit_text(
                config.MESSAGES['error'].format(str(e))
            )
    
    async def _add_bookmark(self, query, user_id: int, url: str, context: ContextTypes.DEFAULT_TYPE):
        """Add URL to bookmarks"""
        from datetime import datetime
        
        bookmarks = context.user_data.get('bookmarks', [])
        
        # Check if already bookmarked
        is_bookmarked = any(b['url'] == url for b in bookmarks)
        
        if is_bookmarked:
            await query.answer("ℹ️ Already bookmarked!", show_alert=False)
            return
        
        # Add bookmark
        bookmarks.append({
            'url': url,
            'title': url,
            'created_at': datetime.now().isoformat(),
            'last_scan': None,
            'last_status': None
        })
        context.user_data['bookmarks'] = bookmarks
        
        await query.answer(config.MESSAGES['bookmark_added'], show_alert=False)
        # Update keyboard to show remove option
        try:
            keyboard = self._get_result_keyboard_with_bookmark(url, bookmarks)
            await query.message.edit_reply_markup(reply_markup=keyboard)
        except Exception as e:
            # Ignore if message not modified
            logger.debug(f"Keyboard update skipped: {e}")
    
    async def _remove_bookmark(self, query, user_id: int, url: str, context: ContextTypes.DEFAULT_TYPE):
        """Remove URL from bookmarks"""
        bookmarks = context.user_data.get('bookmarks', [])
        
        # Remove bookmark
        bookmarks = [b for b in bookmarks if b['url'] != url]
        context.user_data['bookmarks'] = bookmarks
        
        await query.answer(config.MESSAGES['bookmark_removed'], show_alert=False)
        # Update keyboard to show add option
        try:
            keyboard = self._get_result_keyboard_with_bookmark(url, bookmarks)
            await query.message.edit_reply_markup(reply_markup=keyboard)
        except Exception as e:
            # Ignore if message not modified
            logger.debug(f"Keyboard update skipped: {e}")
    
    async def _show_logs(self, query, user_id: int, url: str, context: ContextTypes.DEFAULT_TYPE):
        """Show scan history logs"""
        
        scan_history = context.user_data.get('scan_history', [])
        
        # Filter history for this URL (last 5)
        url_history = [h for h in scan_history if h.get('url') == url][-5:]
        
        if not url_history:
            await query.answer("No scan history found", show_alert=True)
            return
        
        log_text = f"*📋 Scan History for*\n`{truncate_text(url, 50)}`\n\n"
        
        for i, scan in enumerate(url_history, 1):
            status = scan.get('results', {}).get('summary', {}).get('status', 'UNKNOWN')
            timestamp = scan.get('timestamp', 'Unknown')
            log_text += f"{i}. {timestamp}\n"
            log_text += f"   Status: {status}\n\n"
        
        await query.message.reply_text(
            log_text,
            parse_mode=ParseMode.MARKDOWN
        )
        await query.answer()
    
    async def _toggle_setting(self, query, user_id: int, setting: str, context: ContextTypes.DEFAULT_TYPE):
        """Toggle a user setting"""
        prefs = context.user_data.get('preferences', {
            'show_detailed': True,
            'auto_bookmark': False,
            'notification_enabled': True
        })
        
        prefs[setting] = not prefs.get(setting, True)
        context.user_data['preferences'] = prefs
        
        # Update keyboard
        keyboard = [
            [InlineKeyboardButton(
                f"{'✅' if prefs.get('show_detailed') else '❌'} Detailed Reports",
                callback_data="toggle:show_detailed"
            )],
            [InlineKeyboardButton(
                f"{'✅' if prefs.get('auto_bookmark') else '❌'} Auto Bookmark",
                callback_data="toggle:auto_bookmark"
            )],
            [InlineKeyboardButton(
                f"{'✅' if prefs.get('notification_enabled') else '❌'} Notifications",
                callback_data="toggle:notification_enabled"
            )],
        ]
        
        await query.message.edit_reply_markup(
            reply_markup=InlineKeyboardMarkup(keyboard)
        )
        await query.answer("✅ Setting updated!")
    
    async def _export_results(self, query, user_id, format, context):
        """Export scan results"""
        last_result = context.user_data.get('last_scan_result')
        
        if not last_result:
            await query.message.edit_text(
                f"{self.visual.EMOJIS['warning']} No scan results available!",
                reply_markup=self._get_main_menu_keyboard()
            )
            return
        
        # Export in requested format
        try:
            if format == 'json':
                content = ExportHandler.to_json(last_result)
                filename = 'scan_result.json'
            elif format == 'csv':
                content = ExportHandler.to_csv(last_result)
                filename = 'scan_result.csv'
            elif format == 'markdown':
                content = ExportHandler.to_markdown(last_result)
                filename = 'scan_result.md'
            elif format == 'html':
                content = ExportHandler.to_html(last_result)
                filename = 'scan_result.html'
            elif format == 'text':
                content = ExportHandler.to_text_report(last_result)
                filename = 'scan_result.txt'
            else:
                await query.message.edit_text(
                    f"{self.visual.EMOJIS['error']} Unknown format!",
                    reply_markup=self._get_main_menu_keyboard()
                )
                return
            
            # Send as document
            from io import BytesIO
            
            file_bytes = BytesIO(content.encode('utf-8'))
            file_bytes.name = filename
            
            await query.message.reply_document(
                document=file_bytes,
                filename=filename,
                caption=f"{self.visual.EMOJIS['success']} Exported as {format.upper()}",
                reply_markup=self._get_main_menu_keyboard()
            )
            
            await query.message.delete()
            
        except Exception as e:
            logger.error(f"Export error: {e}")
            await query.message.edit_text(
                f"{self.visual.EMOJIS['error']} Export failed: {str(e)}",
                reply_markup=self._get_main_menu_keyboard()
            )
    
    async def _refresh_bookmarks(self, query, user_id: int, context: ContextTypes.DEFAULT_TYPE):
        """Refresh all bookmarks status"""
        from datetime import datetime
        
        await query.answer("🔄 Refreshing bookmarks...")
        
        bookmarks = context.user_data.get('bookmarks', [])
        
        if not bookmarks:
            await query.answer("No bookmarks to refresh", show_alert=True)
            return
        
        # Scan all bookmarks (limit concurrent scans)
        semaphore = asyncio.Semaphore(config.MAX_PARALLEL_SCANS)
        
        async def scan_bookmark(bookmark):
            async with semaphore:
                try:
                    results = await self.aggregator.scan_url(bookmark['url'])
                    bookmark['last_status'] = results.get('summary', {}).get('status', 'UNKNOWN')
                    bookmark['last_scan'] = datetime.now().isoformat()
                except Exception as e:
                    logger.error(f"Error refreshing {bookmark['url']}: {e}")
        
        await asyncio.gather(*[scan_bookmark(b) for b in bookmarks])
        
        await query.message.reply_text(
            "✅ All bookmarks refreshed!\n"
            "Use /bookmarks to view updated status."
        )
    
    def _get_main_keyboard(self) -> InlineKeyboardMarkup:
        """Get main menu keyboard"""
        keyboard = [
            [InlineKeyboardButton("📚 Bookmarks", callback_data="show_bookmarks")],
            [InlineKeyboardButton("📊 Statistics", callback_data="show_stats")],
            [InlineKeyboardButton("⚙️ Settings", callback_data="show_settings")],
        ]
        return InlineKeyboardMarkup(keyboard)
    
    def _get_result_keyboard_with_bookmark(self, url: str, bookmarks: list) -> InlineKeyboardMarkup:
        """Get result action keyboard with bookmark status"""
        is_bookmarked = any(b['url'] == url for b in bookmarks)
        
        bookmark_button = (
            InlineKeyboardButton("🗑️ Remove Bookmark", callback_data=f"unbookmark:{url}")
            if is_bookmarked
            else InlineKeyboardButton("🔖 Bookmark", callback_data=f"bookmark:{url}")
        )
        
        keyboard = [
            [
                InlineKeyboardButton("🔄 Scan Again", callback_data=f"scan:{url}"),
                InlineKeyboardButton("📋 Logs", callback_data=f"logs:{url}")
            ],
            [
                bookmark_button,
                InlineKeyboardButton("📤 Share", switch_inline_query=url)
            ]
        ]
        return InlineKeyboardMarkup(keyboard)
    
    async def batch_scan_command(self, update: Update, context: ContextTypes.DEFAULT_TYPE):
        """Handle batch scan command"""
        # Set state for batch input
        context.user_data['expecting_batch_urls'] = True
        
        message = (
            f"{self.visual.EMOJIS['package']} **Batch Scan Mode**\n\n"
            f"Send me multiple URLs to scan at once:\n\n"
            f"{self.visual.EMOJIS['bullet']} One URL per line\n"
            f"{self.visual.EMOJIS['bullet']} Up to 10 URLs at a time\n"
            f"{self.visual.EMOJIS['bullet']} Example:\n\n"
            f"`https://google.com\n"
            f"https://github.com\n"
            f"https://stackoverflow.com`\n\n"
            f"{self.visual.EMOJIS['info']} Type /cancel to exit batch mode"
        )
        
        await update.message.reply_text(
            message,
            parse_mode=ParseMode.MARKDOWN,
            reply_markup=self._get_main_menu_keyboard()
        )
    
    async def deep_scan_command(self, update: Update, context: ContextTypes.DEFAULT_TYPE):
        """Handle deep scan command"""
        context.user_data['expecting_deep_scan_url'] = True
        
        message = (
            f"{self.visual.EMOJIS['tools']} **Deep Scan Mode**\n\n"
            f"Send me a URL to perform deep scan:\n\n"
            f"{self.visual.EMOJIS['bullet']} Scans main page + linked pages\n"
            f"{self.visual.EMOJIS['bullet']} Follows links up to 2 levels deep\n"
            f"{self.visual.EMOJIS['bullet']} Maximum 10 pages per scan\n"
            f"{self.visual.EMOJIS['bullet']} Same-domain links only\n\n"
            f"{self.visual.EMOJIS['warning']} Deep scans take longer!\n\n"
            f"{self.visual.EMOJIS['info']} Type /cancel to exit deep scan mode"
        )
        
        await update.message.reply_text(
            message,
            parse_mode=ParseMode.MARKDOWN,
            reply_markup=self._get_main_menu_keyboard()
        )
    
    async def export_command(self, update: Update, context: ContextTypes.DEFAULT_TYPE):
        """Handle export command"""
        # Get last scan result for this user
        last_result = context.user_data.get('last_scan_result')
        
        if not last_result:
            await update.message.reply_text(
                f"{self.visual.EMOJIS['warning']} No scan results to export!\n\n"
                f"Scan a URL first, then use this command.",
                reply_markup=self._get_main_menu_keyboard()
            )
            return
        
        # Show export format options
        keyboard = [
            [
                InlineKeyboardButton("📄 JSON", callback_data="export_json"),
                InlineKeyboardButton("📊 CSV", callback_data="export_csv")
            ],
            [
                InlineKeyboardButton("📝 Markdown", callback_data="export_markdown"),
                InlineKeyboardButton("🌐 HTML", callback_data="export_html")
            ],
            [
                InlineKeyboardButton("📃 Text Report", callback_data="export_text")
            ]
        ]
        
        await update.message.reply_text(
            f"{self.visual.EMOJIS['doc']} **Export Scan Results**\n\n"
            f"Choose format:",
            parse_mode=ParseMode.MARKDOWN,
            reply_markup=InlineKeyboardMarkup(keyboard)
        )
    
    async def cancel_command(self, update: Update, context: ContextTypes.DEFAULT_TYPE):
        """Cancel any active operation or special mode"""
        user_id = update.effective_user.id
        
        # Cancel modes
        context.user_data.pop('expecting_batch_urls', None)
        context.user_data.pop('expecting_deep_scan_url', None)
        
        # Set cancel flag for active scans
        self.cancel_flags[user_id] = True
        
        # Remove from active scans
        self.active_scans.pop(user_id, None)
        
        await update.message.reply_text(
            f"{self.visual.EMOJIS['check']} Operation cancelled. Back to normal operation.",
            reply_markup=self._get_main_menu_keyboard()
        )
    
    async def handle_webapp_data(self, update: Update, context: ContextTypes.DEFAULT_TYPE):
        """Handle data from WebApp"""
        import json
        
        try:
            # Parse WebApp data
            data = json.loads(update.effective_message.web_app_data.data)
            mode = data.get('mode')
            urls = data.get('urls', [])
            
            if not urls:
                await update.message.reply_text(
                    f"{self.visual.EMOJIS['error']} No URLs provided!",
                    reply_markup=self._get_main_menu_keyboard()
                )
                return
            
            # Acknowledge receipt
            await update.message.reply_text(
                f"{self.visual.EMOJIS['check']} Received from Web App!\n\n"
                f"Mode: {mode}\n"
                f"URLs: {len(urls)}\n\n"
                f"Processing...",
                reply_markup=self._get_main_menu_keyboard()
            )
            
            # Process based on mode
            if mode == 'single':
                # Regular scan
                url = urls[0]
                user_id = update.effective_user.id
                
                # Check if already scanning
                if user_id in self.active_scans:
                    await update.message.reply_text(
                        "⏳ Please wait for the current scan to complete.",
                        reply_markup=self._get_main_menu_keyboard()
                    )
                    return
                
                self.active_scans[user_id] = True
                
                status_message = await update.message.reply_text(
                    f"{self.visual.EMOJIS['loading'][0]} Scanning...\n\n"
                    f"URL: {url[:50]}{'...' if len(url) > 50 else ''}"
                )
                
                try:
                    results = await asyncio.wait_for(
                        self.aggregator.scan_url(normalize_url(url)),
                        timeout=60.0
                    )
                    
                    # Save result
                    if 'scan_history' not in context.user_data:
                        context.user_data['scan_history'] = []
                    
                    from datetime import datetime
                    context.user_data['scan_history'].append({
                        'url': url,
                        'results': results,
                        'timestamp': datetime.now().isoformat()
                    })
                    context.user_data['last_scan_result'] = results
                    
                    # Format and send results
                    result_text = self.visual.format_scan_result(results)
                    await status_message.delete()
                    await update.message.reply_text(
                        result_text,
                        parse_mode=None,
                        reply_markup=self._get_main_menu_keyboard()
                    )
                    
                except asyncio.TimeoutError:
                    await status_message.edit_text(
                        f"{self.visual.EMOJIS['cross']} Scan timeout!",
                        reply_markup=self._get_main_menu_keyboard()
                    )
                finally:
                    self.active_scans.pop(user_id, None)
                    
            elif mode == 'batch':
                await self.process_batch_scan(update, urls)
                
            elif mode == 'deep':
                await self.process_deep_scan(update, urls[0], context)
                
        except Exception as e:
            logger.error(f"WebApp data error: {e}")
            await update.message.reply_text(
                f"{self.visual.EMOJIS['error']} Error processing WebApp data: {str(e)}",
                reply_markup=self._get_main_menu_keyboard()
            )
    
    async def process_batch_scan(self, update: Update, urls: list):
        """Process batch URL scan"""
        import time
        
        # Show initial message
        progress_msg = await update.message.reply_text(
            f"{self.visual.EMOJIS['loading'][0]} Starting Batch Scan\n\n"
            f"Processing {len(urls)} URLs..."
        )
        
        # Progress callback
        async def progress_callback(completed, total, url):
            try:
                progress_bar = self.visual.create_progress_bar(completed, total)
                await progress_msg.edit_text(
                    f"{self.visual.EMOJIS['scan']} Batch Scanning\n\n"
                    f"{progress_bar}\n\n"
                    f"Completed: {completed}/{total}\n"
                    f"Current: {url[:50]}..."
                )
            except Exception as e:
                logger.debug(f"Progress update skipped: {e}")
        
        # Scan batch with cancel support and timeout (3 minutes for batch)
        start_time = time.time()
        user_id = update.effective_user.id
        cancel_check = lambda: self.cancel_flags.get(user_id, False)
        
        try:
            results = await asyncio.wait_for(
                self.batch_scanner.scan_batch(urls, progress_callback, cancel_check),
                timeout=180.0
            )
        except asyncio.TimeoutError:
            await progress_msg.edit_text(
                f"{self.visual.EMOJIS['cross']} Batch scan timeout! Scan took too long.\n\n"
                f"Try scanning fewer URLs.",
                reply_markup=self._get_main_menu_keyboard()
            )
            self.cancel_flags.pop(user_id, None)
            return
            
        total_time = time.time() - start_time
        
        # Check if cancelled
        if self.cancel_flags.get(user_id):
            await progress_msg.edit_text(
                f"{self.visual.EMOJIS['cross']} Batch scan cancelled by user.",
                reply_markup=self._get_main_menu_keyboard()
            )
            self.cancel_flags.pop(user_id, None)
            return
        
        # Format result
        result_message = self.visual.format_batch_result(results, total_time)
        
        # Try to edit, fallback to new message if edit fails
        try:
            await progress_msg.edit_text(
                result_message,
                parse_mode=None
            )
        except Exception as e:
            logger.warning(f"Failed to edit batch result message: {e}")
            await progress_msg.delete()
            await update.message.reply_text(
                result_message,
                parse_mode=None,
                reply_markup=self._get_main_menu_keyboard()
            )
        
        # Save results for export
        context.user_data['last_scan_result'] = {
            'type': 'batch',
            'results': results,
            'total_time': total_time
        }
    
    async def process_deep_scan(self, update: Update, url: str, context: ContextTypes.DEFAULT_TYPE):
        """Process deep scan"""
        # Show initial message
        progress_msg = await update.message.reply_text(
            f"{self.visual.EMOJIS['loading'][0]} Starting Deep Scan\n\n"
            f"Analyzing {url}...\n\n"
            f"💡 Use /cancel to stop"
        )
        
        # Progress callback
        async def progress_callback(completed, total, current_url):
            try:
                progress_bar = self.visual.create_progress_bar(completed, total)
                await progress_msg.edit_text(
                    f"{self.visual.EMOJIS['tools']} Deep Scanning\n\n"
                    f"{progress_bar}\n\n"
                    f"Pages scanned: {completed}/{total}\n"
                    f"Current: {current_url[:50]}...\n\n"
                    f"💡 Use /cancel to stop"
                )
            except Exception as e:
                logger.debug(f"Progress update skipped: {e}")
        
        # Check cancel support
        user_id = update.effective_user.id
        cancel_check = lambda: self.cancel_flags.get(user_id, False)
        
        # Perform deep scan with cancel support and timeout (2 minutes max)
        try:
            result = await asyncio.wait_for(
                self.deep_scanner.deep_scan(url, progress_callback, cancel_check),
                timeout=120.0
            )
        except asyncio.TimeoutError:
            await progress_msg.edit_text(
                f"{self.visual.EMOJIS['cross']} Deep scan timeout! Scan took too long.\n\n"
                f"Try scanning fewer pages.",
                reply_markup=self._get_main_menu_keyboard()
            )
            self.cancel_flags.pop(user_id, None)
            return
        
        # Check if cancelled
        if self.cancel_flags.get(user_id):
            await progress_msg.edit_text(
                f"{self.visual.EMOJIS['cross']} Deep scan cancelled by user.",
                reply_markup=self._get_main_menu_keyboard()
            )
            self.cancel_flags.pop(user_id, None)
            return
        
        # Format result
        result_message = self.visual.format_deep_scan_result(result)
        
        # Try to edit, fallback to new message if edit fails
        try:
            await progress_msg.edit_text(
                result_message,
                parse_mode=None
            )
        except Exception as e:
            logger.warning(f"Failed to edit deep scan result message: {e}")
            await progress_msg.delete()
            await update.message.reply_text(
                result_message,
                parse_mode=None,
                reply_markup=self._get_main_menu_keyboard()
            )
        
        # Save results for export
        context.user_data['last_scan_result'] = {
            'type': 'deep_scan',
            'result': result
        }
    
    def run(self):
        """Start the bot"""
        
        if not config.TELEGRAM_BOT_TOKEN:
            logger.error("TELEGRAM_BOT_TOKEN not set in configuration!")
            return
        
        # Setup persistence to store user data separately per user
        persistence = PicklePersistence(
            filepath=self.user_data_dir / "bot_persistence.pkl",
            store_data=PersistenceInput(
                user_data=True,  # Store per-user data
                bot_data=True,   # Store bot-wide data
                chat_data=False,  # Don't need chat data
                callback_data=False
            )
        )
        
        # Create application with persistence
        application = (
            Application.builder()
            .token(config.TELEGRAM_BOT_TOKEN)
            .persistence(persistence)
            .post_init(self.post_init)
            .post_shutdown(self.post_shutdown)
            .job_queue(None)  # Disable job queue
            .build()
        )
        
        # Store application reference for cleanup
        self.application = application
        
        # The post_shutdown callback will handle status updates
        # No custom signal handlers needed - python-telegram-bot handles SIGINT/SIGTERM
        
        # Add handlers
        application.add_handler(CommandHandler("start", self.start_command))
        application.add_handler(CommandHandler("help", self.help_command))
        application.add_handler(CommandHandler("status", self.status_command))
        application.add_handler(CommandHandler("bookmarks", self.bookmarks_command))
        application.add_handler(CommandHandler("stats", self.stats_command))
        application.add_handler(CommandHandler("settings", self.settings_command))
        application.add_handler(CommandHandler("cleardata", self.cleardata_command))
        application.add_handler(CommandHandler("batch", self.batch_scan_command))
        application.add_handler(CommandHandler("deepscan", self.deep_scan_command))
        application.add_handler(CommandHandler("export", self.export_command))
        application.add_handler(CommandHandler("cancel", self.cancel_command))
        
        # Handle keyboard buttons (updated regex)
        application.add_handler(MessageHandler(
            filters.Regex('^(📚 Bookmarks|📊 Stats|📦 Batch Scan|🔬 Deep Scan|📤 Export|🔴 Status|❓ Help)$'),
            self.handle_keyboard_button
        ))
        
        # Handle URLs
        application.add_handler(MessageHandler(
            filters.TEXT & ~filters.COMMAND,
            self.handle_url
        ))
        
        # Handle callbacks
        application.add_handler(CallbackQueryHandler(self.handle_callback))
        
        # Handle WebApp data
        application.add_handler(MessageHandler(
            filters.StatusUpdate.WEB_APP_DATA,
            self.handle_webapp_data
        ))
        
        # Start bot
        logger.info("Starting bot...")
        
        # Create event loop for Python 3.14+
        try:
            loop = asyncio.get_event_loop()
        except RuntimeError:
            loop = asyncio.new_event_loop()
            asyncio.set_event_loop(loop)
        
        application.run_polling(allowed_updates=Update.ALL_TYPES)


def main():
    """Main entry point"""
    bot = APIBot()
    bot.run()


if __name__ == '__main__':
    main()