""" Diagnose and fix scraper stopping issues. """ from scipaperloader import create_app from scipaperloader.models import ScraperState, ActivityLog from scipaperloader.scrapers.factory import get_scraper app = create_app() def check_scraper_status(): """Check the current status of the scraper in the database.""" with app.app_context(): scraper_state = ScraperState.query.first() if scraper_state: print(f"Scraper state in DB: active={scraper_state.is_active}, paused={scraper_state.is_paused}") else: print("No scraper state found in database") def check_scheduler_jobs(): """Check the current jobs in APScheduler.""" with app.app_context(): scheduler = app.config.get('SCHEDULER') if not scheduler: print("❌ APScheduler not found in app config") else: jobs = scheduler.get_paper_jobs() print("Scheduled jobs:", jobs) def check_recent_logs(): """Check recent activity logs for clues.""" with app.app_context(): logs = ActivityLog.query.filter_by(category='scraper_command').order_by(ActivityLog.timestamp.desc()).limit(5).all() print("\n=== RECENT COMMAND LOGS ===") for log in logs: print(f"[{log.timestamp}] {log.action}: {log.description}") activity_logs = ActivityLog.query.filter_by(category='scraper_activity').order_by(ActivityLog.timestamp.desc()).limit(5).all() print("\n=== RECENT ACTIVITY LOGS ===") for log in activity_logs: print(f"[{log.timestamp}] {log.action}: {log.description}") def force_stop_scraper(): """Force stop the scraper by setting the state and revoking all tasks.""" with app.app_context(): # Update scraper state scraper_state = ScraperState.query.first() if scraper_state: scraper_state.is_active = False scraper_state.is_paused = False from scipaperloader.db import db db.session.commit() print("Set scraper state to inactive") # Revoke all tasks scheduler = app.config.get('SCHEDULER') if not scheduler: print("❌ APScheduler not found in app config") else: revoked_count = scheduler.revoke_all_scraper_jobs() print(f"✅ Revoked {revoked_count} jobs from APScheduler") # Log the action ActivityLog.log_scraper_command( action="force_stop_scraper", status="success", description=f"Force stopped scraper, revoked {revoked_count} tasks" ) print(f"\nRevoked {revoked_count} tasks in total") if __name__ == "__main__": print("=== SCRAPER STATUS DIAGNOSTIC TOOL ===") check_scraper_status() check_scheduler_jobs() check_recent_logs() stop_confirmation = input("\nDo you want to force stop the scraper? (y/n): ") if stop_confirmation.lower() == 'y': force_stop_scraper() print("\nScraper force stopped. Current state:") check_scraper_status() else: print("No changes made.")