121 lines
4.2 KiB
Python
Executable File
121 lines
4.2 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
Emergency force stop utility for the scraper.
|
|
|
|
This script will:
|
|
1. Set the scraper state to inactive
|
|
2. Revoke all running/scheduled tasks
|
|
3. Purge task queues
|
|
4. Revert any papers in "Pending" state to their previous status
|
|
|
|
Use this to recover from a misbehaving scraper or when the web UI is unresponsive.
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
import time
|
|
from datetime import datetime
|
|
|
|
# Add project root to path
|
|
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')))
|
|
|
|
# Import required modules
|
|
from scipaperloader import create_app
|
|
from scipaperloader.db import db
|
|
from scipaperloader.models import PaperMetadata, ActivityLog, ScraperState
|
|
from scipaperloader.celery import celery
|
|
|
|
app = create_app()
|
|
|
|
def emergency_stop():
|
|
"""Force stop the scraper and revert all pending papers"""
|
|
with app.app_context():
|
|
print("Emergency Scraper Stop")
|
|
print("-" * 50)
|
|
|
|
# 1. Set scraper state to inactive
|
|
ScraperState.set_active(False)
|
|
ScraperState.set_paused(False)
|
|
print("✓ Set scraper state to inactive")
|
|
|
|
# 2. Revoke all tasks
|
|
print("\nRevoking running tasks...")
|
|
try:
|
|
i = celery.control.inspect()
|
|
active = i.active() or {}
|
|
scheduled = i.scheduled() or {}
|
|
reserved = i.reserved() or {}
|
|
|
|
revoked_count = 0
|
|
|
|
# Revoke active tasks
|
|
for worker, tasks in active.items():
|
|
for task in tasks:
|
|
if 'id' in task:
|
|
celery.control.revoke(task['id'], terminate=True)
|
|
revoked_count += 1
|
|
print(f" Revoked active task: {task.get('name', 'unknown')}")
|
|
|
|
# Revoke scheduled tasks
|
|
for worker, tasks in scheduled.items():
|
|
for task in tasks:
|
|
if 'id' in task:
|
|
celery.control.revoke(task['id'], terminate=True)
|
|
revoked_count += 1
|
|
|
|
# Revoke reserved tasks
|
|
for worker, tasks in reserved.items():
|
|
for task in tasks:
|
|
if 'id' in task:
|
|
celery.control.revoke(task['id'], terminate=True)
|
|
revoked_count += 1
|
|
|
|
print(f"✓ Revoked {revoked_count} tasks")
|
|
|
|
# 3. Purge queues
|
|
celery.control.purge()
|
|
print("✓ Purged all task queues")
|
|
|
|
except Exception as e:
|
|
print(f"⚠ Error revoking tasks: {str(e)}")
|
|
|
|
# 4. Revert papers in "Pending" status
|
|
try:
|
|
print("\nReverting papers from 'Pending' status...")
|
|
pending_papers = PaperMetadata.query.filter_by(status="Pending").all()
|
|
reverted_count = 0
|
|
|
|
for paper in pending_papers:
|
|
# Get previous status or use "New" as fallback
|
|
previous_status = paper.previous_status if hasattr(paper, 'previous_status') and paper.previous_status else "New"
|
|
paper.status = previous_status
|
|
|
|
ActivityLog.log_scraper_activity(
|
|
action="emergency_revert",
|
|
paper_id=paper.id,
|
|
status="info",
|
|
description=f"Emergency reversion from 'Pending' to '{previous_status}'",
|
|
)
|
|
reverted_count += 1
|
|
print(f" Reverted paper ID {paper.id}: {paper.title} -> {previous_status}")
|
|
|
|
# Commit changes
|
|
db.session.commit()
|
|
print(f"✓ Reverted {reverted_count} papers")
|
|
|
|
ActivityLog.log_scraper_command(
|
|
action="emergency_stop",
|
|
status="success",
|
|
description=f"Emergency stop performed. Revoked {revoked_count} tasks and reverted {reverted_count} papers."
|
|
)
|
|
|
|
except Exception as e:
|
|
db.session.rollback()
|
|
print(f"⚠ Error reverting papers: {str(e)}")
|
|
|
|
print("\nEmergency stop completed!")
|
|
print(f"Current time: {datetime.now()}")
|
|
|
|
if __name__ == "__main__":
|
|
emergency_stop()
|