diff --git a/scipaperloader/blueprints/config.py b/scipaperloader/blueprints/config.py index 4a69c5e..76d9511 100644 --- a/scipaperloader/blueprints/config.py +++ b/scipaperloader/blueprints/config.py @@ -10,6 +10,11 @@ from scipaperloader.scrapers import __path__ as scrapers_path # Import the cache invalidation function from our new module from ..cache_utils import invalidate_hourly_quota_cache +import random +from datetime import datetime, timedelta +from uuid import uuid4 + + bp = Blueprint("config", __name__, url_prefix="/config") @@ -248,12 +253,108 @@ def schedule(): app_title="Configuration" ) +@bp.route("/database") +def database(): + """Show database configuration page.""" -# Remove old update_volume route -# @bp.route("/update/volume", methods=["POST"]) -# def update_volume(): ... + return render_template( + "config/index.html.jinja", + active_tab="database", + app_title="Configuration" + ) + + +@bp.route("/generate_test_papers", methods=["POST"]) +def generate_test_papers(): + """Generate random test papers for the database.""" + try: + # Get the requested number of papers (with validation) + try: + paper_count = int(request.form.get("paper_count", "100")) + if paper_count < 1: + paper_count = 1 + elif paper_count > 1000: + paper_count = 1000 + except (ValueError, TypeError): + paper_count = 100 + + # Get the download path for file paths + download_path = DownloadPathConfig.get_path() + + # Sample journal names for realistic test data + journals = [ + "Nature", "Science", "Cell", "PNAS", "Journal of Biological Chemistry", + "IEEE Transactions on Neural Networks", "Artificial Intelligence", + "Machine Learning", "Neural Computation", "Journal of Machine Learning Research", + "Journal of Artificial Intelligence Research", "Data Mining and Knowledge Discovery", + "Pattern Recognition", "Neural Networks", "Journal of Physical Chemistry" + ] + + # Sample paper types + paper_types = ["Article", "Review", "Conference", "Preprint", "Book Chapter"] + + # Sample languages + languages = ["English", "German", "French", "Chinese", "Spanish", "Japanese"] + + # Generate random papers + papers_added = 0 + for i in range(paper_count): + # Generate a random DOI + doi = f"10.{random.randint(1000, 9999)}/{uuid4().hex[:8]}" + + # Skip if DOI already exists + if PaperMetadata.query.filter_by(doi=doi).first(): + continue + + # Random publishing date within the last 5 years + days_ago = random.randint(0, 5 * 365) + pub_date = datetime.now() - timedelta(days=days_ago) + + # Create paper + paper = PaperMetadata( + title=f"Test Paper {i+1}: {''.join(random.choice('ABCDEFGHIJKLMNOPQRSTUVWXYZ') for _ in range(5))}", + doi=doi, + alt_id=f"ALT-{random.randint(10000, 99999)}", + issn=f"{random.randint(1000, 9999)}-{random.randint(1000, 9999)}", + journal=random.choice(journals), + type=random.choice(paper_types), + language=random.choice(languages), + published_online=pub_date.date(), + status=random.choice(["Pending", "Done", "Failed"]), + file_path=f"{download_path}/test_paper_{i+1}.pdf" if random.random() > 0.3 else None, + error_msg="Download failed: connection timeout" if random.random() < 0.1 else None, + created_at=datetime.now() - timedelta(days=random.randint(0, 30)) + ) + db.session.add(paper) + papers_added += 1 + + # Commit in batches to improve performance + if i % 100 == 0: + db.session.commit() + + # Final commit + db.session.commit() + + # Log the action using the existing log_import_activity method + ActivityLog.log_import_activity( + action="generate_test_papers", + status="success", + description=f"Generated {papers_added} test papers for the database" + ) + + flash(f"Successfully generated {papers_added} test papers.", "success") + + except Exception as e: + db.session.rollback() + flash(f"Failed to generate test papers: {str(e)}", "error") + ActivityLog.log_error( + error_message=f"Failed to generate test papers: {str(e)}", + exception=e, + source="config.generate_test_papers" + ) + + return redirect(url_for("config.database")) -# Add new route to handle general settings form @bp.route("/update/general", methods=["POST"]) def update_general(): """Update general configuration (Volume and Download Path).""" diff --git a/scipaperloader/templates/config/database.html.jinja b/scipaperloader/templates/config/database.html.jinja new file mode 100644 index 0000000..dd7e484 --- /dev/null +++ b/scipaperloader/templates/config/database.html.jinja @@ -0,0 +1,76 @@ + +
+
+
+
+
Database Configuration
+
+
+ + {% include "partials/flash_messages.html.jinja" %} + + +
+
+
+
+
Generate Test Papers
+
+
+
+
Add Test Papers for Testing
+

Generate random test papers to populate your database for + testing purposes.

+ +
+
+ +
+ + Enter a number between 1 and + 1000 +
+
+ +
+
+
+
+
+
+
+
+ + +
+
+
+
+
Database Management
+
+
+
+
Delete All Papers
+

This action will permanently delete all paper records from the + database. This cannot be undone.

+ +
+ +
+
+
+
+
+
+
+
+
+
\ No newline at end of file diff --git a/scipaperloader/templates/config/general.html.jinja b/scipaperloader/templates/config/general.html.jinja index 24324ce..fca82f0 100644 --- a/scipaperloader/templates/config/general.html.jinja +++ b/scipaperloader/templates/config/general.html.jinja @@ -91,30 +91,6 @@ - -
-
-
-
-
Database Management
-
-
-
-
Delete All Papers
-

This action will permanently delete all paper records from the - database. This cannot be undone.

- -
- -
-
-
-
-
-
diff --git a/scipaperloader/templates/config/index.html.jinja b/scipaperloader/templates/config/index.html.jinja index 0fa3d93..a4f767b 100644 --- a/scipaperloader/templates/config/index.html.jinja +++ b/scipaperloader/templates/config/index.html.jinja @@ -37,6 +37,10 @@ Schedule +
  • + Schedule +
    • @@ -44,6 +48,8 @@ {% include "config/general.html.jinja" %} {% elif active_tab == 'schedule' %} {% include "config/schedule.html.jinja" %} + {% elif active_tab == 'database' %} + {% include "config/database.html.jinja" %} {% endif %}
    diff --git a/scipaperloader/templates/scraper.html.jinja b/scipaperloader/templates/scraper.html.jinja index 388a7ee..a457c4e 100644 --- a/scipaperloader/templates/scraper.html.jinja +++ b/scipaperloader/templates/scraper.html.jinja @@ -433,7 +433,7 @@ // Show notification showFlashMessage(data.message, 'success'); - + // Set up polling to check paper status and refresh activity pollPaperStatus(paperId, 3000, 20); } else { @@ -615,14 +615,14 @@ // Poll paper status until it changes from Pending function pollPaperStatus(paperId, interval = 3000, maxAttempts = 20) { let attempts = 0; - + // Immediately refresh activity log to show the initial pending status loadRecentActivity(); - + const checkStatus = () => { attempts++; console.log(`Checking status of paper ${paperId}, attempt ${attempts}/${maxAttempts}`); - + // Fetch the current paper status fetch(`/api/papers/${paperId}`) .then(response => response.json()) @@ -630,13 +630,13 @@ if (data && data.paper) { const paper = data.paper; console.log(`Paper status: ${paper.status}`); - + // Update the UI with the current status const row = document.querySelector(`.process-paper-btn[data-paper-id="${paperId}"]`).closest('tr'); if (row) { const statusCell = row.querySelector('td:nth-child(4)'); let statusBadge = ''; - + if (paper.status === 'New') { statusBadge = 'New'; } else if (paper.status === 'Pending') { @@ -648,9 +648,9 @@ } else { statusBadge = `${paper.status}`; } - + statusCell.innerHTML = statusBadge; - + // Update processing status message if status changed if (paper.status !== 'Pending') { if (paper.status === 'Done') { @@ -662,26 +662,26 @@ } } } - + // Always refresh activity log loadRecentActivity(); - + // If status is still pending and we haven't reached max attempts, check again if (paper.status === 'Pending' && attempts < maxAttempts) { setTimeout(checkStatus, interval); } else { // If status changed or we reached max attempts, refresh chart data too loadActivityStats(currentTimeRange); - + // Show notification if status changed if (paper.status !== 'Pending') { const status = paper.status === 'Done' ? 'success' : 'error'; - const message = paper.status === 'Done' + const message = paper.status === 'Done' ? `Paper processed successfully: ${paper.title}` : `Paper processing failed: ${paper.error_msg || 'Unknown error'}`; showFlashMessage(message, status); } - + // If we hit max attempts but status is still pending, show a message if (paper.status === 'Pending' && attempts >= maxAttempts) { processingStatus.textContent = 'Paper is still being processed. Check the activity log for updates.'; @@ -698,7 +698,7 @@ } }); }; - + // Start checking setTimeout(checkStatus, interval); }