adds db config page and an option to add test papers

scipaperloader/blueprints/config.py

@@ -10,6 +10,11 @@ from scipaperloader.scrapers import __path__ as scrapers_path
 # Import the cache invalidation function from our new module
 from ..cache_utils import invalidate_hourly_quota_cache
 
+import random
+from datetime import datetime, timedelta
+from uuid import uuid4
+
+
 bp = Blueprint("config", __name__, url_prefix="/config")
 
 
@@ -248,12 +253,108 @@ def schedule():
         app_title="Configuration"
     )
 
+@bp.route("/database")
+def database():
+    """Show database configuration page."""
 
-# Remove old update_volume route
+    return render_template(
-# @bp.route("/update/volume", methods=["POST"])
+        "config/index.html.jinja",
-# def update_volume(): ...
+        active_tab="database",
+        app_title="Configuration"
+    )
+
+
+@bp.route("/generate_test_papers", methods=["POST"])
+def generate_test_papers():
+    """Generate random test papers for the database."""
+    try:
+        # Get the requested number of papers (with validation)
+        try:
+            paper_count = int(request.form.get("paper_count", "100"))
+            if paper_count < 1:
+                paper_count = 1
+            elif paper_count > 1000:
+                paper_count = 1000
+        except (ValueError, TypeError):
+            paper_count = 100
+            
+        # Get the download path for file paths
+        download_path = DownloadPathConfig.get_path()
+        
+        # Sample journal names for realistic test data
+        journals = [
+            "Nature", "Science", "Cell", "PNAS", "Journal of Biological Chemistry",
+            "IEEE Transactions on Neural Networks", "Artificial Intelligence",
+            "Machine Learning", "Neural Computation", "Journal of Machine Learning Research",
+            "Journal of Artificial Intelligence Research", "Data Mining and Knowledge Discovery",
+            "Pattern Recognition", "Neural Networks", "Journal of Physical Chemistry"
+        ]
+        
+        # Sample paper types
+        paper_types = ["Article", "Review", "Conference", "Preprint", "Book Chapter"]
+        
+        # Sample languages
+        languages = ["English", "German", "French", "Chinese", "Spanish", "Japanese"]
+        
+        # Generate random papers
+        papers_added = 0
+        for i in range(paper_count):
+            # Generate a random DOI
+            doi = f"10.{random.randint(1000, 9999)}/{uuid4().hex[:8]}"
+            
+            # Skip if DOI already exists
+            if PaperMetadata.query.filter_by(doi=doi).first():
+                continue
+                
+            # Random publishing date within the last 5 years
+            days_ago = random.randint(0, 5 * 365)
+            pub_date = datetime.now() - timedelta(days=days_ago)
+            
+            # Create paper
+            paper = PaperMetadata(
+                title=f"Test Paper {i+1}: {''.join(random.choice('ABCDEFGHIJKLMNOPQRSTUVWXYZ') for _ in range(5))}",
+                doi=doi,
+                alt_id=f"ALT-{random.randint(10000, 99999)}",
+                issn=f"{random.randint(1000, 9999)}-{random.randint(1000, 9999)}",
+                journal=random.choice(journals),
+                type=random.choice(paper_types),
+                language=random.choice(languages),
+                published_online=pub_date.date(),
+                status=random.choice(["Pending", "Done", "Failed"]),
+                file_path=f"{download_path}/test_paper_{i+1}.pdf" if random.random() > 0.3 else None,
+                error_msg="Download failed: connection timeout" if random.random() < 0.1 else None,
+                created_at=datetime.now() - timedelta(days=random.randint(0, 30))
+            )
+            db.session.add(paper)
+            papers_added += 1
+            
+            # Commit in batches to improve performance
+            if i % 100 == 0:
+                db.session.commit()
+        
+        # Final commit
+        db.session.commit()
+        
+        # Log the action using the existing log_import_activity method
+        ActivityLog.log_import_activity(
+            action="generate_test_papers",
+            status="success",
+            description=f"Generated {papers_added} test papers for the database"
+        )
+        
+        flash(f"Successfully generated {papers_added} test papers.", "success")
+    
+    except Exception as e:
+        db.session.rollback()
+        flash(f"Failed to generate test papers: {str(e)}", "error")
+        ActivityLog.log_error(
+            error_message=f"Failed to generate test papers: {str(e)}",
+            exception=e,
+            source="config.generate_test_papers"
+        )
+    
+    return redirect(url_for("config.database"))
 
-# Add new route to handle general settings form
 @bp.route("/update/general", methods=["POST"])
 def update_general():
     """Update general configuration (Volume and Download Path)."""

scipaperloader/templates/config/database.html.jinja

@@ -0,0 +1,76 @@
+<!-- General Configuration Tab -->
+<div class="tab-pane active">
+    <div class="config-form">
+        <div class="card">
+            <div class="card-header">
+                <h5>Database Configuration</h5>
+            </div>
+            <div class="card-body">
+                <!-- include flash messages template -->
+                {% include "partials/flash_messages.html.jinja" %}
+
+                <!-- Generate Test Papers Section -->
+                <div class="row mt-4">
+                    <div class="col-12">
+                        <div class="card border-primary">
+                            <div class="card-header bg-primary text-white">
+                                <h5>Generate Test Papers</h5>
+                            </div>
+                            <div class="card-body">
+                                <div class="form-section">
+                                    <h6>Add Test Papers for Testing</h6>
+                                    <p class="text-muted">Generate random test papers to populate your database for
+                                        testing purposes.</p>
+
+                                    <form method="post" action="{{ url_for('config.generate_test_papers') }}"
+                                        class="mt-3">
+                                        <div class="form-group row">
+                                            <label for="paper_count" class="col-sm-3 col-form-label">Number of
+                                                Papers:</label>
+                                            <div class="col-sm-4">
+                                                <input type="number" class="form-control" id="paper_count"
+                                                    name="paper_count" min="1" max="1000" value="100" required>
+                                                <small class="form-text text-muted">Enter a number between 1 and
+                                                    1000</small>
+                                            </div>
+                                            <div class="col-sm-5">
+                                                <button type="submit" class="btn btn-primary">
+                                                    <i class="fas fa-plus-circle"></i> Generate Test Papers
+                                                </button>
+                                            </div>
+                                        </div>
+                                    </form>
+                                </div>
+                            </div>
+                        </div>
+                    </div>
+                </div>
+
+                <!-- Database Management Section -->
+                <div class="row mt-4">
+                    <div class="col-12">
+                        <div class="card border-danger">
+                            <div class="card-header bg-danger text-white">
+                                <h5>Database Management</h5>
+                            </div>
+                            <div class="card-body">
+                                <div class="form-section">
+                                    <h6>Delete All Papers</h6>
+                                    <p class="text-muted">This action will permanently delete all paper records from the
+                                        database. This cannot be undone.</p>
+
+                                    <form method="post" action="{{ url_for('config.delete_all_papers') }}" class="mt-3"
+                                        onsubmit="return confirm('WARNING: You are about to delete ALL papers from the database. This action cannot be undone. Are you sure you want to proceed?');">
+                                        <button type="submit" class="btn btn-danger">
+                                            <i class="fas fa-trash-alt"></i> Delete All Papers
+                                        </button>
+                                    </form>
+                                </div>
+                            </div>
+                        </div>
+                    </div>
+                </div>
+            </div>
+        </div>
+    </div>
+</div>

scipaperloader/templates/config/general.html.jinja

@@ -91,30 +91,6 @@
                     </div>
                 </div>
 
-                <!-- Database Management Section -->
-                <div class="row mt-4">
-                    <div class="col-12">
-                        <div class="card border-danger">
-                            <div class="card-header bg-danger text-white">
-                                <h5>Database Management</h5>
-                            </div>
-                            <div class="card-body">
-                                <div class="form-section">
-                                    <h6>Delete All Papers</h6>
-                                    <p class="text-muted">This action will permanently delete all paper records from the
-                                        database. This cannot be undone.</p>
-
-                                    <form method="post" action="{{ url_for('config.delete_all_papers') }}" class="mt-3"
-                                        onsubmit="return confirm('WARNING: You are about to delete ALL papers from the database. This action cannot be undone. Are you sure you want to proceed?');">
-                                        <button type="submit" class="btn btn-danger">
-                                            <i class="fas fa-trash-alt"></i> Delete All Papers
-                                        </button>
-                                    </form>
-                                </div>
-                            </div>
-                        </div>
-                    </div>
-                </div>
             </div>
         </div>
     </div>

scipaperloader/templates/config/index.html.jinja

@@ -37,6 +37,10 @@
             <a class="nav-link {% if active_tab == 'schedule' %}active{% endif %}"
                 href="{{ url_for('config.schedule') }}">Schedule</a>
         </li>
+        <li class="nav-item">
+            <a class="nav-link {% if active_tab == 'database' %}active{% endif %}"
+                href="{{ url_for('config.database') }}">Schedule</a>
+        </li>
     </ul>
 
     <div class="tab-content">
@@ -44,6 +48,8 @@
         {% include "config/general.html.jinja" %}
         {% elif active_tab == 'schedule' %}
         {% include "config/schedule.html.jinja" %}
+        {% elif active_tab == 'database' %}
+        {% include "config/database.html.jinja" %}
         {% endif %}
     </div>
 </div>

scipaperloader/templates/scraper.html.jinja

@@ -433,7 +433,7 @@
 
                     // Show notification
                     showFlashMessage(data.message, 'success');
-                    
+
                     // Set up polling to check paper status and refresh activity
                     pollPaperStatus(paperId, 3000, 20);
                 } else {
@@ -615,14 +615,14 @@
     // Poll paper status until it changes from Pending
     function pollPaperStatus(paperId, interval = 3000, maxAttempts = 20) {
         let attempts = 0;
-        
+
         // Immediately refresh activity log to show the initial pending status
         loadRecentActivity();
-        
+
         const checkStatus = () => {
             attempts++;
             console.log(`Checking status of paper ${paperId}, attempt ${attempts}/${maxAttempts}`);
-            
+
             // Fetch the current paper status
             fetch(`/api/papers/${paperId}`)
                 .then(response => response.json())
@@ -630,13 +630,13 @@
                     if (data && data.paper) {
                         const paper = data.paper;
                         console.log(`Paper status: ${paper.status}`);
-                        
+
                         // Update the UI with the current status
                         const row = document.querySelector(`.process-paper-btn[data-paper-id="${paperId}"]`).closest('tr');
                         if (row) {
                             const statusCell = row.querySelector('td:nth-child(4)');
                             let statusBadge = '';
-                            
+
                             if (paper.status === 'New') {
                                 statusBadge = '<span class="badge bg-info">New</span>';
                             } else if (paper.status === 'Pending') {
@@ -648,9 +648,9 @@
                             } else {
                                 statusBadge = `<span class="badge bg-secondary">${paper.status}</span>`;
                             }
-                            
+
                             statusCell.innerHTML = statusBadge;
-                            
+
                             // Update processing status message if status changed
                             if (paper.status !== 'Pending') {
                                 if (paper.status === 'Done') {
@@ -662,26 +662,26 @@
                                 }
                             }
                         }
-                        
+
                         // Always refresh activity log
                         loadRecentActivity();
-                        
+
                         // If status is still pending and we haven't reached max attempts, check again
                         if (paper.status === 'Pending' && attempts < maxAttempts) {
                             setTimeout(checkStatus, interval);
                         } else {
                             // If status changed or we reached max attempts, refresh chart data too
                             loadActivityStats(currentTimeRange);
-                            
+
                             // Show notification if status changed
                             if (paper.status !== 'Pending') {
                                 const status = paper.status === 'Done' ? 'success' : 'error';
-                                const message = paper.status === 'Done' 
+                                const message = paper.status === 'Done'
                                     ? `Paper processed successfully: ${paper.title}`
                                     : `Paper processing failed: ${paper.error_msg || 'Unknown error'}`;
                                 showFlashMessage(message, status);
                             }
-                            
+
                             // If we hit max attempts but status is still pending, show a message
                             if (paper.status === 'Pending' && attempts >= maxAttempts) {
                                 processingStatus.textContent = 'Paper is still being processed. Check the activity log for updates.';
@@ -698,7 +698,7 @@
                     }
                 });
         };
-        
+
         // Start checking
         setTimeout(checkStatus, interval);
     }