mischbeck/SciPaperLoader
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

adds db config page and an option to add test papers

Browse Source
This commit is contained in:
Michael Beck 2025-05-23 20:03:39 +02:00
parent 36ba835980
commit f42be483d6
5 changed files with 201 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

109
scipaperloader/blueprints/config.py
View File

@ -10,6 +10,11 @@ from scipaperloader.scrapers import __path__ as scrapers_path
# Import the cache invalidation function from our new module
from ..cache_utils import invalidate_hourly_quota_cache
import random
from datetime import datetime, timedelta
from uuid import uuid4
bp = Blueprint("config", __name__, url_prefix="/config")
@ -248,12 +253,108 @@ def schedule():
app_title="Configuration"
)
@bp.route("/database")
def database():
"""Show database configuration page."""
# Remove old update_volume route
# @bp.route("/update/volume", methods=["POST"])
# def update_volume(): ...
return render_template(
"config/index.html.jinja",
active_tab="database",
app_title="Configuration"
)
@bp.route("/generate_test_papers", methods=["POST"])
def generate_test_papers():
"""Generate random test papers for the database."""
try:
# Get the requested number of papers (with validation)
try:
paper_count = int(request.form.get("paper_count", "100"))
if paper_count < 1:
paper_count = 1
elif paper_count > 1000:
paper_count = 1000
except (ValueError, TypeError):
paper_count = 100
# Get the download path for file paths
download_path = DownloadPathConfig.get_path()
# Sample journal names for realistic test data
journals = [
"Nature", "Science", "Cell", "PNAS", "Journal of Biological Chemistry",
"IEEE Transactions on Neural Networks", "Artificial Intelligence",
"Machine Learning", "Neural Computation", "Journal of Machine Learning Research",
"Journal of Artificial Intelligence Research", "Data Mining and Knowledge Discovery",
"Pattern Recognition", "Neural Networks", "Journal of Physical Chemistry"
]
# Sample paper types
paper_types = ["Article", "Review", "Conference", "Preprint", "Book Chapter"]
# Sample languages
languages = ["English", "German", "French", "Chinese", "Spanish", "Japanese"]
# Generate random papers
papers_added = 0
for i in range(paper_count):
# Generate a random DOI
doi = f"10.{random.randint(1000, 9999)}/{uuid4().hex[:8]}"
# Skip if DOI already exists
if PaperMetadata.query.filter_by(doi=doi).first():
continue
# Random publishing date within the last 5 years
days_ago = random.randint(0, 5 * 365)
pub_date = datetime.now() - timedelta(days=days_ago)
# Create paper
paper = PaperMetadata(
title=f"Test Paper {i+1}: {''.join(random.choice('ABCDEFGHIJKLMNOPQRSTUVWXYZ') for _ in range(5))}",
doi=doi,
alt_id=f"ALT-{random.randint(10000, 99999)}",
issn=f"{random.randint(1000, 9999)}-{random.randint(1000, 9999)}",
journal=random.choice(journals),
type=random.choice(paper_types),
language=random.choice(languages),
published_online=pub_date.date(),
status=random.choice(["Pending", "Done", "Failed"]),
file_path=f"{download_path}/test_paper_{i+1}.pdf" if random.random() > 0.3 else None,
error_msg="Download failed: connection timeout" if random.random() < 0.1 else None,
created_at=datetime.now() - timedelta(days=random.randint(0, 30))
)
db.session.add(paper)
papers_added += 1
# Commit in batches to improve performance
if i % 100 == 0:
db.session.commit()
# Final commit
db.session.commit()
# Log the action using the existing log_import_activity method
ActivityLog.log_import_activity(
action="generate_test_papers",
status="success",
description=f"Generated {papers_added} test papers for the database"
)
flash(f"Successfully generated {papers_added} test papers.", "success")
except Exception as e:
db.session.rollback()
flash(f"Failed to generate test papers: {str(e)}", "error")
ActivityLog.log_error(
error_message=f"Failed to generate test papers: {str(e)}",
exception=e,
source="config.generate_test_papers"
)
return redirect(url_for("config.database"))
# Add new route to handle general settings form
@bp.route("/update/general", methods=["POST"])
def update_general():
"""Update general configuration (Volume and Download Path)."""

76
scipaperloader/templates/config/database.html.jinja Normal file
View File

@ -0,0 +1,76 @@
<!-- General Configuration Tab -->
<div class="tab-pane active">
<div class="config-form">
<div class="card">
<div class="card-header">
<h5>Database Configuration</h5>
</div>
<div class="card-body">
<!-- include flash messages template -->
{% include "partials/flash_messages.html.jinja" %}
<!-- Generate Test Papers Section -->
<div class="row mt-4">
<div class="col-12">
<div class="card border-primary">
<div class="card-header bg-primary text-white">
<h5>Generate Test Papers</h5>
</div>
<div class="card-body">
<div class="form-section">
<h6>Add Test Papers for Testing</h6>
<p class="text-muted">Generate random test papers to populate your database for
testing purposes.</p>
<form method="post" action="{{ url_for('config.generate_test_papers') }}"
class="mt-3">
<div class="form-group row">
<label for="paper_count" class="col-sm-3 col-form-label">Number of
Papers:</label>
<div class="col-sm-4">
<input type="number" class="form-control" id="paper_count"
name="paper_count" min="1" max="1000" value="100" required>
<small class="form-text text-muted">Enter a number between 1 and
1000</small>
</div>
<div class="col-sm-5">
<button type="submit" class="btn btn-primary">
<i class="fas fa-plus-circle"></i> Generate Test Papers
</button>
</div>
</div>
</form>
</div>
</div>
</div>
</div>
</div>
<!-- Database Management Section -->
<div class="row mt-4">
<div class="col-12">
<div class="card border-danger">
<div class="card-header bg-danger text-white">
<h5>Database Management</h5>
</div>
<div class="card-body">
<div class="form-section">
<h6>Delete All Papers</h6>
<p class="text-muted">This action will permanently delete all paper records from the
database. This cannot be undone.</p>
<form method="post" action="{{ url_for('config.delete_all_papers') }}" class="mt-3"
onsubmit="return confirm('WARNING: You are about to delete ALL papers from the database. This action cannot be undone. Are you sure you want to proceed?');">
<button type="submit" class="btn btn-danger">
<i class="fas fa-trash-alt"></i> Delete All Papers
</button>
</form>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
</div>

24
scipaperloader/templates/config/general.html.jinja
View File

@ -91,30 +91,6 @@
</div>
</div>
<!-- Database Management Section -->
<div class="row mt-4">
<div class="col-12">
<div class="card border-danger">
<div class="card-header bg-danger text-white">
<h5>Database Management</h5>
</div>
<div class="card-body">
<div class="form-section">
<h6>Delete All Papers</h6>
<p class="text-muted">This action will permanently delete all paper records from the
database. This cannot be undone.</p>
<form method="post" action="{{ url_for('config.delete_all_papers') }}" class="mt-3"
onsubmit="return confirm('WARNING: You are about to delete ALL papers from the database. This action cannot be undone. Are you sure you want to proceed?');">
<button type="submit" class="btn btn-danger">
<i class="fas fa-trash-alt"></i> Delete All Papers
</button>
</form>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
</div>

6
scipaperloader/templates/config/index.html.jinja
View File

@ -37,6 +37,10 @@
<a class="nav-link {% if active_tab == 'schedule' %}active{% endif %}"
href="{{ url_for('config.schedule') }}">Schedule</a>
</li>
<li class="nav-item">
<a class="nav-link {% if active_tab == 'database' %}active{% endif %}"
href="{{ url_for('config.database') }}">Schedule</a>
</li>
</ul>
<div class="tab-content">
@ -44,6 +48,8 @@
{% include "config/general.html.jinja" %}
{% elif active_tab == 'schedule' %}
{% include "config/schedule.html.jinja" %}
{% elif active_tab == 'database' %}
{% include "config/database.html.jinja" %}
{% endif %}
</div>
</div>

28
scipaperloader/templates/scraper.html.jinja
View File

@ -433,7 +433,7 @@
// Show notification
showFlashMessage(data.message, 'success');
// Set up polling to check paper status and refresh activity
pollPaperStatus(paperId, 3000, 20);
} else {
@ -615,14 +615,14 @@
// Poll paper status until it changes from Pending
function pollPaperStatus(paperId, interval = 3000, maxAttempts = 20) {
let attempts = 0;
// Immediately refresh activity log to show the initial pending status
loadRecentActivity();
const checkStatus = () => {
attempts++;
console.log(`Checking status of paper ${paperId}, attempt ${attempts}/${maxAttempts}`);
// Fetch the current paper status
fetch(`/api/papers/${paperId}`)
.then(response => response.json())
@ -630,13 +630,13 @@
if (data && data.paper) {
const paper = data.paper;
console.log(`Paper status: ${paper.status}`);
// Update the UI with the current status
const row = document.querySelector(`.process-paper-btn[data-paper-id="${paperId}"]`).closest('tr');
if (row) {
const statusCell = row.querySelector('td:nth-child(4)');
let statusBadge = '';
if (paper.status === 'New') {
statusBadge = '<span class="badge bg-info">New</span>';
} else if (paper.status === 'Pending') {
@ -648,9 +648,9 @@
} else {
statusBadge = `<span class="badge bg-secondary">${paper.status}</span>`;
}
statusCell.innerHTML = statusBadge;
// Update processing status message if status changed
if (paper.status !== 'Pending') {
if (paper.status === 'Done') {
@ -662,26 +662,26 @@
}
}
}
// Always refresh activity log
loadRecentActivity();
// If status is still pending and we haven't reached max attempts, check again
if (paper.status === 'Pending' && attempts < maxAttempts) {
setTimeout(checkStatus, interval);
} else {
// If status changed or we reached max attempts, refresh chart data too
loadActivityStats(currentTimeRange);
// Show notification if status changed
if (paper.status !== 'Pending') {
const status = paper.status === 'Done' ? 'success' : 'error';
const message = paper.status === 'Done'
const message = paper.status === 'Done'
? `Paper processed successfully: ${paper.title}`
: `Paper processing failed: ${paper.error_msg || 'Unknown error'}`;
showFlashMessage(message, status);
}
// If we hit max attempts but status is still pending, show a message
if (paper.status === 'Pending' && attempts >= maxAttempts) {
processingStatus.textContent = 'Paper is still being processed. Check the activity log for updates.';
@ -698,7 +698,7 @@
}
});
};
// Start checking
setTimeout(checkStatus, interval);
}