adds db config page and an option to add test papers
This commit is contained in:
Michael Beck
parent
36ba835980
commit
f42be483d6
@ -10,6 +10,11 @@ from scipaperloader.scrapers import __path__ as scrapers_path
|
||||
# Import the cache invalidation function from our new module
|
||||
from ..cache_utils import invalidate_hourly_quota_cache
|
||||
|
||||
import random
|
||||
from datetime import datetime, timedelta
|
||||
from uuid import uuid4
|
||||
|
||||
|
||||
bp = Blueprint("config", __name__, url_prefix="/config")
|
||||
|
||||
|
||||
@ -248,12 +253,108 @@ def schedule():
|
||||
app_title="Configuration"
|
||||
)
|
||||
|
||||
@bp.route("/database")
|
||||
def database():
|
||||
"""Show database configuration page."""
|
||||
|
||||
# Remove old update_volume route
|
||||
# @bp.route("/update/volume", methods=["POST"])
|
||||
# def update_volume(): ...
|
||||
return render_template(
|
||||
"config/index.html.jinja",
|
||||
active_tab="database",
|
||||
app_title="Configuration"
|
||||
)
|
||||
|
||||
|
||||
@bp.route("/generate_test_papers", methods=["POST"])
|
||||
def generate_test_papers():
|
||||
"""Generate random test papers for the database."""
|
||||
try:
|
||||
# Get the requested number of papers (with validation)
|
||||
try:
|
||||
paper_count = int(request.form.get("paper_count", "100"))
|
||||
if paper_count < 1:
|
||||
paper_count = 1
|
||||
elif paper_count > 1000:
|
||||
paper_count = 1000
|
||||
except (ValueError, TypeError):
|
||||
paper_count = 100
|
||||
|
||||
# Get the download path for file paths
|
||||
download_path = DownloadPathConfig.get_path()
|
||||
|
||||
# Sample journal names for realistic test data
|
||||
journals = [
|
||||
"Nature", "Science", "Cell", "PNAS", "Journal of Biological Chemistry",
|
||||
"IEEE Transactions on Neural Networks", "Artificial Intelligence",
|
||||
"Machine Learning", "Neural Computation", "Journal of Machine Learning Research",
|
||||
"Journal of Artificial Intelligence Research", "Data Mining and Knowledge Discovery",
|
||||
"Pattern Recognition", "Neural Networks", "Journal of Physical Chemistry"
|
||||
]
|
||||
|
||||
# Sample paper types
|
||||
paper_types = ["Article", "Review", "Conference", "Preprint", "Book Chapter"]
|
||||
|
||||
# Sample languages
|
||||
languages = ["English", "German", "French", "Chinese", "Spanish", "Japanese"]
|
||||
|
||||
# Generate random papers
|
||||
papers_added = 0
|
||||
for i in range(paper_count):
|
||||
# Generate a random DOI
|
||||
doi = f"10.{random.randint(1000, 9999)}/{uuid4().hex[:8]}"
|
||||
|
||||
# Skip if DOI already exists
|
||||
if PaperMetadata.query.filter_by(doi=doi).first():
|
||||
continue
|
||||
|
||||
# Random publishing date within the last 5 years
|
||||
days_ago = random.randint(0, 5 * 365)
|
||||
pub_date = datetime.now() - timedelta(days=days_ago)
|
||||
|
||||
# Create paper
|
||||
paper = PaperMetadata(
|
||||
title=f"Test Paper {i+1}: {''.join(random.choice('ABCDEFGHIJKLMNOPQRSTUVWXYZ') for _ in range(5))}",
|
||||
doi=doi,
|
||||
alt_id=f"ALT-{random.randint(10000, 99999)}",
|
||||
issn=f"{random.randint(1000, 9999)}-{random.randint(1000, 9999)}",
|
||||
journal=random.choice(journals),
|
||||
type=random.choice(paper_types),
|
||||
language=random.choice(languages),
|
||||
published_online=pub_date.date(),
|
||||
status=random.choice(["Pending", "Done", "Failed"]),
|
||||
file_path=f"{download_path}/test_paper_{i+1}.pdf" if random.random() > 0.3 else None,
|
||||
error_msg="Download failed: connection timeout" if random.random() < 0.1 else None,
|
||||
created_at=datetime.now() - timedelta(days=random.randint(0, 30))
|
||||
)
|
||||
db.session.add(paper)
|
||||
papers_added += 1
|
||||
|
||||
# Commit in batches to improve performance
|
||||
if i % 100 == 0:
|
||||
db.session.commit()
|
||||
|
||||
# Final commit
|
||||
db.session.commit()
|
||||
|
||||
# Log the action using the existing log_import_activity method
|
||||
ActivityLog.log_import_activity(
|
||||
action="generate_test_papers",
|
||||
status="success",
|
||||
description=f"Generated {papers_added} test papers for the database"
|
||||
)
|
||||
|
||||
flash(f"Successfully generated {papers_added} test papers.", "success")
|
||||
|
||||
except Exception as e:
|
||||
db.session.rollback()
|
||||
flash(f"Failed to generate test papers: {str(e)}", "error")
|
||||
ActivityLog.log_error(
|
||||
error_message=f"Failed to generate test papers: {str(e)}",
|
||||
exception=e,
|
||||
source="config.generate_test_papers"
|
||||
)
|
||||
|
||||
return redirect(url_for("config.database"))
|
||||
|
||||
# Add new route to handle general settings form
|
||||
@bp.route("/update/general", methods=["POST"])
|
||||
def update_general():
|
||||
"""Update general configuration (Volume and Download Path)."""
|
||||
|
||||
76
scipaperloader/templates/config/database.html.jinja
Normal file
76
scipaperloader/templates/config/database.html.jinja
Normal file
@ -0,0 +1,76 @@
|
||||
<!-- General Configuration Tab -->
|
||||
<div class="tab-pane active">
|
||||
<div class="config-form">
|
||||
<div class="card">
|
||||
<div class="card-header">
|
||||
<h5>Database Configuration</h5>
|
||||
</div>
|
||||
<div class="card-body">
|
||||
<!-- include flash messages template -->
|
||||
{% include "partials/flash_messages.html.jinja" %}
|
||||
|
||||
<!-- Generate Test Papers Section -->
|
||||
<div class="row mt-4">
|
||||
<div class="col-12">
|
||||
<div class="card border-primary">
|
||||
<div class="card-header bg-primary text-white">
|
||||
<h5>Generate Test Papers</h5>
|
||||
</div>
|
||||
<div class="card-body">
|
||||
<div class="form-section">
|
||||
<h6>Add Test Papers for Testing</h6>
|
||||
<p class="text-muted">Generate random test papers to populate your database for
|
||||
testing purposes.</p>
|
||||
|
||||
<form method="post" action="{{ url_for('config.generate_test_papers') }}"
|
||||
class="mt-3">
|
||||
<div class="form-group row">
|
||||
<label for="paper_count" class="col-sm-3 col-form-label">Number of
|
||||
Papers:</label>
|
||||
<div class="col-sm-4">
|
||||
<input type="number" class="form-control" id="paper_count"
|
||||
name="paper_count" min="1" max="1000" value="100" required>
|
||||
<small class="form-text text-muted">Enter a number between 1 and
|
||||
1000</small>
|
||||
</div>
|
||||
<div class="col-sm-5">
|
||||
<button type="submit" class="btn btn-primary">
|
||||
<i class="fas fa-plus-circle"></i> Generate Test Papers
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Database Management Section -->
|
||||
<div class="row mt-4">
|
||||
<div class="col-12">
|
||||
<div class="card border-danger">
|
||||
<div class="card-header bg-danger text-white">
|
||||
<h5>Database Management</h5>
|
||||
</div>
|
||||
<div class="card-body">
|
||||
<div class="form-section">
|
||||
<h6>Delete All Papers</h6>
|
||||
<p class="text-muted">This action will permanently delete all paper records from the
|
||||
database. This cannot be undone.</p>
|
||||
|
||||
<form method="post" action="{{ url_for('config.delete_all_papers') }}" class="mt-3"
|
||||
onsubmit="return confirm('WARNING: You are about to delete ALL papers from the database. This action cannot be undone. Are you sure you want to proceed?');">
|
||||
<button type="submit" class="btn btn-danger">
|
||||
<i class="fas fa-trash-alt"></i> Delete All Papers
|
||||
</button>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@ -91,30 +91,6 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Database Management Section -->
|
||||
<div class="row mt-4">
|
||||
<div class="col-12">
|
||||
<div class="card border-danger">
|
||||
<div class="card-header bg-danger text-white">
|
||||
<h5>Database Management</h5>
|
||||
</div>
|
||||
<div class="card-body">
|
||||
<div class="form-section">
|
||||
<h6>Delete All Papers</h6>
|
||||
<p class="text-muted">This action will permanently delete all paper records from the
|
||||
database. This cannot be undone.</p>
|
||||
|
||||
<form method="post" action="{{ url_for('config.delete_all_papers') }}" class="mt-3"
|
||||
onsubmit="return confirm('WARNING: You are about to delete ALL papers from the database. This action cannot be undone. Are you sure you want to proceed?');">
|
||||
<button type="submit" class="btn btn-danger">
|
||||
<i class="fas fa-trash-alt"></i> Delete All Papers
|
||||
</button>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@ -37,6 +37,10 @@
|
||||
<a class="nav-link {% if active_tab == 'schedule' %}active{% endif %}"
|
||||
href="{{ url_for('config.schedule') }}">Schedule</a>
|
||||
</li>
|
||||
<li class="nav-item">
|
||||
<a class="nav-link {% if active_tab == 'database' %}active{% endif %}"
|
||||
href="{{ url_for('config.database') }}">Schedule</a>
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
<div class="tab-content">
|
||||
@ -44,6 +48,8 @@
|
||||
{% include "config/general.html.jinja" %}
|
||||
{% elif active_tab == 'schedule' %}
|
||||
{% include "config/schedule.html.jinja" %}
|
||||
{% elif active_tab == 'database' %}
|
||||
{% include "config/database.html.jinja" %}
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user