mischbeck/SciPaperLoader
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Finishes Scheduler View & Controller

Browse Source
This commit is contained in:
Michael Beck 2025-03-31 01:28:07 +02:00
parent 2868916cf6
commit 1534dbb0ba
4 changed files with 427 additions and 298 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
scipaperloader/templates/base.html
View File

@ -1,18 +1,22 @@
<!DOCTYPE html> <!DOCTYPE html>
<html lang="en"> <html lang="en">
<head> <head>
<meta charset="UTF-8"> <meta charset="UTF-8" />
<title>{{ app_title }}</title> <title>{{ app_title }}</title>
<link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.3/dist/css/bootstrap.min.css" rel="stylesheet"> <link
href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.3/dist/css/bootstrap.min.css"
rel="stylesheet"
/>
<script src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.3/dist/js/bootstrap.bundle.min.js"></script> <script src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.3/dist/js/bootstrap.bundle.min.js"></script>
<!-- Optional Alpine.js --> <!-- Optional Alpine.js -->
<script defer src="https://cdn.jsdelivr.net/npm/alpinejs@3.x.x/dist/cdn.min.js"></script> <script
</head> defer
<body> src="https://cdn.jsdelivr.net/npm/alpinejs@3.x.x/dist/cdn.min.js"
{% include 'nav.html' %} ></script>
<div class="container py-4"> </head>
{% block content %}{% endblock %} <body>
</div> {% include 'nav.html' %}
{% include 'footer.html' %} <main class="container my-5">{% block content %}{% endblock %}</main>
</body> {% include 'footer.html' %}
</body>
</html> </html>

44
scipaperloader/templates/index.html
View File

@ -1,19 +1,21 @@
{% extends 'base.html' %} {% extends 'base.html' %} {% block content %}
{% block content %}
<main class="container my-5"> <div class="container text-center">
<div class="container text-center">
<h1 class="display-4">Welcome to SciPaperLoader</h1> <h1 class="display-4">Welcome to SciPaperLoader</h1>
<p class="lead">Your paper scraping tool is ready.</p> <p class="lead">Your paper scraping tool is ready.</p>
<p class="text-muted">A simple tool to scrape papers from Zotero API.</p> <p class="text-muted">A simple tool to scrape papers from Zotero API.</p>
</div> </div>
<div class="row g-4"> <div class="row g-4">
<div class="col-md-6"> <div class="col-md-6">
<div class="card shadow-sm"> <div class="card shadow-sm">
<div class="card-body"> <div class="card-body">
<h5 class="card-title">📄 CSV Import</h5> <h5 class="card-title">📄 CSV Import</h5>
<p class="card-text">Upload a 37-column CSV to import paper metadata. Only relevant fields (title, DOI, ISSN, etc.) are stored. Errors are reported without aborting the batch.</p> <p class="card-text">
Upload a 37-column CSV to import paper metadata. Only relevant fields
(title, DOI, ISSN, etc.) are stored. Errors are reported without
aborting the batch.
</p>
<a href="/import" class="btn btn-sm btn-outline-primary">Upload Now</a> <a href="/import" class="btn btn-sm btn-outline-primary">Upload Now</a>
</div> </div>
</div> </div>
@ -23,7 +25,10 @@
<div class="card shadow-sm"> <div class="card shadow-sm">
<div class="card-body"> <div class="card-body">
<h5 class="card-title">🧠 Background Scraper</h5> <h5 class="card-title">🧠 Background Scraper</h5>
<p class="card-text">A daemon process runs hourly to fetch papers using Zotero API. Downloads are randomized to mimic human behavior and avoid detection.</p> <p class="card-text">
A daemon process runs hourly to fetch papers using Zotero API.
Downloads are randomized to mimic human behavior and avoid detection.
</p>
<a href="/logs" class="btn btn-sm btn-outline-secondary">View Logs</a> <a href="/logs" class="btn btn-sm btn-outline-secondary">View Logs</a>
</div> </div>
</div> </div>
@ -33,8 +38,14 @@
<div class="card shadow-sm"> <div class="card shadow-sm">
<div class="card-body"> <div class="card-body">
<h5 class="card-title">📚 Paper Management</h5> <h5 class="card-title">📚 Paper Management</h5>
<p class="card-text">Monitor paper status (Pending, Done, Failed), download PDFs, and inspect errors. Files are stored on disk in structured folders per DOI.</p> <p class="card-text">
<a href="/papers" class="btn btn-sm btn-outline-success">Browse Papers</a> Monitor paper status (Pending, Done, Failed), download PDFs, and
inspect errors. Files are stored on disk in structured folders per
DOI.
</p>
<a href="/papers" class="btn btn-sm btn-outline-success"
>Browse Papers</a
>
</div> </div>
</div> </div>
</div> </div>
@ -43,11 +54,16 @@
<div class="card shadow-sm"> <div class="card shadow-sm">
<div class="card-body"> <div class="card-body">
<h5 class="card-title">🕒 Download Schedule</h5> <h5 class="card-title">🕒 Download Schedule</h5>
<p class="card-text">Control how many papers are downloaded per hour. Configure hourly volume (e.g. 2/hour at daytime, 0 at night) to match your bandwidth or usage pattern.</p> <p class="card-text">
<a href="/schedule" class="btn btn-sm btn-outline-warning">Adjust Schedule</a> Control how many papers are downloaded per hour. Configure hourly
volume (e.g. 2/hour at daytime, 0 at night) to match your bandwidth or
usage pattern.
</p>
<a href="/schedule" class="btn btn-sm btn-outline-warning"
>Adjust Schedule</a
>
</div> </div>
</div> </div>
</div> </div>
</div> </div>
</main>
{% endblock %} {% endblock %}

198
scipaperloader/templates/schedule.html
View File

@ -1,17 +1,16 @@
{% extends 'base.html' %} {% extends 'base.html' %} {% block content %}
{% block content %}
<style> <style>
.timeline { .timeline {
display: flex; display: flex;
flex-wrap: wrap; flex-wrap: wrap;
gap: 4px; gap: 3px;
user-select: none; /* Prevent text selection during drag */ user-select: none; /* Prevent text selection during drag */
} }
.hour-block { .hour-block {
width: 60px; width: 49px;
height: 70px; /* Increased height to fit additional text */ height: 70px; /* Increased height to fit additional text */
border-radius: 6px; border-radius: 5px;
text-align: center; text-align: center;
line-height: 1.2; line-height: 1.2;
font-size: 0.9rem; font-size: 0.9rem;
@ -32,11 +31,19 @@
} }
.flash-message { .flash-message {
position: fixed;
top: 30%;
left: 50%;
transform: translate(-50%, -50%);
z-index: 1000;
width: 300px;
text-align: center;
font-weight: bold;
padding: 12px; padding: 12px;
margin-bottom: 20px; margin-bottom: 20px;
border-radius: 6px; border-radius: 6px;
opacity: 1; opacity: 1;
transition: opacity 2s ease-in-out; transition: opacity 5s ease-in-out;
} }
.flash-message.success { .flash-message.success {
@ -54,78 +61,123 @@
.flash-message.fade { .flash-message.fade {
opacity: 0; opacity: 0;
} }
</style>
</style> <script>
<script>
const initialSchedule = {{ schedule | tojson }}; const initialSchedule = {{ schedule | tojson }};
const totalVolume = {{ volume }}; const totalVolume = {{ volume }};
</script> </script>
<div x-data="scheduleManager(initialSchedule, totalVolume)" class="container">
<div x-data="scheduleManager(initialSchedule, totalVolume)" class="container my-5"> <h1 class="mb-4">🕒 Configure Hourly Download Weights</h1>
<h2 class="mb-4">🕒 Configure Hourly Download Weights</h2>
<!-- Flash Messages --> <!-- Flash Messages -->
{% with messages = get_flashed_messages(with_categories=true) %} {% with messages = get_flashed_messages(with_categories=true) %} {% if
{% if messages %} messages %}
<div id="flash-messages"> <div id="flash-messages">
{% for category, message in messages %} {% for category, message in messages %}
<div class="flash-message {{ category }}" x-data="{}" x-init="setTimeout(() => $el.classList.add('fade'), 100); setTimeout(() => $el.remove(), 5000)"> <div
class="flash-message {{ category }}"
x-data="{}"
x-init="setTimeout(() => $el.classList.add('fade'), 100); setTimeout(() => $el.remove(), 5000)"
>
{{ message }} {{ message }}
</div> </div>
{% endfor %} {% endfor %}
</div> </div>
{% endif %} {% endif %} {% endwith %}
{% endwith %}
<!-- Content -->
<div class="mb-3">
<h3>How it Works</h3>
<p class="text-muted mb-0">
This page allows you to configure the daily volume of papers to be
downloaded and the hourly download weights for the papers. The weights
determine how many papers will be downloaded during each hour of the day.
The total volume (<strong x-text="volume"></strong> papers/day) is split
across all hours based on their relative weights. Each weight controls the
proportion of papers downloaded during that hour. Click to select one or
more hours below. Then assign a weight to them using the input and apply
it. Color indicates relative intensity. The total daily volume will be
split proportionally across these weights.
<strong>Don't forget to submit the changes!</strong>
</p>
<h3>Example</h3>
<p class="text-muted mb-0">
If the total volume is <strong>240 papers</strong> and hours are
<strong>weighted as 1.0, 2.0, and 3.0</strong>, they will receive
<strong>40, 80, and 120 papers</strong> respectively.
</p>
</div>
<h2 class="mt-4">Volume</h2>
<div class="align-items-start flex-wrap gap-2">
<p class="text-muted"> <p class="text-muted">
Click to select one or more hours below. Then assign a weight to them using the input and apply it. Color indicates relative intensity. The total daily volume will be split proportionally across these weights. The total volume of data to be downloaded each day is
<strong x-text="volume"></strong> papers.
</p> </p>
<div class="d-flex align-items-center mb-3">
<h3 class="mt-4">Volume</h3> <form
<p class="text-muted">The total volume of data to be downloaded each day is <strong x-text="volume"></strong> papers.</p> method="POST"
action="{{ url_for('main.schedule') }}"
<div class="d-flex justify-content-between align-items-start flex-wrap gap-2"> class="input-group w-50"
<p class="text-muted mb-0" style="max-width: 600px;"> >
Click to select one or more hours below. Then assign a weight to them using the input and apply it. Color indicates relative intensity. The total daily volume will be split proportionally across these weights. <label class="input-group-text">Papers per day:</label>
</p> <input
type="number"
<div class="hour-block" class="form-control"
style="pointer-events: none;" name="total_volume"
x-init="$nextTick(() => previewWeight = 1.0)" value="{{ volume }}"
x-data="{ previewWeight: 1.0 }" min="1"
:style="getBackgroundStyleFromValue(previewWeight)"> max="1000"
<div><strong>14:00</strong></div> required
<div class="weight"><span x-text="previewWeight.toFixed(1)"></span></div> />
<div class="papers text-muted">example</div> <button type="submit" class="btn btn-primary">Update Volume</button>
</form>
</div> </div>
</div> </div>
<h2 class="mt-4">Current Schedule</h2>
<h3 class="mt-4">Current Schedule</h3>
<form method="POST" action="{{ url_for('main.schedule') }}"> <form method="POST" action="{{ url_for('main.schedule') }}">
<div class="timeline mb-3" @mouseup="endDrag()" @mouseleave="endDrag()"> <div class="timeline mb-3" @mouseup="endDrag()" @mouseleave="endDrag()">
<template x-for="hour in Object.keys(schedule)" :key="hour"> <template x-for="hour in Object.keys(schedule)" :key="hour">
<div class="hour-block" <div
class="hour-block"
:id="'hour-' + hour" :id="'hour-' + hour"
:data-hour="hour" :data-hour="hour"
:style="getBackgroundStyle(hour)" :style="getBackgroundStyle(hour)"
:class="{'selected': isSelected(hour)}" :class="{'selected': isSelected(hour)}"
@mousedown="startDrag($event, hour)" @mousedown="startDrag($event, hour)"
@mouseover="dragSelect(hour)"> @mouseover="dragSelect(hour)"
>
<div><strong x-text="formatHour(hour)"></strong></div> <div><strong x-text="formatHour(hour)"></strong></div>
<div class="weight"><span x-text="schedule[hour]"></span></div> <div class="weight"><span x-text="schedule[hour]"></span></div>
<div class="papers"><span x-text="getPapersPerHour(hour)"></span> p.</div> <div class="papers">
<input type="hidden" :name="'hour_' + hour" :value="schedule[hour]"> <span x-text="getPapersPerHour(hour)"></span> p.
</div>
<input type="hidden" :name="'hour_' + hour" :value="schedule[hour]" />
</div> </div>
</template> </template>
</div> </div>
<div class="input-group mb-4 w-50"> <div class="input-group mb-4 w-50">
<label class="input-group-text">Set Weight:</label> <label class="input-group-text">Set Weight:</label>
<input type="number" step="0.1" min="0" max="5" x-model="newWeight" class="form-control"> <input
<button type="button" class="btn btn-outline-primary" @click="applyWeight()">Apply to Selected</button> type="number"
step="0.1"
min="0"
max="5"
x-model="newWeight"
class="form-control"
/>
<button
type="button"
class="btn btn-outline-primary"
@click="applyWeight()"
>
Apply to Selected
</button>
</div> </div>
<div class="d-flex justify-content-between"> <div class="d-flex justify-content-between">
@ -133,9 +185,9 @@
<button type="submit" class="btn btn-success">💾 Save Schedule</button> <button type="submit" class="btn btn-success">💾 Save Schedule</button>
</div> </div>
</form> </form>
</div> </div>
<script> <script>
function scheduleManager(initial, volume) { function scheduleManager(initial, volume) {
return { return {
schedule: { ...initial }, schedule: { ...initial },
@ -146,7 +198,7 @@
dragOperation: null, dragOperation: null,
formatHour(h) { formatHour(h) {
return String(h).padStart(2, '0') + ":00"; return String(h).padStart(2, "0") + ":00";
}, },
getBackgroundStyle(hour) { getBackgroundStyle(hour) {
@ -157,31 +209,50 @@
const t = Math.min(weight / maxWeight, 1.0); const t = Math.min(weight / maxWeight, 1.0);
// Interpolate HSL lightness: 95% (light) to 30% (dark) // Interpolate HSL lightness: 95% (light) to 30% (dark)
const lightness = 95 - (t * 65); // 95 → 30 const lightness = 95 - t * 65; // 95 → 30
const backgroundColor = `hsl(210, 10%, ${lightness}%)`; // soft gray-blue palette const backgroundColor = `hsl(210, 10%, ${lightness}%)`; // soft gray-blue palette
const textColor = t > 0.65 ? 'white' : 'black'; // adaptive text color const textColor = t > 0.65 ? "white" : "black"; // adaptive text color
return { return {
backgroundColor, backgroundColor,
color: textColor color: textColor,
};
},
getBackgroundStyleFromValue(value) {
const weight = parseFloat(value);
const maxWeight = 2.5; // You can adjust this
// Normalize weight (0.0 to 1.0)
const t = Math.min(weight / maxWeight, 1.0);
// Interpolate HSL lightness: 95% (light) to 30% (dark)
const lightness = 95 - t * 65; // 95 → 30
const backgroundColor = `hsl(210, 10%, ${lightness}%)`; // soft gray-blue palette
const textColor = t > 0.65 ? "white" : "black"; // adaptive text color
return {
backgroundColor,
color: textColor,
}; };
}, },
startDrag(event, hour) { startDrag(event, hour) {
event.preventDefault(); event.preventDefault();
this.isDragging = true; this.isDragging = true;
this.dragOperation = this.isSelected(hour) ? 'remove' : 'add'; this.dragOperation = this.isSelected(hour) ? "remove" : "add";
this.toggleSelect(hour); this.toggleSelect(hour);
}, },
dragSelect(hour) { dragSelect(hour) {
if (!this.isDragging) return; if (!this.isDragging) return;
const selected = this.isSelected(hour); const selected = this.isSelected(hour);
if (this.dragOperation === 'add' && !selected) { if (this.dragOperation === "add" && !selected) {
this.selectedHours.push(hour); this.selectedHours.push(hour);
} else if (this.dragOperation === 'remove' && selected) { } else if (this.dragOperation === "remove" && selected) {
this.selectedHours = this.selectedHours.filter(h => h !== hour); this.selectedHours = this.selectedHours.filter((h) => h !== hour);
} }
}, },
@ -191,7 +262,7 @@
toggleSelect(hour) { toggleSelect(hour) {
if (this.isSelected(hour)) { if (this.isSelected(hour)) {
this.selectedHours = this.selectedHours.filter(h => h !== hour); this.selectedHours = this.selectedHours.filter((h) => h !== hour);
} else { } else {
this.selectedHours.push(hour); this.selectedHours.push(hour);
} }
@ -202,21 +273,28 @@
}, },
applyWeight() { applyWeight() {
this.selectedHours.forEach(hour => { this.selectedHours.forEach((hour) => {
this.schedule[hour] = parseFloat(this.newWeight).toFixed(1); this.schedule[hour] = parseFloat(this.newWeight).toFixed(1);
}); });
this.selectedHours = []; this.selectedHours = [];
}, },
getTotalWeight() { getTotalWeight() {
return Object.values(this.schedule).reduce((sum, w) => sum + parseFloat(w), 0); return Object.values(this.schedule).reduce(
(sum, w) => sum + parseFloat(w),
0
);
}, },
getPapersPerHour(hour) { getPapersPerHour(hour) {
const total = this.getTotalWeight(); const total = this.getTotalWeight();
if (total === 0) return 0; if (total === 0) return 0;
return ((parseFloat(this.schedule[hour]) / total) * this.volume).toFixed(1); return (
} (parseFloat(this.schedule[hour]) / total) *
this.volume
).toFixed(1);
},
}; };
} }
</script>{% endblock %} </script>
{% endblock %}

35
scipaperloader/views.py
View File

@ -4,10 +4,12 @@ from .db import db
bp = Blueprint('main', __name__) bp = Blueprint('main', __name__)
@bp.route("/") @bp.route("/")
def index(): def index():
return render_template("index.html") return render_template("index.html")
@bp.route("/upload", methods=["GET", "POST"]) @bp.route("/upload", methods=["GET", "POST"])
def upload(): def upload():
if request.method == "POST": if request.method == "POST":
@ -15,14 +17,39 @@ def upload():
pass pass
return render_template("upload.html") return render_template("upload.html")
@bp.route("/papers") @bp.route("/papers")
def papers(): def papers():
return render_template("papers.html", app_title="PaperScraper") return render_template("papers.html", app_title="PaperScraper")
@bp.route("/schedule", methods=["GET", "POST"]) @bp.route("/schedule", methods=["GET", "POST"])
def schedule(): def schedule():
if request.method == "POST": if request.method == "POST":
try: try:
# Check if we're updating volume or schedule
if 'total_volume' in request.form:
# Volume update
try:
new_volume = float(request.form.get('total_volume', 0))
if new_volume <= 0 or new_volume > 1000:
raise ValueError("Volume must be between 1 and 1000")
volume_config = VolumeConfig.query.first()
if not volume_config:
volume_config = VolumeConfig(volume=new_volume)
db.session.add(volume_config)
else:
volume_config.volume = new_volume
db.session.commit()
flash("Volume updated successfully!", "success")
except ValueError as e:
db.session.rollback()
flash(f"Error updating volume: {str(e)}", "error")
else:
# Schedule update logic
# Validate form data # Validate form data
for hour in range(24): for hour in range(24):
key = f"hour_{hour}" key = f"hour_{hour}"
@ -32,7 +59,8 @@ def schedule():
try: try:
weight = float(request.form.get(key, 0)) weight = float(request.form.get(key, 0))
if weight < 0 or weight > 5: if weight < 0 or weight > 5:
raise ValueError(f"Weight for hour {hour} must be between 0 and 5") raise ValueError(
f"Weight for hour {hour} must be between 0 and 5")
except ValueError: except ValueError:
raise ValueError(f"Invalid weight value for hour {hour}") raise ValueError(f"Invalid weight value for hour {hour}")
@ -53,14 +81,17 @@ def schedule():
db.session.rollback() db.session.rollback()
flash(f"Error updating schedule: {str(e)}", "error") flash(f"Error updating schedule: {str(e)}", "error")
schedule = {sc.hour: sc.weight for sc in ScheduleConfig.query.order_by(ScheduleConfig.hour).all()} schedule = {sc.hour: sc.weight for sc in ScheduleConfig.query.order_by(
ScheduleConfig.hour).all()}
volume = VolumeConfig.query.first() volume = VolumeConfig.query.first()
return render_template("schedule.html", schedule=schedule, volume=volume.volume, app_title="PaperScraper") return render_template("schedule.html", schedule=schedule, volume=volume.volume, app_title="PaperScraper")
@bp.route("/logs") @bp.route("/logs")
def logs(): def logs():
return render_template("logs.html", app_title="PaperScraper") return render_template("logs.html", app_title="PaperScraper")
@bp.route("/about") @bp.route("/about")
def about(): def about():
return render_template("about.html", app_title="PaperScraper") return render_template("about.html", app_title="PaperScraper")