mischbeck/SciPaperLoader
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Finishes Scheduler View & Controller

Browse Source
This commit is contained in:
Michael Beck 2025-03-31 01:28:07 +02:00
parent 2868916cf6
commit 1534dbb0ba
4 changed files with 427 additions and 298 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
scipaperloader/templates/base.html
View File

@ -1,18 +1,22 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta charset="UTF-8" />
<title>{{ app_title }}</title>
<link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.3/dist/css/bootstrap.min.css" rel="stylesheet">
<link
href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.3/dist/css/bootstrap.min.css"
rel="stylesheet"
/>
<script src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.3/dist/js/bootstrap.bundle.min.js"></script>
<!-- Optional Alpine.js -->
<script defer src="https://cdn.jsdelivr.net/npm/alpinejs@3.x.x/dist/cdn.min.js"></script>
<script
defer
src="https://cdn.jsdelivr.net/npm/alpinejs@3.x.x/dist/cdn.min.js"
></script>
</head>
<body>
{% include 'nav.html' %}
<div class="container py-4">
{% block content %}{% endblock %}
</div>
<main class="container my-5">{% block content %}{% endblock %}</main>
{% include 'footer.html' %}
</body>
</html>

36
scipaperloader/templates/index.html
View File

@ -1,7 +1,5 @@
{% extends 'base.html' %}
{% block content %}
{% extends 'base.html' %} {% block content %}
<main class="container my-5">
<div class="container text-center">
<h1 class="display-4">Welcome to SciPaperLoader</h1>
<p class="lead">Your paper scraping tool is ready.</p>
@ -13,7 +11,11 @@
<div class="card shadow-sm">
<div class="card-body">
<h5 class="card-title">📄 CSV Import</h5>
<p class="card-text">Upload a 37-column CSV to import paper metadata. Only relevant fields (title, DOI, ISSN, etc.) are stored. Errors are reported without aborting the batch.</p>
<p class="card-text">
Upload a 37-column CSV to import paper metadata. Only relevant fields
(title, DOI, ISSN, etc.) are stored. Errors are reported without
aborting the batch.
</p>
<a href="/import" class="btn btn-sm btn-outline-primary">Upload Now</a>
</div>
</div>
@ -23,7 +25,10 @@
<div class="card shadow-sm">
<div class="card-body">
<h5 class="card-title">🧠 Background Scraper</h5>
<p class="card-text">A daemon process runs hourly to fetch papers using Zotero API. Downloads are randomized to mimic human behavior and avoid detection.</p>
<p class="card-text">
A daemon process runs hourly to fetch papers using Zotero API.
Downloads are randomized to mimic human behavior and avoid detection.
</p>
<a href="/logs" class="btn btn-sm btn-outline-secondary">View Logs</a>
</div>
</div>
@ -33,8 +38,14 @@
<div class="card shadow-sm">
<div class="card-body">
<h5 class="card-title">📚 Paper Management</h5>
<p class="card-text">Monitor paper status (Pending, Done, Failed), download PDFs, and inspect errors. Files are stored on disk in structured folders per DOI.</p>
<a href="/papers" class="btn btn-sm btn-outline-success">Browse Papers</a>
<p class="card-text">
Monitor paper status (Pending, Done, Failed), download PDFs, and
inspect errors. Files are stored on disk in structured folders per
DOI.
</p>
<a href="/papers" class="btn btn-sm btn-outline-success"
>Browse Papers</a
>
</div>
</div>
</div>
@ -43,11 +54,16 @@
<div class="card shadow-sm">
<div class="card-body">
<h5 class="card-title">🕒 Download Schedule</h5>
<p class="card-text">Control how many papers are downloaded per hour. Configure hourly volume (e.g. 2/hour at daytime, 0 at night) to match your bandwidth or usage pattern.</p>
<a href="/schedule" class="btn btn-sm btn-outline-warning">Adjust Schedule</a>
<p class="card-text">
Control how many papers are downloaded per hour. Configure hourly
volume (e.g. 2/hour at daytime, 0 at night) to match your bandwidth or
usage pattern.
</p>
<a href="/schedule" class="btn btn-sm btn-outline-warning"
>Adjust Schedule</a
>
</div>
</div>
</div>
</div>
</main>
{% endblock %}

188
scipaperloader/templates/schedule.html
View File

@ -1,17 +1,16 @@
{% extends 'base.html' %}
{% block content %}
{% extends 'base.html' %} {% block content %}
<style>
.timeline {
display: flex;
flex-wrap: wrap;
gap: 4px;
gap: 3px;
user-select: none; /* Prevent text selection during drag */
}
.hour-block {
width: 60px;
width: 49px;
height: 70px; /* Increased height to fit additional text */
border-radius: 6px;
border-radius: 5px;
text-align: center;
line-height: 1.2;
font-size: 0.9rem;
@ -32,11 +31,19 @@
}
.flash-message {
position: fixed;
top: 30%;
left: 50%;
transform: translate(-50%, -50%);
z-index: 1000;
width: 300px;
text-align: center;
font-weight: bold;
padding: 12px;
margin-bottom: 20px;
border-radius: 6px;
opacity: 1;
transition: opacity 2s ease-in-out;
transition: opacity 5s ease-in-out;
}
.flash-message.success {
@ -54,7 +61,6 @@
.flash-message.fade {
opacity: 0;
}
</style>
<script>
@ -62,70 +68,116 @@
const totalVolume = {{ volume }};
</script>
<div x-data="scheduleManager(initialSchedule, totalVolume)" class="container my-5">
<h2 class="mb-4">🕒 Configure Hourly Download Weights</h2>
<div x-data="scheduleManager(initialSchedule, totalVolume)" class="container">
<h1 class="mb-4">🕒 Configure Hourly Download Weights</h1>
<!-- Flash Messages -->
{% with messages = get_flashed_messages(with_categories=true) %}
{% if messages %}
{% with messages = get_flashed_messages(with_categories=true) %} {% if
messages %}
<div id="flash-messages">
{% for category, message in messages %}
<div class="flash-message {{ category }}" x-data="{}" x-init="setTimeout(() => $el.classList.add('fade'), 100); setTimeout(() => $el.remove(), 5000)">
<div
class="flash-message {{ category }}"
x-data="{}"
x-init="setTimeout(() => $el.classList.add('fade'), 100); setTimeout(() => $el.remove(), 5000)"
>
{{ message }}
</div>
{% endfor %}
</div>
{% endif %}
{% endwith %}
{% endif %} {% endwith %}
<!-- Content -->
<div class="mb-3">
<h3>How it Works</h3>
<p class="text-muted mb-0">
This page allows you to configure the daily volume of papers to be
downloaded and the hourly download weights for the papers. The weights
determine how many papers will be downloaded during each hour of the day.
The total volume (<strong x-text="volume"></strong> papers/day) is split
across all hours based on their relative weights. Each weight controls the
proportion of papers downloaded during that hour. Click to select one or
more hours below. Then assign a weight to them using the input and apply
it. Color indicates relative intensity. The total daily volume will be
split proportionally across these weights.
<strong>Don't forget to submit the changes!</strong>
</p>
<h3>Example</h3>
<p class="text-muted mb-0">
If the total volume is <strong>240 papers</strong> and hours are
<strong>weighted as 1.0, 2.0, and 3.0</strong>, they will receive
<strong>40, 80, and 120 papers</strong> respectively.
</p>
</div>
<h2 class="mt-4">Volume</h2>
<div class="align-items-start flex-wrap gap-2">
<p class="text-muted">
Click to select one or more hours below. Then assign a weight to them using the input and apply it. Color indicates relative intensity. The total daily volume will be split proportionally across these weights.
The total volume of data to be downloaded each day is
<strong x-text="volume"></strong> papers.
</p>
<h3 class="mt-4">Volume</h3>
<p class="text-muted">The total volume of data to be downloaded each day is <strong x-text="volume"></strong> papers.</p>
<div class="d-flex justify-content-between align-items-start flex-wrap gap-2">
<p class="text-muted mb-0" style="max-width: 600px;">
Click to select one or more hours below. Then assign a weight to them using the input and apply it. Color indicates relative intensity. The total daily volume will be split proportionally across these weights.
</p>
<div class="hour-block"
style="pointer-events: none;"
x-init="$nextTick(() => previewWeight = 1.0)"
x-data="{ previewWeight: 1.0 }"
:style="getBackgroundStyleFromValue(previewWeight)">
<div><strong>14:00</strong></div>
<div class="weight"><span x-text="previewWeight.toFixed(1)"></span></div>
<div class="papers text-muted">example</div>
<div class="d-flex align-items-center mb-3">
<form
method="POST"
action="{{ url_for('main.schedule') }}"
class="input-group w-50"
>
<label class="input-group-text">Papers per day:</label>
<input
type="number"
class="form-control"
name="total_volume"
value="{{ volume }}"
min="1"
max="1000"
required
/>
<button type="submit" class="btn btn-primary">Update Volume</button>
</form>
</div>
</div>
<h3 class="mt-4">Current Schedule</h3>
<h2 class="mt-4">Current Schedule</h2>
<form method="POST" action="{{ url_for('main.schedule') }}">
<div class="timeline mb-3" @mouseup="endDrag()" @mouseleave="endDrag()">
<template x-for="hour in Object.keys(schedule)" :key="hour">
<div class="hour-block"
<div
class="hour-block"
:id="'hour-' + hour"
:data-hour="hour"
:style="getBackgroundStyle(hour)"
:class="{'selected': isSelected(hour)}"
@mousedown="startDrag($event, hour)"
@mouseover="dragSelect(hour)">
@mouseover="dragSelect(hour)"
>
<div><strong x-text="formatHour(hour)"></strong></div>
<div class="weight"><span x-text="schedule[hour]"></span></div>
<div class="papers"><span x-text="getPapersPerHour(hour)"></span> p.</div>
<input type="hidden" :name="'hour_' + hour" :value="schedule[hour]">
<div class="papers">
<span x-text="getPapersPerHour(hour)"></span> p.
</div>
<input type="hidden" :name="'hour_' + hour" :value="schedule[hour]" />
</div>
</template>
</div>
<div class="input-group mb-4 w-50">
<label class="input-group-text">Set Weight:</label>
<input type="number" step="0.1" min="0" max="5" x-model="newWeight" class="form-control">
<button type="button" class="btn btn-outline-primary" @click="applyWeight()">Apply to Selected</button>
<input
type="number"
step="0.1"
min="0"
max="5"
x-model="newWeight"
class="form-control"
/>
<button
type="button"
class="btn btn-outline-primary"
@click="applyWeight()"
>
Apply to Selected
</button>
</div>
<div class="d-flex justify-content-between">
@ -146,7 +198,7 @@
dragOperation: null,
formatHour(h) {
return String(h).padStart(2, '0') + ":00";
return String(h).padStart(2, "0") + ":00";
},
getBackgroundStyle(hour) {
@ -157,31 +209,50 @@
const t = Math.min(weight / maxWeight, 1.0);
// Interpolate HSL lightness: 95% (light) to 30% (dark)
const lightness = 95 - (t * 65); // 95 → 30
const lightness = 95 - t * 65; // 95 → 30
const backgroundColor = `hsl(210, 10%, ${lightness}%)`; // soft gray-blue palette
const textColor = t > 0.65 ? 'white' : 'black'; // adaptive text color
const textColor = t > 0.65 ? "white" : "black"; // adaptive text color
return {
backgroundColor,
color: textColor
color: textColor,
};
},
getBackgroundStyleFromValue(value) {
const weight = parseFloat(value);
const maxWeight = 2.5; // You can adjust this
// Normalize weight (0.0 to 1.0)
const t = Math.min(weight / maxWeight, 1.0);
// Interpolate HSL lightness: 95% (light) to 30% (dark)
const lightness = 95 - t * 65; // 95 → 30
const backgroundColor = `hsl(210, 10%, ${lightness}%)`; // soft gray-blue palette
const textColor = t > 0.65 ? "white" : "black"; // adaptive text color
return {
backgroundColor,
color: textColor,
};
},
startDrag(event, hour) {
event.preventDefault();
this.isDragging = true;
this.dragOperation = this.isSelected(hour) ? 'remove' : 'add';
this.dragOperation = this.isSelected(hour) ? "remove" : "add";
this.toggleSelect(hour);
},
dragSelect(hour) {
if (!this.isDragging) return;
const selected = this.isSelected(hour);
if (this.dragOperation === 'add' && !selected) {
if (this.dragOperation === "add" && !selected) {
this.selectedHours.push(hour);
} else if (this.dragOperation === 'remove' && selected) {
this.selectedHours = this.selectedHours.filter(h => h !== hour);
} else if (this.dragOperation === "remove" && selected) {
this.selectedHours = this.selectedHours.filter((h) => h !== hour);
}
},
@ -191,7 +262,7 @@
toggleSelect(hour) {
if (this.isSelected(hour)) {
this.selectedHours = this.selectedHours.filter(h => h !== hour);
this.selectedHours = this.selectedHours.filter((h) => h !== hour);
} else {
this.selectedHours.push(hour);
}
@ -202,21 +273,28 @@
},
applyWeight() {
this.selectedHours.forEach(hour => {
this.selectedHours.forEach((hour) => {
this.schedule[hour] = parseFloat(this.newWeight).toFixed(1);
});
this.selectedHours = [];
},
getTotalWeight() {
return Object.values(this.schedule).reduce((sum, w) => sum + parseFloat(w), 0);
return Object.values(this.schedule).reduce(
(sum, w) => sum + parseFloat(w),
0
);
},
getPapersPerHour(hour) {
const total = this.getTotalWeight();
if (total === 0) return 0;
return ((parseFloat(this.schedule[hour]) / total) * this.volume).toFixed(1);
}
return (
(parseFloat(this.schedule[hour]) / total) *
this.volume
).toFixed(1);
},
};
}
</script>{% endblock %}
</script>
{% endblock %}

35
scipaperloader/views.py
View File

@ -4,10 +4,12 @@ from .db import db
bp = Blueprint('main', __name__)
@bp.route("/")
def index():
return render_template("index.html")
@bp.route("/upload", methods=["GET", "POST"])
def upload():
if request.method == "POST":
@ -15,14 +17,39 @@ def upload():
pass
return render_template("upload.html")
@bp.route("/papers")
def papers():
return render_template("papers.html", app_title="PaperScraper")
@bp.route("/schedule", methods=["GET", "POST"])
def schedule():
if request.method == "POST":
try:
# Check if we're updating volume or schedule
if 'total_volume' in request.form:
# Volume update
try:
new_volume = float(request.form.get('total_volume', 0))
if new_volume <= 0 or new_volume > 1000:
raise ValueError("Volume must be between 1 and 1000")
volume_config = VolumeConfig.query.first()
if not volume_config:
volume_config = VolumeConfig(volume=new_volume)
db.session.add(volume_config)
else:
volume_config.volume = new_volume
db.session.commit()
flash("Volume updated successfully!", "success")
except ValueError as e:
db.session.rollback()
flash(f"Error updating volume: {str(e)}", "error")
else:
# Schedule update logic
# Validate form data
for hour in range(24):
key = f"hour_{hour}"
@ -32,7 +59,8 @@ def schedule():
try:
weight = float(request.form.get(key, 0))
if weight < 0 or weight > 5:
raise ValueError(f"Weight for hour {hour} must be between 0 and 5")
raise ValueError(
f"Weight for hour {hour} must be between 0 and 5")
except ValueError:
raise ValueError(f"Invalid weight value for hour {hour}")
@ -53,14 +81,17 @@ def schedule():
db.session.rollback()
flash(f"Error updating schedule: {str(e)}", "error")
schedule = {sc.hour: sc.weight for sc in ScheduleConfig.query.order_by(ScheduleConfig.hour).all()}
schedule = {sc.hour: sc.weight for sc in ScheduleConfig.query.order_by(
ScheduleConfig.hour).all()}
volume = VolumeConfig.query.first()
return render_template("schedule.html", schedule=schedule, volume=volume.volume, app_title="PaperScraper")
@bp.route("/logs")
def logs():
return render_template("logs.html", app_title="PaperScraper")
@bp.route("/about")
def about():
return render_template("about.html", app_title="PaperScraper")