mischbeck/SciPaperLoader
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fixes quota recalculation

Browse Source
This commit is contained in:
Michael Beck 2025-05-23 22:22:49 +02:00
parent 74e713e8a6
commit 243e24e100
4 changed files with 201 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
scipaperloader/blueprints/config.py
View File

@ -49,6 +49,19 @@ def _update_volume(new_volume):
) )
db.session.commit() db.session.commit()
# Invalidate and recalculate the hourly quota cache
try:
# Import the calculation function from the scraper module
from ..blueprints.scraper import calculate_papers_for_current_hour
invalidate_hourly_quota_cache(calculate_papers_for_current_hour)
except Exception as e:
# Log the error but don't fail the update
ActivityLog.log_error(
error_message=f"Error invalidating hourly quota cache: {str(e)}",
source="_update_volume"
)
return True, "Volume updated successfully!", volume_config return True, "Volume updated successfully!", volume_config
except (ValueError, TypeError) as e: except (ValueError, TypeError) as e:
@ -175,9 +188,11 @@ def _update_schedule(schedule_data):
db.session.commit() db.session.commit()
# Invalidate hourly quota cache using the cache_utils module # Invalidate hourly quota cache and immediately recalculate
try: try:
invalidate_hourly_quota_cache() # Import the calculation function from the scraper module
from ..blueprints.scraper import calculate_papers_for_current_hour
invalidate_hourly_quota_cache(calculate_papers_for_current_hour)
except Exception as e: except Exception as e:
# Log the error but don't fail the update # Log the error but don't fail the update
ActivityLog.log_error( ActivityLog.log_error(

36
scipaperloader/cache_utils.py
View File

@ -12,17 +12,37 @@ HOURLY_QUOTA_CACHE = {
'last_config_update': None, # Last time volume or schedule config was updated 'last_config_update': None, # Last time volume or schedule config was updated
} }
def invalidate_hourly_quota_cache(): def invalidate_hourly_quota_cache(calculate_function=None):
"""Invalidate the hourly quota cache when configuration changes.""" """
Invalidate the hourly quota cache when configuration changes.
Args:
calculate_function (callable, optional): Function to recalculate quota immediately.
If None, recalculation will happen during next get_cached_hourly_quota() call.
"""
global HOURLY_QUOTA_CACHE global HOURLY_QUOTA_CACHE
HOURLY_QUOTA_CACHE['last_config_update'] = None HOURLY_QUOTA_CACHE['last_config_update'] = None
# Log the cache invalidation # If a calculation function is provided, recalculate immediately
ActivityLog.log_scraper_activity( if calculate_function:
action="cache_invalidated", current_hour = datetime.now().hour
status="info", quota = calculate_function()
description="Hourly quota cache was invalidated due to configuration changes" HOURLY_QUOTA_CACHE['hour'] = current_hour
) HOURLY_QUOTA_CACHE['quota'] = quota
HOURLY_QUOTA_CACHE['last_config_update'] = datetime.now()
ActivityLog.log_scraper_activity(
action="cache_recalculated",
status="info",
description=f"Hourly quota immediately recalculated after config change: {quota} papers"
)
else:
# Log the cache invalidation
ActivityLog.log_scraper_activity(
action="cache_invalidated",
status="info",
description="Hourly quota cache was invalidated due to configuration changes"
)
def get_cached_hourly_quota(calculate_function): def get_cached_hourly_quota(calculate_function):
""" """

211
scipaperloader/templates/config/schedule.html.jinja
View File

@ -30,6 +30,13 @@
font-size: 0.7rem; font-size: 0.7rem;
margin-top: 2px; margin-top: 2px;
} }
.weight-gradient {
width: 50px;
height: 15px;
background: linear-gradient(to right, hsl(210, 10%, 95%), hsl(210, 10%, 30%));
border-radius: 2px;
}
</style> </style>
<script> <script>
@ -39,86 +46,168 @@
<div x-data="scheduleManager(initialSchedule, totalVolume)" class="tab-pane active"> <div x-data="scheduleManager(initialSchedule, totalVolume)" class="tab-pane active">
<div class="card"> <div class="card">
<div class="card-header"> <div class="card-header d-flex justify-content-between">
<h5>Scheduling Configuration</h5> <h5>Scheduling Configuration</h5>
<span>
<button class="btn btn-sm btn-outline-secondary" type="button" data-bs-toggle="collapse"
data-bs-target="#helpContent">
<i class="fas fa-question-circle"></i> Help
</button>
</span>
</div> </div>
<div class="card-body"> <div class="card-body">
<!-- include flash messages template --> <!-- include flash messages template -->
{% include "partials/flash_messages.html.jinja" %} {% include "partials/flash_messages.html.jinja" %}
<!-- Content --> <!-- Collapsible Help Content -->
<div class="mb-3"> <div class="collapse mt-3" id="helpContent">
<h3>How it Works</h3> <div class="card card-body">
<p class="text-muted mb-0"> <ul class="nav nav-tabs" id="helpTabs" role="tablist">
This page allows you to configure the daily volume of papers to be <li class="nav-item" role="presentation">
downloaded and the hourly download weights for the papers. The weights <button class="nav-link active" id="calculation-tab" data-bs-toggle="tab"
determine how many papers will be downloaded during each hour of the day. data-bs-target="#calculation" type="button">Calculation</button>
The total volume (<strong x-text="volume"></strong> papers/day) is split </li>
across all hours based on their relative weights. Each weight controls the <li class="nav-item" role="presentation">
proportion of papers downloaded during that hour. Click to select one or <button class="nav-link" id="usage-tab" data-bs-toggle="tab" data-bs-target="#usage"
more hours below. Then assign a weight to them using the input and apply type="button">Usage</button>
it. Color indicates relative intensity. The total daily volume will be </li>
split proportionally across these weights. <li class="nav-item" role="presentation">
<strong>Don't forget to submit the changes!</strong> <button class="nav-link" id="example-tab" data-bs-toggle="tab" data-bs-target="#example"
</p> type="button">Example</button>
<h3>Example</h3> </li>
<p class="text-muted mb-0"> </ul>
If the total volume is <strong>240 papers</strong> and hours are <div class="tab-content p-3 border border-top-0 rounded-bottom">
<strong>weighted as 1.0, 2.0, and 3.0</strong>, they will receive <!-- Calculation Tab -->
<strong>40, 80, and 120 papers</strong> respectively. <div class="tab-pane fade show active" id="calculation" role="tabpanel">
</p> <h5>Quota Calculation</h5>
</div> <p>Each hour's quota is calculated as:</p>
<div class="bg-light p-2 mb-2 rounded">
<code>Papers per hour = (Hour Weight ÷ Total Weight) × Daily Volume</code>
</div>
<p class="small mb-0">Changes to either volume or schedule weights will immediately
recalculate all hourly quotas.</p>
</div>
<h2 class="mt-4">Volume</h2> <!-- Usage Instructions Tab -->
<div class="tab-pane fade" id="usage" role="tabpanel">
<h5>Usage Instructions</h5>
<ol class="mb-0">
<li>Click to select one or more hour blocks (use drag to select multiple)</li>
<li>Adjust the weight value for selected hours (0.1-5.0)</li>
<li>Click "Apply to Selected" to set the weights</li>
<li>Click "Save Schedule" to commit your changes</li>
</ol>
</div>
<div class="align-items-start flex-wrap gap-2"> <!-- Example Tab -->
<p class="text-muted"> <div class="tab-pane fade" id="example" role="tabpanel">
The total volume of data to be downloaded each day is <h5>Example</h5>
<strong x-text="volume"></strong> papers. <p class="mb-0">
</p> With a daily volume of <strong>240 papers</strong> and hour weights of
<div class="d-flex align-items-center mb-3" x-data="{ volumeValue: volume }"> <strong>1.0, 2.0, and 3.0</strong>, the distribution will be
<div class="input-group w-50"> <strong>40, 80, and 120 papers</strong> respectively
<label class="input-group-text">Papers per day:</label> (based on ratios 1:2:3 of the total weight 6.0).
<input type="number" class="form-control" x-model="volumeValue" min="1" max="{{ max_volume }}" </p>
required /> </div>
<button type="button" class="btn btn-primary" @click="updateVolume()">
Update Volume
</button>
</div> </div>
</div> </div>
</div> </div>
<h2 class="mt-4">Current Schedule</h2> <!-- Volume and Schedule Controls -->
<form x-data id="scheduleForm"> <div class="row g-3 mb-3">
<div class="timeline mb-3" @mouseup="endDrag()" @mouseleave="endDrag()"> <!-- Daily Volume Column -->
<template x-for="hour in Object.keys(schedule)" :key="hour"> <div class="col-md-4" x-data="{ volumeValue: volume }">
<div class="hour-block" :id="'hour-' + hour" :data-hour="hour" :style="getBackgroundStyle(hour)" <div class="card h-100">
:class="{'selected': isSelected(hour)}" @mousedown="startDrag($event, hour)" <div class="card-header bg-light">
@mouseover="dragSelect(hour)"> <h5 class="mb-0"><i class="fas fa-chart-line"></i> Daily Volume</h5>
<div><strong x-text="formatHour(hour)"></strong></div>
<div class="weight"><span x-text="schedule[hour]"></span></div>
<div class="papers">
<span x-text="getPapersPerHour(hour)"></span> p.
</div>
<input type="hidden" :name="'hour_' + hour" :value="schedule[hour]" />
</div> </div>
</template> <div class="card-body">
<p class="lead mb-2">
<strong x-text="volume"></strong> papers/day
</p>
<div class="input-group input-group-sm mb-2">
<input type="number" class="form-control" x-model="volumeValue" min="1"
max="{{ max_volume }}" required />
<button type="button" class="btn btn-primary" @click="updateVolume()">
<i class="fas fa-save"></i> Update
</button>
</div>
<small class="text-muted">Range: 1-{{ max_volume }}</small>
</div>
</div>
</div> </div>
<div class="input-group mb-4 w-50"> <!-- Legend Column -->
<label class="input-group-text">Set Weight:</label> <div class="col-md-8">
<input type="number" step="0.1" min="0" max="5" x-model="newWeight" class="form-control" /> <div class="card h-100">
<button type="button" class="btn btn-outline-primary" @click="applyWeight()"> <div class="card-header bg-light">
Apply to Selected <h5 class="mb-0"><i class="fas fa-info-circle"></i> Quick Guide</h5>
</button> </div>
<div class="card-body">
<div class="d-flex align-items-center justify-content-between mb-2">
<div class="d-flex align-items-center">
<div class="weight-gradient me-2"></div>
<small>Darker blocks = higher weight</small>
</div>
<div class="badge bg-info">Formula: (Weight ÷ Total) × Volume</div>
</div>
<div class="d-flex align-items-center small text-muted">
<div class="me-3"><i class="fas fa-mouse-pointer"></i> Click to select hours</div>
<div class="me-3"><i class="fas fa-arrows-alt-h"></i> Drag to select multiple</div>
<div><i class="fas fa-save"></i> Save after changes</div>
</div>
</div>
</div>
</div> </div>
</div>
<div class="d-flex justify-content-between"> <!-- 24-Hour Schedule -->
<a href="{{ url_for('config.general') }}" class="btn btn-outline-secondary">⬅ Back</a> <form id="scheduleForm">
<button type="button" class="btn btn-success" @click="saveSchedule()">💾 Save Schedule</button> <div class="card">
<div class="card-header d-flex justify-content-between align-items-center bg-light">
<h5 class="mb-0"><i class="fas fa-clock"></i> 24-Hour Schedule</h5>
<span class="badge bg-info"
x-text="selectedHours.length ? selectedHours.length + ' hours selected' : ''"></span>
</div>
<div class="card-body">
<div class="timeline mb-3" @mouseup="endDrag()" @mouseleave="endDrag()">
<template x-for="hour in Object.keys(schedule)" :key="hour">
<div class="hour-block" :id="'hour-' + hour" :data-hour="hour"
:style="getBackgroundStyle(hour)" :class="{'selected': isSelected(hour)}"
@mousedown="startDrag($event, hour)" @mouseover="dragSelect(hour)">
<div><strong x-text="formatHour(hour)"></strong></div>
<div class="weight"><span x-text="schedule[hour]"></span></div>
<div class="papers">
<span x-text="getPapersPerHour(hour)"></span> p.
</div>
<input type="hidden" :name="'hour_' + hour" :value="schedule[hour]" />
</div>
</template>
</div>
<div class="input-group mb-2">
<label class="input-group-text"><i class="fas fa-weight"></i> Weight</label>
<input type="number" step="0.1" min="0.1" max="5" x-model="newWeight"
class="form-control" />
<button type="button" class="btn btn-primary" @click="applyWeight()"
:disabled="selectedHours.length === 0">
Apply to <span x-text="selectedHours.length"></span> Selected
</button>
</div>
</div>
<div class="card-footer">
<div class="d-flex justify-content-between">
<a href="{{ url_for('config.general') }}" class="btn btn-outline-secondary">
<i class="fas fa-arrow-left"></i> Back
</a>
<button type="button" class="btn btn-success" @click="saveSchedule()">
<i class="fas fa-save"></i> Save Schedule
</button>
</div>
</div>
</div> </div>
</form> </form>
</div> </div>
</div> </div>
</div> </div>

6
scipaperloader/templates/papers.html.jinja
View File

@ -16,6 +16,12 @@
<label>Status</label> <label>Status</label>
<select name="status" class="form-select"> <select name="status" class="form-select">
<option value="">All</option> <option value="">All</option>
{% if request.args.get('status') == 'New' %}
<option value="New" selected>New</option>
{% else %}
<option value="New">New</option>
{% endif %}
{% if request.args.get('status') == 'Pending' %} {% if request.args.get('status') == 'Pending' %}
<option value="Pending" selected>Pending</option> <option value="Pending" selected>Pending</option>
{% else %} {% else %}