From 4085b4746042fb0b2b529d844f9af6dc934ff242 Mon Sep 17 00:00:00 2001 From: Michael Beck Date: Wed, 16 Apr 2025 16:09:26 +0200 Subject: [PATCH] adds a MAX_VOLUME variable to defaults --- scipaperloader/blueprints/config.py | 7 +++++-- scipaperloader/blueprints/scraper.py | 19 ++++++++++++------- scipaperloader/defaults.py | 3 +++ .../templates/config/general.html.jinja | 4 ++-- .../templates/config/schedule.html.jinja | 3 ++- scipaperloader/templates/scraper.html.jinja | 4 +++- 6 files changed, 27 insertions(+), 13 deletions(-) diff --git a/scipaperloader/blueprints/config.py b/scipaperloader/blueprints/config.py index e91c2c7..8d12b45 100644 --- a/scipaperloader/blueprints/config.py +++ b/scipaperloader/blueprints/config.py @@ -2,6 +2,7 @@ from flask import Blueprint, render_template, redirect, url_for, request, flash, jsonify from ..db import db from ..models import VolumeConfig, ScheduleConfig, ActivityLog +from ..defaults import MAX_VOLUME bp = Blueprint("config", __name__, url_prefix="/config") @@ -19,8 +20,8 @@ def _update_volume(new_volume): """ try: new_volume = float(new_volume) - if new_volume <= 0 or new_volume > 1000: - return False, "Volume must be between 1 and 1000", None + if new_volume <= 0 or new_volume > MAX_VOLUME: + return False, f"Volume must be between 1 and {MAX_VOLUME}", None volume_config = VolumeConfig.query.first() if not volume_config: @@ -110,6 +111,7 @@ def general(): "config/index.html.jinja", active_tab="general", volume_config=volume_config, + max_volume=MAX_VOLUME, app_title="Configuration" ) @@ -144,6 +146,7 @@ def schedule(): active_tab="schedule", schedule=schedule_config, volume=volume_config.volume, + max_volume=MAX_VOLUME, app_title="Configuration" ) diff --git a/scipaperloader/blueprints/scraper.py b/scipaperloader/blueprints/scraper.py index fd81d49..8b8f746 100644 --- a/scipaperloader/blueprints/scraper.py +++ b/scipaperloader/blueprints/scraper.py @@ -2,11 +2,12 @@ import random import json import time import math -from datetime import datetime +from datetime import datetime, timedelta from flask import Blueprint, jsonify, render_template, request, current_app, flash from ..models import VolumeConfig, ActivityLog, PaperMetadata, ActivityCategory, ScheduleConfig from ..db import db from ..celery import celery +from ..defaults import MAX_VOLUME bp = Blueprint("scraper", __name__, url_prefix="/scraper") @@ -29,7 +30,8 @@ def index(): "scraper.html.jinja", volume_config=volume_config, scraper_active=SCRAPER_ACTIVE, - scraper_paused=SCRAPER_PAUSED + scraper_paused=SCRAPER_PAUSED, + max_volume=MAX_VOLUME ) @bp.route("/start", methods=["POST"]) @@ -143,20 +145,23 @@ def scraper_stats(): except ValueError: pass - cutoff_time = datetime.utcnow().replace( + current_time = datetime.utcnow() + # Use timedelta for proper date calculation instead of simple hour subtraction + cutoff_time = (current_time - timedelta(hours=hours)).replace( minute=0, second=0, microsecond=0 ) # Get activity logs for scraper actions logs = ActivityLog.query.filter( ActivityLog.category == ActivityCategory.SCRAPER_ACTIVITY.value, - ActivityLog.timestamp >= cutoff_time.replace(hour=cutoff_time.hour - hours) + ActivityLog.timestamp >= cutoff_time ).all() # Group by hour and status stats = {} for hour in range(hours): - target_hour = (cutoff_time.hour - hour) % 24 + # Calculate the hour as offset from current time + target_hour = (current_time.hour - hour) % 24 stats[target_hour] = { "success": 0, "error": 0, @@ -190,10 +195,10 @@ def update_config(): new_volume = float(data["volume"]) # Validate volume value - if new_volume <= 0 or new_volume > 1000: + if new_volume <= 0 or new_volume > MAX_VOLUME: return jsonify({ "success": False, - "message": "Volume must be between 1 and 1000" + "message": f"Volume must be between 1 and {MAX_VOLUME}" }) volume_config = VolumeConfig.query.first() diff --git a/scipaperloader/defaults.py b/scipaperloader/defaults.py index d1955d7..6bce8fb 100644 --- a/scipaperloader/defaults.py +++ b/scipaperloader/defaults.py @@ -20,3 +20,6 @@ DUPLICATE_STRATEGIES = { # "is_default": False # } } + +# Configuration limits +MAX_VOLUME = 100000 # Maximum volume limit for scraper configuration diff --git a/scipaperloader/templates/config/general.html.jinja b/scipaperloader/templates/config/general.html.jinja index ff88fb3..36e431e 100644 --- a/scipaperloader/templates/config/general.html.jinja +++ b/scipaperloader/templates/config/general.html.jinja @@ -17,8 +17,8 @@
-
Enter a value between 1 and 1000
+ max="{{ max_volume }}" value="{{ volume_config.volume }}" required> +
Enter a value between 1 and {{ max_volume }}
diff --git a/scipaperloader/templates/config/schedule.html.jinja b/scipaperloader/templates/config/schedule.html.jinja index 78f8dd8..89346e3 100644 --- a/scipaperloader/templates/config/schedule.html.jinja +++ b/scipaperloader/templates/config/schedule.html.jinja @@ -80,7 +80,8 @@
- + diff --git a/scipaperloader/templates/scraper.html.jinja b/scipaperloader/templates/scraper.html.jinja index cd07ff7..b142ae9 100644 --- a/scipaperloader/templates/scraper.html.jinja +++ b/scipaperloader/templates/scraper.html.jinja @@ -77,7 +77,9 @@
+ value="{{ volume_config.volume if volume_config else 100 }}" min="1" + max="{{ max_volume }}"> +
Enter a value between 1 and {{ max_volume }}