adds a MAX_VOLUME variable to defaults

scipaperloader/blueprints/config.py

@@ -2,6 +2,7 @@
 from flask import Blueprint, render_template, redirect, url_for, request, flash, jsonify
 from ..db import db
 from ..models import VolumeConfig, ScheduleConfig, ActivityLog
+from ..defaults import MAX_VOLUME
 
 bp = Blueprint("config", __name__, url_prefix="/config")
 
@@ -19,8 +20,8 @@ def _update_volume(new_volume):
     """
     try:
         new_volume = float(new_volume)
-        if new_volume <= 0 or new_volume > 1000:
+        if new_volume <= 0 or new_volume > MAX_VOLUME:
-            return False, "Volume must be between 1 and 1000", None
+            return False, f"Volume must be between 1 and {MAX_VOLUME}", None
 
         volume_config = VolumeConfig.query.first()
         if not volume_config:
@@ -110,6 +111,7 @@ def general():
         "config/index.html.jinja",
         active_tab="general",
         volume_config=volume_config,
+        max_volume=MAX_VOLUME,
         app_title="Configuration"
     )
 
@@ -144,6 +146,7 @@ def schedule():
         active_tab="schedule",
         schedule=schedule_config,
         volume=volume_config.volume,
+        max_volume=MAX_VOLUME,
         app_title="Configuration"
     )
 

scipaperloader/blueprints/scraper.py

@@ -2,11 +2,12 @@ import random
 import json
 import time
 import math
-from datetime import datetime
+from datetime import datetime, timedelta
 from flask import Blueprint, jsonify, render_template, request, current_app, flash
 from ..models import VolumeConfig, ActivityLog, PaperMetadata, ActivityCategory, ScheduleConfig
 from ..db import db
 from ..celery import celery
+from ..defaults import MAX_VOLUME
 
 bp = Blueprint("scraper", __name__, url_prefix="/scraper")
 
@@ -29,7 +30,8 @@ def index():
         "scraper.html.jinja", 
         volume_config=volume_config,
         scraper_active=SCRAPER_ACTIVE,
-        scraper_paused=SCRAPER_PAUSED
+        scraper_paused=SCRAPER_PAUSED,
+        max_volume=MAX_VOLUME
     )
 
 @bp.route("/start", methods=["POST"])
@@ -143,20 +145,23 @@ def scraper_stats():
         except ValueError:
             pass
     
-    cutoff_time = datetime.utcnow().replace(
+    current_time = datetime.utcnow()
+    # Use timedelta for proper date calculation instead of simple hour subtraction
+    cutoff_time = (current_time - timedelta(hours=hours)).replace(
         minute=0, second=0, microsecond=0
     )
     
     # Get activity logs for scraper actions
     logs = ActivityLog.query.filter(
         ActivityLog.category == ActivityCategory.SCRAPER_ACTIVITY.value,
-        ActivityLog.timestamp >= cutoff_time.replace(hour=cutoff_time.hour - hours)
+        ActivityLog.timestamp >= cutoff_time
     ).all()
     
     # Group by hour and status
     stats = {}
     for hour in range(hours):
-        target_hour = (cutoff_time.hour - hour) % 24
+        # Calculate the hour as offset from current time
+        target_hour = (current_time.hour - hour) % 24
         stats[target_hour] = {
             "success": 0,
             "error": 0,
@@ -190,10 +195,10 @@ def update_config():
                 new_volume = float(data["volume"])
                 
                 # Validate volume value
-                if new_volume <= 0 or new_volume > 1000:
+                if new_volume <= 0 or new_volume > MAX_VOLUME:
                     return jsonify({
                         "success": False, 
-                        "message": "Volume must be between 1 and 1000"
+                        "message": f"Volume must be between 1 and {MAX_VOLUME}"
                     })
                     
                 volume_config = VolumeConfig.query.first()

scipaperloader/defaults.py

@@ -20,3 +20,6 @@ DUPLICATE_STRATEGIES = {
     #     "is_default": False
     # }
 }
+
+# Configuration limits
+MAX_VOLUME = 100000  # Maximum volume limit for scraper configuration

scipaperloader/templates/config/general.html.jinja

@@ -17,8 +17,8 @@
                         <div class="mb-3">
                             <label for="totalVolume" class="form-label">Papers per day:</label>
                             <input type="number" class="form-control" id="totalVolume" name="total_volume" min="1"
-                                max="1000" value="{{ volume_config.volume }}" required>
+                                max="{{ max_volume }}" value="{{ volume_config.volume }}" required>
-                            <div class="form-text">Enter a value between 1 and 1000</div>
+                            <div class="form-text">Enter a value between 1 and {{ max_volume }}</div>
                         </div>
                     </div>
 

scipaperloader/templates/config/schedule.html.jinja

@@ -80,7 +80,8 @@
                 <div class="d-flex align-items-center mb-3" x-data="{ volumeValue: volume }">
                     <div class="input-group w-50">
                         <label class="input-group-text">Papers per day:</label>
-                        <input type="number" class="form-control" x-model="volumeValue" min="1" max="1000" required />
+                        <input type="number" class="form-control" x-model="volumeValue" min="1" max="{{ max_volume }}"
+                            required />
                         <button type="button" class="btn btn-primary" @click="updateVolume()">
                             Update Volume
                         </button>

scipaperloader/templates/scraper.html.jinja

@@ -77,7 +77,9 @@
                         <div class="form-group">
                             <label for="volumeInput">Papers per day:</label>
                             <input type="number" class="form-control" id="volumeInput"
-                                value="{{ volume_config.volume if volume_config else 100 }}">
+                                value="{{ volume_config.volume if volume_config else 100 }}" min="1"
+                                max="{{ max_volume }}">
+                            <div class="form-text">Enter a value between 1 and {{ max_volume }}</div>
                         </div>
                         <button type="submit" class="btn btn-primary mt-2">Update Volume</button>
                     </form>