From adf82074612369036c529a71ed8effabaf568e05 Mon Sep 17 00:00:00 2001 From: Michael Beck Date: Wed, 16 Apr 2025 15:19:28 +0200 Subject: [PATCH] redesigns config, including some placeholders --- scipaperloader/blueprints/__init__.py | 4 +- scipaperloader/blueprints/config.py | 255 +++++++++++ scipaperloader/blueprints/schedule.py | 243 ++++++++--- scipaperloader/blueprints/scraper.py | 235 +--------- .../templates/config/general.html.jinja | 49 +++ .../templates/config/index.html.jinja | 50 +++ .../templates/config/schedule.html.jinja | 325 ++++++++++++++ scipaperloader/templates/nav.html.jinja | 2 +- .../partials/flash_messages.html.jinja | 0 scipaperloader/templates/scraper.html.jinja | 412 +++--------------- 10 files changed, 945 insertions(+), 630 deletions(-) create mode 100644 scipaperloader/blueprints/config.py create mode 100644 scipaperloader/templates/config/general.html.jinja create mode 100644 scipaperloader/templates/config/index.html.jinja create mode 100644 scipaperloader/templates/config/schedule.html.jinja create mode 100644 scipaperloader/templates/partials/flash_messages.html.jinja diff --git a/scipaperloader/blueprints/__init__.py b/scipaperloader/blueprints/__init__.py index c6cefe1..0c8474a 100644 --- a/scipaperloader/blueprints/__init__.py +++ b/scipaperloader/blueprints/__init__.py @@ -8,6 +8,7 @@ from .schedule import bp as schedule_bp from .logger import bp as logger_bp from .api import bp as api_bp from .scraper import bp as scraper_bp +from .config import bp as config_bp def register_blueprints(app: Flask): @@ -18,4 +19,5 @@ def register_blueprints(app: Flask): app.register_blueprint(schedule_bp, url_prefix='/schedule') app.register_blueprint(logger_bp, url_prefix='/logs') app.register_blueprint(api_bp, url_prefix='/api') - app.register_blueprint(scraper_bp, url_prefix='/scraper') \ No newline at end of file + app.register_blueprint(scraper_bp, url_prefix='/scraper') + app.register_blueprint(config_bp) \ No newline at end of file diff --git a/scipaperloader/blueprints/config.py b/scipaperloader/blueprints/config.py new file mode 100644 index 0000000..e91c2c7 --- /dev/null +++ b/scipaperloader/blueprints/config.py @@ -0,0 +1,255 @@ +"""Configuration management blueprint.""" +from flask import Blueprint, render_template, redirect, url_for, request, flash, jsonify +from ..db import db +from ..models import VolumeConfig, ScheduleConfig, ActivityLog + +bp = Blueprint("config", __name__, url_prefix="/config") + + +# Helper functions for configuration updates +def _update_volume(new_volume): + """ + Helper function to update volume configuration. + + Args: + new_volume (float): The new volume value + + Returns: + tuple: (success, message, volume_config) + """ + try: + new_volume = float(new_volume) + if new_volume <= 0 or new_volume > 1000: + return False, "Volume must be between 1 and 1000", None + + volume_config = VolumeConfig.query.first() + if not volume_config: + volume_config = VolumeConfig(volume=new_volume) + db.session.add(volume_config) + else: + old_value = volume_config.volume + volume_config.volume = new_volume + ActivityLog.log_config_change( + config_key="scraper_volume", + old_value=old_value, + new_value=new_volume, + description="Updated scraper volume" + ) + + db.session.commit() + return True, "Volume updated successfully!", volume_config + + except (ValueError, TypeError) as e: + db.session.rollback() + return False, f"Error updating volume: {str(e)}", None + + +def _update_schedule(schedule_data): + """ + Helper function to update schedule configuration. + + Args: + schedule_data (dict): Dictionary with hour:weight pairs + + Returns: + tuple: (success, message) + """ + try: + # Validate all entries first + for hour_str, weight in schedule_data.items(): + try: + hour = int(hour_str) + weight = float(weight) + + if hour < 0 or hour > 23: + return False, f"Hour value must be between 0 and 23, got {hour}" + + if weight < 0.1 or weight > 5: + return False, f"Weight for hour {hour} must be between 0.1 and 5, got {weight}" + except ValueError: + return False, f"Invalid data format for hour {hour_str}" + + # Update schedule after validation + for hour_str, weight in schedule_data.items(): + hour = int(hour_str) + weight = float(weight) + + config = ScheduleConfig.query.get(hour) + if not config: + config = ScheduleConfig(hour=hour, weight=weight) + db.session.add(config) + else: + old_value = config.weight + config.weight = weight + ActivityLog.log_config_change( + config_key=f"schedule_hour_{hour}", + old_value=old_value, + new_value=weight, + description=f"Updated schedule weight for hour {hour}" + ) + + db.session.commit() + return True, "Schedule updated successfully!" + + except Exception as e: + db.session.rollback() + return False, f"Error updating schedule: {str(e)}" + + +@bp.route("/") +@bp.route("/general") +def general(): + """Show general configuration page.""" + volume_config = VolumeConfig.query.first() + if not volume_config: + volume_config = VolumeConfig(volume=100) # Default value + db.session.add(volume_config) + db.session.commit() + + return render_template( + "config/index.html.jinja", + active_tab="general", + volume_config=volume_config, + app_title="Configuration" + ) + + +@bp.route("/schedule") +def schedule(): + """Show schedule configuration page.""" + # Ensure we have schedule config for all hours + existing_hours = {record.hour: record for record in ScheduleConfig.query.all()} + schedule_config = {} + + for hour in range(24): + if hour in existing_hours: + schedule_config[hour] = existing_hours[hour].weight + else: + # Create default schedule entry (weight 1.0) + new_config = ScheduleConfig(hour=hour, weight=1.0) + db.session.add(new_config) + schedule_config[hour] = 1.0 + + if len(existing_hours) < 24: + db.session.commit() + + volume_config = VolumeConfig.query.first() + if not volume_config: + volume_config = VolumeConfig(volume=100) # Default value + db.session.add(volume_config) + db.session.commit() + + return render_template( + "config/index.html.jinja", + active_tab="schedule", + schedule=schedule_config, + volume=volume_config.volume, + app_title="Configuration" + ) + + +@bp.route("/update/volume", methods=["POST"]) +def update_volume(): + """Update volume configuration.""" + new_volume = request.form.get("total_volume", 0) + success, message, _ = _update_volume(new_volume) + + if success: + flash(message, "success") + else: + flash(message, "error") + + return redirect(url_for("config.general")) + + +@bp.route("/update/schedule", methods=["POST"]) +def update_schedule(): + """Update schedule configuration.""" + schedule_data = {} + for hour in range(24): + key = f"hour_{hour}" + if key not in request.form: + flash(f"Missing data for hour {hour}", "error") + return redirect(url_for("config.schedule")) + schedule_data[str(hour)] = request.form.get(key, 0) + + success, message = _update_schedule(schedule_data) + + if success: + flash(message, "success") + else: + flash(message, "error") + + return redirect(url_for("config.schedule")) + + +@bp.route("/api/schedule/stats") +def schedule_stats(): + """Get statistics about the current schedule configuration.""" + volume_config = VolumeConfig.query.first() + if not volume_config: + return jsonify({"error": "No volume configuration found"}) + + total_volume = volume_config.volume + schedule_configs = ScheduleConfig.query.all() + + if not schedule_configs: + return jsonify({"error": "No schedule configuration found"}) + + # Calculate total weight + total_weight = sum(config.weight for config in schedule_configs) + + # Calculate papers per hour + papers_per_hour = {} + hourly_weights = {} + for config in schedule_configs: + weight_ratio = config.weight / total_weight if total_weight > 0 else 0 + papers = weight_ratio * total_volume + papers_per_hour[config.hour] = papers + hourly_weights[config.hour] = config.weight + + return jsonify({ + "total_volume": total_volume, + "total_weight": total_weight, + "papers_per_hour": papers_per_hour, + "hourly_weights": hourly_weights + }) + + +@bp.route("/api/update_config", methods=["POST"]) +def api_update_config(): + """API endpoint to update configuration.""" + data = request.json + response = {"success": True, "updates": []} + + try: + # Update volume if provided + if "volume" in data: + success, message, volume_config = _update_volume(data["volume"]) + response["updates"].append({ + "type": "volume", + "success": success, + "message": message + }) + if not success: + response["success"] = False + + # Update schedule if provided + if "schedule" in data: + success, message = _update_schedule(data["schedule"]) + response["updates"].append({ + "type": "schedule", + "success": success, + "message": message + }) + if not success: + response["success"] = False + + return jsonify(response) + + except Exception as e: + db.session.rollback() + return jsonify({ + "success": False, + "message": f"Unexpected error: {str(e)}" + }) \ No newline at end of file diff --git a/scipaperloader/blueprints/schedule.py b/scipaperloader/blueprints/schedule.py index b76c764..86aebc6 100644 --- a/scipaperloader/blueprints/schedule.py +++ b/scipaperloader/blueprints/schedule.py @@ -1,79 +1,212 @@ -"""Schedule configuration routes.""" -from flask import Blueprint, flash, render_template, request +"""Schedule configuration and scheduling logic.""" +from datetime import datetime +import random +import json +from flask import Blueprint, flash, render_template, request, jsonify from ..db import db -from ..models import ScheduleConfig, VolumeConfig +from ..models import ScheduleConfig, VolumeConfig, ActivityLog, ActivityCategory +from ..celery import celery +from .scraper import SCRAPER_ACTIVE, SCRAPER_PAUSED, dummy_scrape_paper +from .config import _update_volume, _update_schedule -bp = Blueprint("schedule", __name__) +bp = Blueprint("schedule", __name__, url_prefix="/schedule") @bp.route("/", methods=["GET", "POST"]) def schedule(): + """Render and handle the schedule configuration page.""" if request.method == "POST": try: # Check if we're updating volume or schedule if "total_volume" in request.form: - # Volume update - try: - new_volume = float(request.form.get("total_volume", 0)) - if new_volume <= 0 or new_volume > 1000: - raise ValueError("Volume must be between 1 and 1000") - - volume_config = VolumeConfig.query.first() - if not volume_config: - volume_config = VolumeConfig(volume=new_volume) - db.session.add(volume_config) - else: - volume_config.volume = new_volume - - db.session.commit() - flash("Volume updated successfully!", "success") - - except ValueError as e: - db.session.rollback() - flash(f"Error updating volume: {str(e)}", "error") + # Volume update using the centralized helper + new_volume = request.form.get("total_volume", 0) + success, message, _ = _update_volume(new_volume) + + if success: + flash(message, "success") + else: + flash(message, "error") else: - # Schedule update logic - # Validate form data + # Schedule update using the centralized helper + schedule_data = {} for hour in range(24): key = f"hour_{hour}" if key not in request.form: - raise ValueError(f"Missing data for hour {hour}") - - try: - weight = float(request.form.get(key, 0)) - if weight < 0 or weight > 5: - raise ValueError( - f"Weight for hour {hour} must be between 0 and 5" - ) - except ValueError: - raise ValueError(f"Invalid weight value for hour {hour}") - - # Update database if validation passes - for hour in range(24): - key = f"hour_{hour}" - weight = float(request.form.get(key, 0)) - config = ScheduleConfig.query.get(hour) - if config: - config.weight = weight + flash(f"Missing data for hour {hour}", "error") + break + schedule_data[str(hour)] = request.form.get(key, 0) + + if len(schedule_data) == 24: + success, message = _update_schedule(schedule_data) + if success: + flash(message, "success") else: - db.session.add(ScheduleConfig(hour=hour, weight=weight)) + flash(message, "error") - db.session.commit() - flash("Schedule updated successfully!", "success") - - except ValueError as e: + except Exception as e: db.session.rollback() - flash(f"Error updating schedule: {str(e)}", "error") + flash(f"Error: {str(e)}", "error") + + # Ensure we have schedule config for all hours + existing_hours = {record.hour: record for record in ScheduleConfig.query.all()} + schedule_config = {} + + for hour in range(24): + if hour in existing_hours: + schedule_config[hour] = existing_hours[hour].weight + else: + # Create default schedule entry (weight 1.0) + new_config = ScheduleConfig(hour=hour, weight=1.0) + db.session.add(new_config) + schedule_config[hour] = 1.0 + + if len(existing_hours) < 24: + db.session.commit() - schedule = { - sc.hour: sc.weight - for sc in ScheduleConfig.query.order_by(ScheduleConfig.hour).all() - } volume = VolumeConfig.query.first() return render_template( "schedule.html.jinja", - schedule=schedule, + schedule=schedule_config, volume=volume.volume if volume else 0, + stats=get_schedule_stats(), app_title="PaperScraper", - ) \ No newline at end of file + ) + + +@bp.route("/update_config", methods=["POST"]) +def update_config(): + """Update schedule configuration via API.""" + data = request.json + response = {"success": True, "updates": []} + + try: + # Update volume if provided + if "volume" in data: + success, message, _ = _update_volume(data["volume"]) + response["updates"].append({ + "type": "volume", + "success": success, + "message": message + }) + if not success: + response["success"] = False + + # Update schedule if provided + if "schedule" in data: + success, message = _update_schedule(data["schedule"]) + response["updates"].append({ + "type": "schedule", + "success": success, + "message": message + }) + if not success: + response["success"] = False + + return jsonify(response) + + except Exception as e: + db.session.rollback() + return jsonify({ + "success": False, + "message": f"Unexpected error: {str(e)}" + }) + + +# Calculate schedule information for visualization/decision making +def get_schedule_stats(): + """Get statistics about the current schedule configuration.""" + volume_config = VolumeConfig.query.first() + if not volume_config: + return {"error": "No volume configuration found"} + + total_volume = volume_config.volume + schedule_configs = ScheduleConfig.query.all() + + if not schedule_configs: + return {"error": "No schedule configuration found"} + + # Calculate total weight + total_weight = sum(config.weight for config in schedule_configs) + + # Calculate papers per hour + papers_per_hour = {} + hourly_weights = {} + for config in schedule_configs: + weight_ratio = config.weight / total_weight if total_weight > 0 else 0 + papers = weight_ratio * total_volume + papers_per_hour[config.hour] = papers + hourly_weights[config.hour] = config.weight + + return { + "total_volume": total_volume, + "total_weight": total_weight, + "papers_per_hour": papers_per_hour, + "hourly_weights": hourly_weights + } + + +# API route to get schedule information +@bp.route("/schedule_info") +def schedule_info(): + """Get information about the current schedule configuration.""" + stats = get_schedule_stats() + return jsonify(stats) + + +# Define the Celery tasks for the scheduler +@celery.task(bind=True) +def start_scheduler(self): + """Start the scheduler when the scraper is started.""" + if SCRAPER_ACTIVE and not SCRAPER_PAUSED: + # Schedule the first run immediately + scheduler_task.delay() + return {"status": "success", "message": "Scheduler started"} + return {"status": "error", "message": "Scraper not active or paused"} + + +@celery.task(bind=True) +def scheduler_task(self): + """Main scheduler task for the scraper.""" + if not SCRAPER_ACTIVE: + return {"status": "Scraper not active"} + + if SCRAPER_PAUSED: + return {"status": "Scraper paused"} + + # Calculate how many papers to scrape based on current hour and configuration + current_hour = datetime.now().hour + hour_config = ScheduleConfig.query.get(current_hour) + volume_config = VolumeConfig.query.first() + + if not hour_config or not volume_config: + return {"status": "Missing configuration"} + + # Calculate papers to scrape this hour + stats = get_schedule_stats() + papers_to_scrape = int(stats["papers_per_hour"].get(current_hour, 0)) + + # Log the scheduling decision + ActivityLog.log_scraper_activity( + action="schedule_papers", + status="success", + description=f"Scheduled {papers_to_scrape} papers for scraping at hour {current_hour}", + extra_data=json.dumps({ + "hour": current_hour, + "weight": hour_config.weight, + "total_volume": volume_config.volume + }) + ) + + # Execute the actual scraping tasks + for _ in range(papers_to_scrape): + # Queue up scraping tasks - in real implementation, this would + # call the actual scraper task + dummy_scrape_paper.delay() + + return { + "status": "success", + "papers_scheduled": papers_to_scrape, + "hour": current_hour + } \ No newline at end of file diff --git a/scipaperloader/blueprints/scraper.py b/scipaperloader/blueprints/scraper.py index a5ae271..b8cb66b 100644 --- a/scipaperloader/blueprints/scraper.py +++ b/scipaperloader/blueprints/scraper.py @@ -2,7 +2,7 @@ import random import json from datetime import datetime from flask import Blueprint, jsonify, render_template, request, current_app, flash -from ..models import ScheduleConfig, VolumeConfig, ActivityLog, PaperMetadata, ActivityCategory +from ..models import VolumeConfig, ActivityLog, PaperMetadata, ActivityCategory from ..db import db from ..celery import celery @@ -23,26 +23,9 @@ def index(): db.session.add(volume_config) db.session.commit() - # Ensure we have schedule config for all hours - existing_hours = {record.hour: record for record in ScheduleConfig.query.all()} - schedule_config = {} - - for hour in range(24): - if hour in existing_hours: - schedule_config[hour] = existing_hours[hour].weight - else: - # Create default schedule entry (weight 1.0) - new_config = ScheduleConfig(hour=hour, weight=1.0) - db.session.add(new_config) - schedule_config[hour] = 1.0 - - if len(existing_hours) < 24: - db.session.commit() - return render_template( "scraper.html.jinja", volume_config=volume_config, - schedule_config=schedule_config, scraper_active=SCRAPER_ACTIVE, scraper_paused=SCRAPER_PAUSED ) @@ -63,13 +46,10 @@ def start_scraper(): description="Scraper started manually" ) - # Start the scheduler task - task = dummy_scraper_scheduler.delay() - + # Trigger the schedule.py to start actual scheduling return jsonify({ "success": True, - "message": "Scraper started", - "task_id": task.id + "message": "Scraper started" }) else: return jsonify({ @@ -205,7 +185,7 @@ def update_config(): try: new_volume = float(data["volume"]) - # Validate volume value (from schedule.py) + # Validate volume value if new_volume <= 0 or new_volume > 1000: return jsonify({ "success": False, @@ -233,219 +213,12 @@ def update_config(): "message": "Invalid volume value" }) - if "schedule" in data: - try: - schedule = data["schedule"] - - # Validate entire schedule - for hour_str, weight in schedule.items(): - try: - hour = int(hour_str) - weight = float(weight) - - if hour < 0 or hour > 23: - return jsonify({ - "success": False, - "message": f"Hour value must be between 0 and 23, got {hour}" - }) - - if weight < 0.1 or weight > 5: - return jsonify({ - "success": False, - "message": f"Weight for hour {hour} must be between 0.1 and 5, got {weight}" - }) - except ValueError: - return jsonify({ - "success": False, - "message": f"Invalid data format for hour {hour_str}" - }) - - # Update schedule after validation - for hour_str, weight in schedule.items(): - hour = int(hour_str) - weight = float(weight) - - schedule_config = ScheduleConfig.query.get(hour) - if not schedule_config: - schedule_config = ScheduleConfig(hour=hour, weight=weight) - db.session.add(schedule_config) - else: - old_value = schedule_config.weight - schedule_config.weight = weight - ActivityLog.log_config_change( - config_key=f"schedule_hour_{hour}", - old_value=old_value, - new_value=weight, - description=f"Updated schedule weight for hour {hour}" - ) - - db.session.commit() - except Exception as e: - db.session.rollback() - return jsonify({ - "success": False, - "message": f"Error updating schedule: {str(e)}" - }) - return jsonify({"success": True, "message": "Configuration updated"}) except Exception as e: db.session.rollback() return jsonify({"success": False, "message": f"Unexpected error: {str(e)}"}) -@bp.route("/schedule", methods=["GET", "POST"]) -def schedule(): - """Legacy route to maintain compatibility with the schedule blueprint.""" - # For GET requests, redirect to the scraper index with the schedule tab active - if request.method == "GET": - return index() - - # For POST requests, handle form data and process like the original schedule blueprint - if request.method == "POST": - try: - # Check if we're updating volume or schedule - if "total_volume" in request.form: - # Volume update - try: - new_volume = float(request.form.get("total_volume", 0)) - if new_volume <= 0 or new_volume > 1000: - raise ValueError("Volume must be between 1 and 1000") - - volume_config = VolumeConfig.query.first() - if not volume_config: - volume_config = VolumeConfig(volume=new_volume) - db.session.add(volume_config) - else: - volume_config.volume = new_volume - - db.session.commit() - flash("Volume updated successfully!", "success") - - except ValueError as e: - db.session.rollback() - flash(f"Error updating volume: {str(e)}", "error") - else: - # Schedule update logic - # Validate form data - for hour in range(24): - key = f"hour_{hour}" - if key not in request.form: - raise ValueError(f"Missing data for hour {hour}") - - try: - weight = float(request.form.get(key, 0)) - if weight < 0 or weight > 5: - raise ValueError( - f"Weight for hour {hour} must be between 0 and 5" - ) - except ValueError: - raise ValueError(f"Invalid weight value for hour {hour}") - - # Update database if validation passes - for hour in range(24): - key = f"hour_{hour}" - weight = float(request.form.get(key, 0)) - config = ScheduleConfig.query.get(hour) - if config: - config.weight = weight - else: - db.session.add(ScheduleConfig(hour=hour, weight=weight)) - - db.session.commit() - flash("Schedule updated successfully!", "success") - - except ValueError as e: - db.session.rollback() - flash(f"Error updating schedule: {str(e)}", "error") - - # Redirect back to the scraper page - return index() - -# Calculate schedule information for visualization/decision making -def get_schedule_stats(): - """Get statistics about the current schedule configuration.""" - volume_config = VolumeConfig.query.first() - if not volume_config: - return {"error": "No volume configuration found"} - - total_volume = volume_config.volume - schedule_configs = ScheduleConfig.query.all() - - if not schedule_configs: - return {"error": "No schedule configuration found"} - - # Calculate total weight - total_weight = sum(config.weight for config in schedule_configs) - - # Calculate papers per hour - papers_per_hour = {} - for config in schedule_configs: - weight_ratio = config.weight / total_weight if total_weight > 0 else 0 - papers = weight_ratio * total_volume - papers_per_hour[config.hour] = papers - - return { - "total_volume": total_volume, - "total_weight": total_weight, - "papers_per_hour": papers_per_hour - } - -# Enhanced API route to get schedule information -@bp.route("/schedule_info") -def schedule_info(): - """Get information about the current schedule configuration.""" - stats = get_schedule_stats() - return jsonify(stats) - -# Define the Celery tasks -@celery.task(bind=True) -def dummy_scraper_scheduler(self): - """Main scheduler task for the dummy scraper.""" - global SCRAPER_ACTIVE, SCRAPER_PAUSED - - if not SCRAPER_ACTIVE: - return {"status": "Scraper not active"} - - if SCRAPER_PAUSED: - return {"status": "Scraper paused"} - - # Calculate how many papers to scrape based on current hour and configuration - current_hour = datetime.now().hour - hour_config = ScheduleConfig.query.get(current_hour) - volume_config = VolumeConfig.query.first() - - if not hour_config or not volume_config: - return {"status": "Missing configuration"} - - # Calculate papers to scrape this hour - hourly_rate = volume_config.volume / 24 # Base rate per hour - adjusted_rate = hourly_rate * (1 / hour_config.weight) # Adjust by weight - papers_to_scrape = int(adjusted_rate) - - # Log the scheduling decision - ActivityLog.log_scraper_activity( - action="schedule_papers", - status="success", - description=f"Scheduled {papers_to_scrape} papers for scraping at hour {current_hour}", - hourly_rate=hourly_rate, - weight=hour_config.weight, - adjusted_rate=adjusted_rate, - ) - - # Launch individual scraping tasks - for _ in range(papers_to_scrape): - if not SCRAPER_ACTIVE or SCRAPER_PAUSED: - break - - # Schedule a new paper to be scraped - dummy_scrape_paper.delay() - - # Schedule the next run in 5 minutes if still active - if SCRAPER_ACTIVE: - dummy_scraper_scheduler.apply_async(countdown=300) # 5 minutes - - return {"status": "success", "papers_scheduled": papers_to_scrape} - @celery.task(bind=True) def dummy_scrape_paper(self): """Simulate scraping a single paper.""" diff --git a/scipaperloader/templates/config/general.html.jinja b/scipaperloader/templates/config/general.html.jinja new file mode 100644 index 0000000..ff88fb3 --- /dev/null +++ b/scipaperloader/templates/config/general.html.jinja @@ -0,0 +1,49 @@ + +
+
+
+
+
General Configuration
+
+
+ + {% include "partials/flash_messages.html.jinja" %} + +
+
+
Scraper Volume
+

Configure the total number of papers to scrape per day.

+ +
+ + +
Enter a value between 1 and 1000
+
+
+ +
+
System Settings
+

Configure general system behavior.

+ +
+ + +
+ +
+ + +
+
+ + +
+
+
+
+
\ No newline at end of file diff --git a/scipaperloader/templates/config/index.html.jinja b/scipaperloader/templates/config/index.html.jinja new file mode 100644 index 0000000..0fa3d93 --- /dev/null +++ b/scipaperloader/templates/config/index.html.jinja @@ -0,0 +1,50 @@ +{% extends "base.html.jinja" %} + +{% block title %}Configuration{% endblock title %} + +{% block styles %} +{{ super() }} + +{% endblock styles %} + +{% block content %} +
+

Configuration

+ + + +
+ {% if active_tab == 'general' %} + {% include "config/general.html.jinja" %} + {% elif active_tab == 'schedule' %} + {% include "config/schedule.html.jinja" %} + {% endif %} +
+
+{% endblock content %} \ No newline at end of file diff --git a/scipaperloader/templates/config/schedule.html.jinja b/scipaperloader/templates/config/schedule.html.jinja new file mode 100644 index 0000000..8c1aa67 --- /dev/null +++ b/scipaperloader/templates/config/schedule.html.jinja @@ -0,0 +1,325 @@ + + + + +
+
+
+
Scheduling Configuration
+
+
+ + + {% include "partials/flash_messages.html.jinja" %} + + +
+

How it Works

+

+ This page allows you to configure the daily volume of papers to be + downloaded and the hourly download weights for the papers. The weights + determine how many papers will be downloaded during each hour of the day. + The total volume ( papers/day) is split + across all hours based on their relative weights. Each weight controls the + proportion of papers downloaded during that hour. Click to select one or + more hours below. Then assign a weight to them using the input and apply + it. Color indicates relative intensity. The total daily volume will be + split proportionally across these weights. + Don't forget to submit the changes! +

+

Example

+

+ If the total volume is 240 papers and hours are + weighted as 1.0, 2.0, and 3.0, they will receive + 40, 80, and 120 papers respectively. +

+
+ +

Volume

+ +
+

+ The total volume of data to be downloaded each day is + papers. +

+
+
+ + + +
+
+
+ +

Current Schedule

+
+
+ +
+ +
+ + + +
+ +
+ ⬅ Back + +
+
+
+
+
+ + \ No newline at end of file diff --git a/scipaperloader/templates/nav.html.jinja b/scipaperloader/templates/nav.html.jinja index a0dd9b4..1adda95 100644 --- a/scipaperloader/templates/nav.html.jinja +++ b/scipaperloader/templates/nav.html.jinja @@ -17,7 +17,7 @@ Papers
  • - Schedule + Configuration
  • {% endblock styles %} @@ -88,193 +43,97 @@

    Paper Scraper Control Panel

    - - - -
    - -
    -
    -
    -
    -
    -
    Scraper Status
    -
    -
    -
    -
    - Inactive -
    - -
    - - - -
    -
    -
    +
    +
    +
    +
    +
    Scraper Status
    - -
    -
    -
    -
    Volume Configuration
    -
    -
    -
    -
    - - -
    - -
    -
    +
    +
    +
    + Inactive
    -
    -
    -
    -
    -
    -
    -
    Scraping Activity
    -
    -
    - - -
    -
    -
    -
    -
    - - - -
    -
    -
    -
    -
    -
    - -
    -
    -
    -
    -
    Recent Activity
    -
    -
    -
    - - - - - - - - - - - - - - -
    TimeActionStatusDescription
    Loading activities...
    -
    -
    +
    + + +
    - -
    - -
    -

    How it Works

    -

    - Configure the daily volume of papers to be downloaded and the hourly download weights. - The weights determine how many papers will be downloaded during each hour of the day. - The total volume ( papers/day) is split across all hours based on - their relative weights. - Lower weights result in higher scraping rates for that hour. -

    -
    Instructions:
    -

    - Click to select one or more hours below. Then assign a weight to them using the input and apply it. - Color indicates relative intensity. Changes are saved immediately when you click "Update Schedule". -

    -
    - -
    +
    +
    -

    Volume Configuration

    +
    Volume Configuration
    -

    - The total volume of data to be downloaded each day is - papers. -

    -
    -
    - Papers per day: - - +
    +
    + + +
    + +
    +
    +
    +
    +
    + +
    +
    +
    +
    +
    Scraping Activity
    +
    +
    + +
    +
    +
    + + + +
    +
    +
    +
    +
    +
    +
    -

    Hourly Weights

    +
    Recent Activity
    -
    - +
    + + + + + + + + + + + + + + +
    TimeActionStatusDescription
    Loading activities...
    - -
    - Set Weight: - - -
    - -
    @@ -287,138 +146,7 @@ {% block scripts %} {{ super() }} - -