755 lines
28 KiB
Django/Jinja
755 lines
28 KiB
Django/Jinja
{% extends "base.html.jinja" %}
|
|
|
|
{% block title %}Paper Scraper Control Panel{% endblock title %}
|
|
|
|
{% block styles %}
|
|
{{ super() }}
|
|
<style>
|
|
.status-indicator {
|
|
width: 15px;
|
|
height: 15px;
|
|
border-radius: 50%;
|
|
display: inline-block;
|
|
margin-right: 5px;
|
|
}
|
|
|
|
.status-active {
|
|
background-color: #28a745;
|
|
}
|
|
|
|
.status-paused {
|
|
background-color: #ffc107;
|
|
}
|
|
|
|
.status-inactive {
|
|
background-color: #dc3545;
|
|
}
|
|
|
|
.stats-chart {
|
|
height: 400px;
|
|
}
|
|
|
|
.notification {
|
|
position: fixed;
|
|
bottom: 20px;
|
|
right: 20px;
|
|
max-width: 350px;
|
|
z-index: 1050;
|
|
}
|
|
|
|
/* Enhanced scheduler styles */
|
|
.timeline {
|
|
display: flex;
|
|
flex-wrap: wrap;
|
|
gap: 3px;
|
|
user-select: none;
|
|
}
|
|
|
|
.hour-block {
|
|
width: 49px;
|
|
height: 70px;
|
|
border-radius: 5px;
|
|
text-align: center;
|
|
line-height: 1.2;
|
|
font-size: 0.9rem;
|
|
padding-top: 6px;
|
|
cursor: pointer;
|
|
user-select: none;
|
|
transition: background-color 0.2s ease-in-out;
|
|
margin: 1px;
|
|
}
|
|
|
|
.hour-block.selected {
|
|
outline: 2px solid #4584b8;
|
|
}
|
|
|
|
.papers {
|
|
font-size: 0.7rem;
|
|
margin-top: 2px;
|
|
}
|
|
|
|
/* Tab styles */
|
|
.nav-tabs .nav-link {
|
|
color: #495057;
|
|
}
|
|
|
|
.nav-tabs .nav-link.active {
|
|
font-weight: bold;
|
|
color: #007bff;
|
|
}
|
|
|
|
.tab-pane {
|
|
padding-top: 1rem;
|
|
}
|
|
</style>
|
|
{% endblock styles %}
|
|
|
|
{% block content %}
|
|
<div class="container mt-4">
|
|
<h1>Paper Scraper Control Panel</h1>
|
|
|
|
<!-- Navigation tabs -->
|
|
<ul class="nav nav-tabs mb-4" id="scraperTabs" role="tablist">
|
|
<li class="nav-item" role="presentation">
|
|
<button class="nav-link active" id="dashboard-tab" data-bs-toggle="tab" data-bs-target="#dashboard"
|
|
type="button" role="tab" aria-controls="dashboard" aria-selected="true">
|
|
Dashboard
|
|
</button>
|
|
</li>
|
|
<li class="nav-item" role="presentation">
|
|
<button class="nav-link" id="schedule-tab" data-bs-toggle="tab" data-bs-target="#schedule" type="button"
|
|
role="tab" aria-controls="schedule" aria-selected="false">
|
|
Schedule Configuration
|
|
</button>
|
|
</li>
|
|
</ul>
|
|
|
|
<div class="tab-content" id="scraperTabsContent">
|
|
<!-- Dashboard Tab -->
|
|
<div class="tab-pane fade show active" id="dashboard" role="tabpanel" aria-labelledby="dashboard-tab">
|
|
<div class="row mb-4">
|
|
<div class="col-md-6">
|
|
<div class="card">
|
|
<div class="card-header">
|
|
<h5>Scraper Status</h5>
|
|
</div>
|
|
<div class="card-body">
|
|
<div class="d-flex align-items-center mb-3">
|
|
<div id="statusIndicator" class="status-indicator status-inactive"></div>
|
|
<span id="statusText">Inactive</span>
|
|
</div>
|
|
|
|
<div class="btn-group" role="group">
|
|
<button id="startButton" class="btn btn-success">Start</button>
|
|
<button id="pauseButton" class="btn btn-warning" disabled>Pause</button>
|
|
<button id="stopButton" class="btn btn-danger" disabled>Stop</button>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<div class="col-md-6">
|
|
<div class="card">
|
|
<div class="card-header">
|
|
<h5>Volume Configuration</h5>
|
|
</div>
|
|
<div class="card-body">
|
|
<form id="volumeForm">
|
|
<div class="form-group">
|
|
<label for="volumeInput">Papers per day:</label>
|
|
<input type="number" class="form-control" id="volumeInput"
|
|
value="{{ volume_config.volume if volume_config else 100 }}">
|
|
</div>
|
|
<button type="submit" class="btn btn-primary mt-2">Update Volume</button>
|
|
</form>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<div class="row mb-4">
|
|
<div class="col-12">
|
|
<div class="card">
|
|
<div class="card-header d-flex justify-content-between align-items-center">
|
|
<h5>Scraping Activity</h5>
|
|
<div>
|
|
<div class="form-check form-switch">
|
|
<input class="form-check-input" type="checkbox" id="notificationsToggle" checked>
|
|
<label class="form-check-label" for="notificationsToggle">Show Notifications</label>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="card-body">
|
|
<div class="btn-group mb-3">
|
|
<button class="btn btn-outline-secondary time-range-btn" data-hours="6">Last 6
|
|
hours</button>
|
|
<button class="btn btn-outline-secondary time-range-btn active" data-hours="24">Last 24
|
|
hours</button>
|
|
<button class="btn btn-outline-secondary time-range-btn" data-hours="72">Last 3
|
|
days</button>
|
|
</div>
|
|
<div class="stats-chart" id="activityChart"></div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<div class="row mb-4">
|
|
<div class="col-12">
|
|
<div class="card">
|
|
<div class="card-header">
|
|
<h5>Recent Activity</h5>
|
|
</div>
|
|
<div class="card-body">
|
|
<div class="table-responsive">
|
|
<table class="table table-striped">
|
|
<thead>
|
|
<tr>
|
|
<th>Time</th>
|
|
<th>Action</th>
|
|
<th>Status</th>
|
|
<th>Description</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody id="activityLog">
|
|
<tr>
|
|
<td colspan="4" class="text-center">Loading activities...</td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- Schedule Configuration Tab -->
|
|
<div class="tab-pane fade" id="schedule" role="tabpanel" aria-labelledby="schedule-tab"
|
|
x-data="scheduleManager({{ schedule_config | tojson }}, {{ volume_config.volume if volume_config else 100 }})">
|
|
|
|
<div class="mb-3">
|
|
<h3>How it Works</h3>
|
|
<p class="text-muted mb-0">
|
|
Configure the daily volume of papers to be downloaded and the hourly download weights.
|
|
The weights determine how many papers will be downloaded during each hour of the day.
|
|
The total volume (<strong x-text="volume"></strong> papers/day) is split across all hours based on
|
|
their relative weights.
|
|
<strong>Lower weights result in higher scraping rates</strong> for that hour.
|
|
</p>
|
|
<h5 class="mt-3">Instructions:</h5>
|
|
<p class="text-muted">
|
|
Click to select one or more hours below. Then assign a weight to them using the input and apply it.
|
|
Color indicates relative intensity. Changes are saved immediately when you click "Update Schedule".
|
|
</p>
|
|
</div>
|
|
|
|
<div class="card mb-4">
|
|
<div class="card-header">
|
|
<h4 class="m-0">Volume Configuration</h4>
|
|
</div>
|
|
<div class="card-body">
|
|
<p class="text-muted">
|
|
The total volume of data to be downloaded each day is
|
|
<strong x-text="volume"></strong> papers.
|
|
</p>
|
|
<div class="d-flex align-items-center mb-3">
|
|
<div class="input-group">
|
|
<span class="input-group-text">Papers per day:</span>
|
|
<input type="number" class="form-control" x-model="volume" min="1" max="1000" />
|
|
<button type="button" class="btn btn-primary" @click="updateVolume()">
|
|
Update Volume
|
|
</button>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<div class="card">
|
|
<div class="card-header">
|
|
<h4 class="m-0">Hourly Weights</h4>
|
|
</div>
|
|
<div class="card-body">
|
|
<div class="timeline mb-3" @mouseup="endDrag()" @mouseleave="endDrag()">
|
|
<template x-for="hour in Object.keys(schedule)" :key="hour">
|
|
<div class="hour-block" :id="'hour-' + hour" :data-hour="hour"
|
|
:style="getBackgroundStyle(hour)" :class="{'selected': isSelected(hour)}"
|
|
@mousedown="startDrag($event, hour)" @mouseover="dragSelect(hour)">
|
|
<div><strong x-text="formatHour(hour)"></strong></div>
|
|
<div class="weight"><span x-text="schedule[hour]"></span></div>
|
|
<div class="papers">
|
|
<span x-text="getPapersPerHour(hour)"></span> p.
|
|
</div>
|
|
</div>
|
|
</template>
|
|
</div>
|
|
|
|
<div class="input-group mb-4 w-50">
|
|
<span class="input-group-text">Set Weight:</span>
|
|
<input type="number" step="0.1" min="0.1" max="5" x-model="newWeight" class="form-control" />
|
|
<button type="button" class="btn btn-outline-primary" @click="applyWeight()">
|
|
Apply to Selected
|
|
</button>
|
|
</div>
|
|
|
|
<button type="button" class="btn btn-success" @click="updateSchedule()">
|
|
💾 Update Schedule
|
|
</button>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- Notification template -->
|
|
<div id="notificationContainer"></div>
|
|
{% endblock content %}
|
|
|
|
{% block scripts %}
|
|
{{ super() }}
|
|
<script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
|
|
<script src="https://cdn.jsdelivr.net/npm/alpinejs@3.x.x/dist/cdn.min.js" defer></script>
|
|
<script>
|
|
// Alpine.js scheduler component
|
|
function scheduleManager(initial, volume) {
|
|
return {
|
|
schedule: initial || {},
|
|
volume: volume,
|
|
selectedHours: [],
|
|
newWeight: 1.0,
|
|
isDragging: false,
|
|
dragOperation: null,
|
|
|
|
formatHour(h) {
|
|
return String(h).padStart(2, "0") + ":00";
|
|
},
|
|
|
|
getBackgroundStyle(hour) {
|
|
const weight = parseFloat(this.schedule[hour]);
|
|
const maxWeight = 2.5; // You can adjust this
|
|
|
|
// Normalize weight (0.0 to 1.0)
|
|
const t = Math.min(weight / maxWeight, 1.0);
|
|
|
|
// Interpolate HSL lightness: 95% (light) to 30% (dark)
|
|
const lightness = 95 - t * 65; // 95 → 30
|
|
const backgroundColor = `hsl(210, 10%, ${lightness}%)`;
|
|
|
|
const textColor = t > 0.65 ? "white" : "black"; // adaptive text color
|
|
|
|
return {
|
|
backgroundColor,
|
|
color: textColor,
|
|
};
|
|
},
|
|
|
|
startDrag(event, hour) {
|
|
event.preventDefault();
|
|
this.isDragging = true;
|
|
this.dragOperation = this.isSelected(hour) ? "remove" : "add";
|
|
this.toggleSelect(hour);
|
|
},
|
|
|
|
dragSelect(hour) {
|
|
if (!this.isDragging) return;
|
|
const selected = this.isSelected(hour);
|
|
if (this.dragOperation === "add" && !selected) {
|
|
this.selectedHours.push(hour);
|
|
} else if (this.dragOperation === "remove" && selected) {
|
|
this.selectedHours = this.selectedHours.filter((h) => h !== hour);
|
|
}
|
|
},
|
|
|
|
endDrag() {
|
|
this.isDragging = false;
|
|
},
|
|
|
|
toggleSelect(hour) {
|
|
if (this.isSelected(hour)) {
|
|
this.selectedHours = this.selectedHours.filter((h) => h !== hour);
|
|
} else {
|
|
this.selectedHours.push(hour);
|
|
}
|
|
},
|
|
|
|
isSelected(hour) {
|
|
return this.selectedHours.includes(hour);
|
|
},
|
|
|
|
applyWeight() {
|
|
this.selectedHours.forEach((hour) => {
|
|
this.schedule[hour] = parseFloat(this.newWeight).toFixed(1);
|
|
});
|
|
},
|
|
|
|
getTotalWeight() {
|
|
return Object.values(this.schedule).reduce(
|
|
(sum, w) => sum + parseFloat(w),
|
|
0
|
|
);
|
|
},
|
|
|
|
getPapersPerHour(hour) {
|
|
const total = this.getTotalWeight();
|
|
if (total === 0) return 0;
|
|
return (
|
|
(parseFloat(this.schedule[hour]) / total) *
|
|
this.volume
|
|
).toFixed(1);
|
|
},
|
|
|
|
updateVolume() {
|
|
fetch('/scraper/update_config', {
|
|
method: 'POST',
|
|
headers: {
|
|
'Content-Type': 'application/json'
|
|
},
|
|
body: JSON.stringify({ volume: parseFloat(this.volume) })
|
|
})
|
|
.then(response => response.json())
|
|
.then(data => {
|
|
if (data.success) {
|
|
showNotification('Volume updated successfully', 'success');
|
|
// Update the volume in the dashboard tab too
|
|
document.getElementById('volumeInput').value = this.volume;
|
|
} else {
|
|
showNotification(data.message, 'danger');
|
|
}
|
|
});
|
|
},
|
|
|
|
updateSchedule() {
|
|
fetch('/scraper/update_config', {
|
|
method: 'POST',
|
|
headers: {
|
|
'Content-Type': 'application/json'
|
|
},
|
|
body: JSON.stringify({ schedule: this.schedule })
|
|
})
|
|
.then(response => response.json())
|
|
.then(data => {
|
|
if (data.success) {
|
|
showNotification('Schedule updated successfully', 'success');
|
|
this.selectedHours = []; // Clear selections after update
|
|
} else {
|
|
showNotification(data.message, 'danger');
|
|
}
|
|
});
|
|
}
|
|
};
|
|
}
|
|
|
|
// Global variables for the scraper dashboard
|
|
let notificationsEnabled = true;
|
|
let activityChart = null;
|
|
let currentTimeRange = 24;
|
|
|
|
// DOM elements
|
|
const statusIndicator = document.getElementById('statusIndicator');
|
|
const statusText = document.getElementById('statusText');
|
|
const startButton = document.getElementById('startButton');
|
|
const pauseButton = document.getElementById('pauseButton');
|
|
const stopButton = document.getElementById('stopButton');
|
|
const notificationsToggle = document.getElementById('notificationsToggle');
|
|
const activityLog = document.getElementById('activityLog');
|
|
|
|
// Initialize the page
|
|
document.addEventListener('DOMContentLoaded', function () {
|
|
initStatusPolling();
|
|
loadActivityStats(currentTimeRange);
|
|
loadRecentActivity();
|
|
|
|
// Initialize event listeners
|
|
startButton.addEventListener('click', startScraper);
|
|
pauseButton.addEventListener('click', togglePauseScraper);
|
|
stopButton.addEventListener('click', stopScraper);
|
|
notificationsToggle.addEventListener('click', toggleNotifications);
|
|
|
|
document.getElementById('volumeForm').addEventListener('submit', function (e) {
|
|
e.preventDefault();
|
|
updateVolume();
|
|
});
|
|
|
|
document.querySelectorAll('.time-range-btn').forEach(btn => {
|
|
btn.addEventListener('click', function () {
|
|
document.querySelectorAll('.time-range-btn').forEach(b => b.classList.remove('active'));
|
|
this.classList.add('active');
|
|
currentTimeRange = parseInt(this.dataset.hours);
|
|
loadActivityStats(currentTimeRange);
|
|
});
|
|
});
|
|
});
|
|
|
|
// Status polling
|
|
function initStatusPolling() {
|
|
updateStatus();
|
|
setInterval(updateStatus, 5000); // Poll every 5 seconds
|
|
}
|
|
|
|
function updateStatus() {
|
|
fetch('/scraper/status')
|
|
.then(response => response.json())
|
|
.then(data => {
|
|
if (data.active) {
|
|
if (data.paused) {
|
|
statusIndicator.className = 'status-indicator status-paused';
|
|
statusText.textContent = 'Paused';
|
|
pauseButton.textContent = 'Resume';
|
|
} else {
|
|
statusIndicator.className = 'status-indicator status-active';
|
|
statusText.textContent = 'Active';
|
|
pauseButton.textContent = 'Pause';
|
|
}
|
|
startButton.disabled = true;
|
|
pauseButton.disabled = false;
|
|
stopButton.disabled = false;
|
|
} else {
|
|
statusIndicator.className = 'status-indicator status-inactive';
|
|
statusText.textContent = 'Inactive';
|
|
startButton.disabled = false;
|
|
pauseButton.disabled = true;
|
|
stopButton.disabled = true;
|
|
}
|
|
});
|
|
}
|
|
|
|
// Action functions
|
|
function startScraper() {
|
|
fetch('/scraper/start', { method: 'POST' })
|
|
.then(response => response.json())
|
|
.then(data => {
|
|
if (data.success) {
|
|
showNotification('Scraper started successfully', 'success');
|
|
updateStatus();
|
|
setTimeout(() => { loadRecentActivity(); }, 1000);
|
|
} else {
|
|
showNotification(data.message, 'danger');
|
|
}
|
|
});
|
|
}
|
|
|
|
function togglePauseScraper() {
|
|
fetch('/scraper/pause', { method: 'POST' })
|
|
.then(response => response.json())
|
|
.then(data => {
|
|
if (data.success) {
|
|
showNotification(data.message, 'info');
|
|
updateStatus();
|
|
setTimeout(() => { loadRecentActivity(); }, 1000);
|
|
} else {
|
|
showNotification(data.message, 'danger');
|
|
}
|
|
});
|
|
}
|
|
|
|
function stopScraper() {
|
|
fetch('/scraper/stop', { method: 'POST' })
|
|
.then(response => response.json())
|
|
.then(data => {
|
|
if (data.success) {
|
|
showNotification('Scraper stopped successfully', 'warning');
|
|
updateStatus();
|
|
setTimeout(() => { loadRecentActivity(); }, 1000);
|
|
} else {
|
|
showNotification(data.message, 'danger');
|
|
}
|
|
});
|
|
}
|
|
|
|
function updateVolume() {
|
|
const volume = document.getElementById('volumeInput').value;
|
|
|
|
fetch('/scraper/update_config', {
|
|
method: 'POST',
|
|
headers: {
|
|
'Content-Type': 'application/json'
|
|
},
|
|
body: JSON.stringify({ volume: volume })
|
|
})
|
|
.then(response => response.json())
|
|
.then(data => {
|
|
if (data.success) {
|
|
showNotification('Volume updated successfully', 'success');
|
|
} else {
|
|
showNotification(data.message, 'danger');
|
|
}
|
|
});
|
|
}
|
|
|
|
function toggleNotifications() {
|
|
notificationsEnabled = notificationsToggle.checked;
|
|
}
|
|
|
|
// Load data functions
|
|
function loadActivityStats(hours) {
|
|
fetch(`/scraper/stats?hours=${hours}`)
|
|
.then(response => response.json())
|
|
.then(data => {
|
|
renderActivityChart(data);
|
|
});
|
|
}
|
|
|
|
function loadRecentActivity() {
|
|
fetch('/api/activity_logs?category=scraper_activity&limit=20')
|
|
.then(response => response.json())
|
|
.then(data => {
|
|
renderActivityLog(data);
|
|
})
|
|
.catch(() => {
|
|
// If the API endpoint doesn't exist, just show a message
|
|
activityLog.innerHTML = '<tr><td colspan="4" class="text-center">Activity log API not available</td></tr>';
|
|
});
|
|
}
|
|
|
|
// Rendering functions
|
|
function renderActivityChart(data) {
|
|
const ctx = document.getElementById('activityChart').getContext('2d');
|
|
|
|
// Extract the data for the chart
|
|
const labels = data.map(item => `${item.hour}:00`);
|
|
const successData = data.map(item => item.success);
|
|
const errorData = data.map(item => item.error);
|
|
const pendingData = data.map(item => item.pending);
|
|
|
|
if (activityChart) {
|
|
activityChart.destroy();
|
|
}
|
|
|
|
activityChart = new Chart(ctx, {
|
|
type: 'bar',
|
|
data: {
|
|
labels: labels,
|
|
datasets: [
|
|
{
|
|
label: 'Success',
|
|
data: successData,
|
|
backgroundColor: '#28a745',
|
|
stack: 'Stack 0'
|
|
},
|
|
{
|
|
label: 'Error',
|
|
data: errorData,
|
|
backgroundColor: '#dc3545',
|
|
stack: 'Stack 0'
|
|
},
|
|
{
|
|
label: 'Pending',
|
|
data: pendingData,
|
|
backgroundColor: '#ffc107',
|
|
stack: 'Stack 0'
|
|
}
|
|
]
|
|
},
|
|
options: {
|
|
responsive: true,
|
|
maintainAspectRatio: false,
|
|
scales: {
|
|
x: {
|
|
stacked: true,
|
|
title: {
|
|
display: true,
|
|
text: 'Hour'
|
|
}
|
|
},
|
|
y: {
|
|
stacked: true,
|
|
beginAtZero: true,
|
|
title: {
|
|
display: true,
|
|
text: 'Papers Scraped'
|
|
}
|
|
}
|
|
}
|
|
}
|
|
});
|
|
}
|
|
|
|
function renderActivityLog(logs) {
|
|
activityLog.innerHTML = '';
|
|
|
|
if (!logs || logs.length === 0) {
|
|
activityLog.innerHTML = '<tr><td colspan="4" class="text-center">No recent activity</td></tr>';
|
|
return;
|
|
}
|
|
|
|
logs.forEach(log => {
|
|
const row = document.createElement('tr');
|
|
|
|
// Format timestamp
|
|
const date = new Date(log.timestamp);
|
|
const timeStr = date.toLocaleTimeString();
|
|
|
|
// Create status badge
|
|
let statusBadge = '';
|
|
if (log.status === 'success') {
|
|
statusBadge = '<span class="badge bg-success">Success</span>';
|
|
} else if (log.status === 'error') {
|
|
statusBadge = '<span class="badge bg-danger">Error</span>';
|
|
} else if (log.status === 'pending') {
|
|
statusBadge = '<span class="badge bg-warning text-dark">Pending</span>';
|
|
} else {
|
|
statusBadge = `<span class="badge bg-secondary">${log.status || 'Unknown'}</span>`;
|
|
}
|
|
|
|
row.innerHTML = `
|
|
<td>${timeStr}</td>
|
|
<td>${log.action}</td>
|
|
<td>${statusBadge}</td>
|
|
<td>${log.description || ''}</td>
|
|
`;
|
|
|
|
activityLog.appendChild(row);
|
|
});
|
|
}
|
|
|
|
// Notification functions
|
|
function showNotification(message, type) {
|
|
if (!notificationsEnabled && type !== 'danger') {
|
|
return;
|
|
}
|
|
|
|
const container = document.getElementById('notificationContainer');
|
|
const notification = document.createElement('div');
|
|
notification.className = `alert alert-${type} notification shadow-sm`;
|
|
notification.innerHTML = `
|
|
${message}
|
|
<button type="button" class="btn-close float-end" aria-label="Close"></button>
|
|
`;
|
|
|
|
container.appendChild(notification);
|
|
|
|
// Add close handler
|
|
notification.querySelector('.btn-close').addEventListener('click', () => {
|
|
notification.remove();
|
|
});
|
|
|
|
// Auto-close after 5 seconds
|
|
setTimeout(() => {
|
|
notification.classList.add('fade');
|
|
setTimeout(() => {
|
|
notification.remove();
|
|
}, 500);
|
|
}, 5000);
|
|
}
|
|
|
|
// WebSocket for real-time notifications
|
|
function setupWebSocket() {
|
|
// If WebSocket is available, implement it here
|
|
// For now we'll poll the server periodically for new papers
|
|
setInterval(checkForNewPapers, 10000); // Check every 10 seconds
|
|
}
|
|
|
|
let lastPaperTimestamp = new Date().toISOString();
|
|
|
|
function checkForNewPapers() {
|
|
fetch(`/api/activity_logs?category=scraper_activity&action=scrape_paper&after=${lastPaperTimestamp}&limit=5`)
|
|
.then(response => response.json())
|
|
.then(data => {
|
|
if (data && data.length > 0) {
|
|
// Update the timestamp
|
|
lastPaperTimestamp = new Date().toISOString();
|
|
|
|
// Show notifications for new papers
|
|
data.forEach(log => {
|
|
const extraData = log.extra_data ? JSON.parse(log.extra_data) : {};
|
|
if (log.status === 'success') {
|
|
showNotification(`New paper scraped: ${extraData.title || 'Unknown title'}`, 'success');
|
|
} else if (log.status === 'error') {
|
|
showNotification(`Failed to scrape paper: ${log.description}`, 'danger');
|
|
}
|
|
});
|
|
|
|
// Refresh the activity chart and log
|
|
loadActivityStats(currentTimeRange);
|
|
loadRecentActivity();
|
|
}
|
|
})
|
|
.catch(() => {
|
|
// If the API endpoint doesn't exist, do nothing
|
|
});
|
|
}
|
|
|
|
// Start checking for new papers
|
|
setupWebSocket();
|
|
</script>
|
|
{% endblock scripts %} |