463 lines
16 KiB
Django/Jinja
463 lines
16 KiB
Django/Jinja
{% extends "base.html.jinja" %}
|
|
|
|
{% block title %}Paper Scraper Control Panel{% endblock title %}
|
|
|
|
{% block styles %}
|
|
{{ super() }}
|
|
<style>
|
|
.status-indicator {
|
|
width: 15px;
|
|
height: 15px;
|
|
border-radius: 50%;
|
|
display: inline-block;
|
|
margin-right: 5px;
|
|
}
|
|
|
|
.status-active {
|
|
background-color: #28a745;
|
|
}
|
|
|
|
.status-paused {
|
|
background-color: #ffc107;
|
|
}
|
|
|
|
.status-inactive {
|
|
background-color: #dc3545;
|
|
}
|
|
|
|
.stats-chart {
|
|
height: 400px;
|
|
}
|
|
|
|
.notification {
|
|
position: fixed;
|
|
bottom: 20px;
|
|
right: 20px;
|
|
max-width: 350px;
|
|
z-index: 1050;
|
|
}
|
|
</style>
|
|
{% endblock styles %}
|
|
|
|
{% block content %}
|
|
<div class="container mt-4">
|
|
<h1>Paper Scraper Control Panel</h1>
|
|
|
|
<!-- Include standardized flash messages -->
|
|
{% include "partials/flash_messages.html.jinja" %}
|
|
|
|
<div class="row mb-4">
|
|
<div class="col-md-6">
|
|
<div class="card">
|
|
<div class="card-header">
|
|
<h5>Scraper Status</h5>
|
|
</div>
|
|
<div class="card-body">
|
|
<div class="d-flex align-items-center mb-3">
|
|
<div id="statusIndicator" class="status-indicator status-inactive"></div>
|
|
<span id="statusText">Inactive</span>
|
|
</div>
|
|
|
|
<div class="btn-group" role="group">
|
|
<button id="startButton" class="btn btn-success">Start</button>
|
|
<button id="pauseButton" class="btn btn-warning" disabled>Pause</button>
|
|
<button id="stopButton" class="btn btn-danger" disabled>Stop</button>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<div class="col-md-6">
|
|
<div class="card">
|
|
<div class="card-header">
|
|
<h5>Volume Configuration</h5>
|
|
</div>
|
|
<div class="card-body">
|
|
<form id="volumeForm">
|
|
<div class="form-group">
|
|
<label for="volumeInput">Papers per day:</label>
|
|
<input type="number" class="form-control" id="volumeInput"
|
|
value="{{ volume_config.volume if volume_config else 100 }}">
|
|
</div>
|
|
<button type="submit" class="btn btn-primary mt-2">Update Volume</button>
|
|
</form>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<div class="row mb-4">
|
|
<div class="col-12">
|
|
<div class="card">
|
|
<div class="card-header d-flex justify-content-between align-items-center">
|
|
<h5>Scraping Activity</h5>
|
|
<div>
|
|
<div class="form-check form-switch">
|
|
<input class="form-check-input" type="checkbox" id="notificationsToggle" checked>
|
|
<label class="form-check-label" for="notificationsToggle">Show Notifications</label>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="card-body">
|
|
<div class="btn-group mb-3">
|
|
<button class="btn btn-outline-secondary time-range-btn" data-hours="6">Last 6
|
|
hours</button>
|
|
<button class="btn btn-outline-secondary time-range-btn active" data-hours="24">Last 24
|
|
hours</button>
|
|
<button class="btn btn-outline-secondary time-range-btn" data-hours="72">Last 3
|
|
days</button>
|
|
</div>
|
|
<div class="stats-chart" id="activityChart"></div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<div class="row mb-4">
|
|
<div class="col-12">
|
|
<div class="card">
|
|
<div class="card-header">
|
|
<h5>Recent Activity</h5>
|
|
</div>
|
|
<div class="card-body">
|
|
<div class="table-responsive">
|
|
<table class="table table-striped">
|
|
<thead>
|
|
<tr>
|
|
<th>Time</th>
|
|
<th>Action</th>
|
|
<th>Status</th>
|
|
<th>Description</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody id="activityLog">
|
|
<tr>
|
|
<td colspan="4" class="text-center">Loading activities...</td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
{% endblock content %}
|
|
|
|
{% block scripts %}
|
|
{{ super() }}
|
|
<script>
|
|
// Global variables for the scraper dashboard
|
|
let notificationsEnabled = true;
|
|
let activityChart = null;
|
|
let currentTimeRange = 24;
|
|
|
|
// DOM elements
|
|
const statusIndicator = document.getElementById('statusIndicator');
|
|
const statusText = document.getElementById('statusText');
|
|
const startButton = document.getElementById('startButton');
|
|
const pauseButton = document.getElementById('pauseButton');
|
|
const stopButton = document.getElementById('stopButton');
|
|
const notificationsToggle = document.getElementById('notificationsToggle');
|
|
const activityLog = document.getElementById('activityLog');
|
|
|
|
// Initialize the page
|
|
document.addEventListener('DOMContentLoaded', function () {
|
|
initStatusPolling();
|
|
loadActivityStats(currentTimeRange);
|
|
loadRecentActivity();
|
|
|
|
// Initialize event listeners
|
|
startButton.addEventListener('click', startScraper);
|
|
pauseButton.addEventListener('click', togglePauseScraper);
|
|
stopButton.addEventListener('click', stopScraper);
|
|
notificationsToggle.addEventListener('click', toggleNotifications);
|
|
|
|
document.getElementById('volumeForm').addEventListener('submit', function (e) {
|
|
e.preventDefault();
|
|
updateVolume();
|
|
});
|
|
|
|
document.querySelectorAll('.time-range-btn').forEach(btn => {
|
|
btn.addEventListener('click', function () {
|
|
document.querySelectorAll('.time-range-btn').forEach(b => b.classList.remove('active'));
|
|
this.classList.add('active');
|
|
currentTimeRange = parseInt(this.dataset.hours);
|
|
loadActivityStats(currentTimeRange);
|
|
});
|
|
});
|
|
});
|
|
|
|
// Status polling
|
|
function initStatusPolling() {
|
|
updateStatus();
|
|
setInterval(updateStatus, 5000); // Poll every 5 seconds
|
|
}
|
|
|
|
function updateStatus() {
|
|
fetch('/scraper/status')
|
|
.then(response => response.json())
|
|
.then(data => {
|
|
if (data.active) {
|
|
if (data.paused) {
|
|
statusIndicator.className = 'status-indicator status-paused';
|
|
statusText.textContent = 'Paused';
|
|
pauseButton.textContent = 'Resume';
|
|
} else {
|
|
statusIndicator.className = 'status-indicator status-active';
|
|
statusText.textContent = 'Active';
|
|
pauseButton.textContent = 'Pause';
|
|
}
|
|
startButton.disabled = true;
|
|
pauseButton.disabled = false;
|
|
stopButton.disabled = false;
|
|
} else {
|
|
statusIndicator.className = 'status-indicator status-inactive';
|
|
statusText.textContent = 'Inactive';
|
|
startButton.disabled = false;
|
|
pauseButton.disabled = true;
|
|
stopButton.disabled = true;
|
|
}
|
|
});
|
|
}
|
|
|
|
// Action functions
|
|
function startScraper() {
|
|
console.log("Start button clicked - sending request to /scraper/start");
|
|
|
|
fetch('/scraper/start', { method: 'POST' })
|
|
.then(response => {
|
|
console.log("Response received:", response);
|
|
return response.json();
|
|
})
|
|
.then(data => {
|
|
console.log("Data received:", data);
|
|
if (data.success) {
|
|
showFlashMessage('Scraper started successfully', 'success');
|
|
updateStatus();
|
|
setTimeout(() => { loadRecentActivity(); }, 1000);
|
|
} else {
|
|
showFlashMessage(data.message, 'error');
|
|
}
|
|
})
|
|
.catch(error => {
|
|
console.error("Error starting scraper:", error);
|
|
showFlashMessage('Error starting scraper: ' + error.message, 'error');
|
|
});
|
|
}
|
|
|
|
function togglePauseScraper() {
|
|
fetch('/scraper/pause', { method: 'POST' })
|
|
.then(response => response.json())
|
|
.then(data => {
|
|
if (data.success) {
|
|
showFlashMessage(data.message, 'info');
|
|
updateStatus();
|
|
setTimeout(() => { loadRecentActivity(); }, 1000);
|
|
} else {
|
|
showFlashMessage(data.message, 'error');
|
|
}
|
|
});
|
|
}
|
|
|
|
function stopScraper() {
|
|
fetch('/scraper/stop', { method: 'POST' })
|
|
.then(response => response.json())
|
|
.then(data => {
|
|
if (data.success) {
|
|
showFlashMessage('Scraper stopped successfully', 'warning');
|
|
updateStatus();
|
|
setTimeout(() => { loadRecentActivity(); }, 1000);
|
|
} else {
|
|
showFlashMessage(data.message, 'error');
|
|
}
|
|
});
|
|
}
|
|
|
|
function updateVolume() {
|
|
const volume = document.getElementById('volumeInput').value;
|
|
|
|
fetch('/scraper/update_config', {
|
|
method: 'POST',
|
|
headers: {
|
|
'Content-Type': 'application/json'
|
|
},
|
|
body: JSON.stringify({ volume: volume })
|
|
})
|
|
.then(response => response.json())
|
|
.then(data => {
|
|
if (data.success) {
|
|
showFlashMessage('Volume updated successfully', 'success');
|
|
} else {
|
|
showFlashMessage(data.message, 'error');
|
|
}
|
|
});
|
|
}
|
|
|
|
function toggleNotifications() {
|
|
notificationsEnabled = notificationsToggle.checked;
|
|
}
|
|
|
|
// Load data functions
|
|
function loadActivityStats(hours) {
|
|
fetch(`/scraper/stats?hours=${hours}`)
|
|
.then(response => response.json())
|
|
.then(data => {
|
|
renderActivityChart(data);
|
|
});
|
|
}
|
|
|
|
function loadRecentActivity() {
|
|
fetch('/api/activity_logs?category=scraper_activity&limit=20')
|
|
.then(response => response.json())
|
|
.then(data => {
|
|
renderActivityLog(data);
|
|
})
|
|
.catch(() => {
|
|
// If the API endpoint doesn't exist, just show a message
|
|
activityLog.innerHTML = '<tr><td colspan="4" class="text-center">Activity log API not available</td></tr>';
|
|
});
|
|
}
|
|
|
|
// Rendering functions
|
|
function renderActivityChart(data) {
|
|
const ctx = document.getElementById('activityChart').getContext('2d');
|
|
|
|
// Extract the data for the chart
|
|
const labels = data.map(item => `${item.hour}:00`);
|
|
const successData = data.map(item => item.success);
|
|
const errorData = data.map(item => item.error);
|
|
const pendingData = data.map(item => item.pending);
|
|
|
|
if (activityChart) {
|
|
activityChart.destroy();
|
|
}
|
|
|
|
activityChart = new Chart(ctx, {
|
|
type: 'bar',
|
|
data: {
|
|
labels: labels,
|
|
datasets: [
|
|
{
|
|
label: 'Success',
|
|
data: successData,
|
|
backgroundColor: '#28a745',
|
|
stack: 'Stack 0'
|
|
},
|
|
{
|
|
label: 'Error',
|
|
data: errorData,
|
|
backgroundColor: '#dc3545',
|
|
stack: 'Stack 0'
|
|
},
|
|
{
|
|
label: 'Pending',
|
|
data: pendingData,
|
|
backgroundColor: '#ffc107',
|
|
stack: 'Stack 0'
|
|
}
|
|
]
|
|
},
|
|
options: {
|
|
responsive: true,
|
|
maintainAspectRatio: false,
|
|
scales: {
|
|
x: {
|
|
stacked: true,
|
|
title: {
|
|
display: true,
|
|
text: 'Hour'
|
|
}
|
|
},
|
|
y: {
|
|
stacked: true,
|
|
beginAtZero: true,
|
|
title: {
|
|
display: true,
|
|
text: 'Papers Scraped'
|
|
}
|
|
}
|
|
}
|
|
}
|
|
});
|
|
}
|
|
|
|
function renderActivityLog(logs) {
|
|
activityLog.innerHTML = '';
|
|
|
|
if (!logs || logs.length === 0) {
|
|
activityLog.innerHTML = '<tr><td colspan="4" class="text-center">No recent activity</td></tr>';
|
|
return;
|
|
}
|
|
|
|
logs.forEach(log => {
|
|
const row = document.createElement('tr');
|
|
|
|
// Format timestamp
|
|
const date = new Date(log.timestamp);
|
|
const timeStr = date.toLocaleTimeString();
|
|
|
|
// Create status badge
|
|
let statusBadge = '';
|
|
if (log.status === 'success') {
|
|
statusBadge = '<span class="badge bg-success">Success</span>';
|
|
} else if (log.status === 'error') {
|
|
statusBadge = '<span class="badge bg-danger">Error</span>';
|
|
} else if (log.status === 'pending') {
|
|
statusBadge = '<span class="badge bg-warning text-dark">Pending</span>';
|
|
} else {
|
|
statusBadge = `<span class="badge bg-secondary">${log.status || 'Unknown'}</span>`;
|
|
}
|
|
|
|
row.innerHTML = `
|
|
<td>${timeStr}</td>
|
|
<td>${log.action}</td>
|
|
<td>${statusBadge}</td>
|
|
<td>${log.description || ''}</td>
|
|
`;
|
|
|
|
activityLog.appendChild(row);
|
|
});
|
|
}
|
|
|
|
// WebSocket for real-time notifications
|
|
function setupWebSocket() {
|
|
// If WebSocket is available, implement it here
|
|
// For now we'll poll the server periodically for new papers
|
|
setInterval(checkForNewPapers, 10000); // Check every 10 seconds
|
|
}
|
|
|
|
let lastPaperTimestamp = new Date().toISOString();
|
|
|
|
function checkForNewPapers() {
|
|
fetch(`/api/activity_logs?category=scraper_activity&action=scrape_paper&after=${lastPaperTimestamp}&limit=5`)
|
|
.then(response => response.json())
|
|
.then(data => {
|
|
if (data && data.length > 0) {
|
|
// Update the timestamp
|
|
lastPaperTimestamp = new Date().toISOString();
|
|
|
|
// Show notifications for new papers
|
|
data.forEach(log => {
|
|
const extraData = log.extra_data ? JSON.parse(log.extra_data) : {};
|
|
if (log.status === 'success') {
|
|
showFlashMessage(`New paper scraped: ${extraData.title || 'Unknown title'}`, 'success');
|
|
} else if (log.status === 'error') {
|
|
showFlashMessage(`Failed to scrape paper: ${log.description}`, 'error');
|
|
}
|
|
});
|
|
|
|
// Refresh the activity chart and log
|
|
loadActivityStats(currentTimeRange);
|
|
loadRecentActivity();
|
|
}
|
|
})
|
|
.catch(() => {
|
|
// If the API endpoint doesn't exist, do nothing
|
|
});
|
|
}
|
|
|
|
// Start checking for new papers
|
|
setupWebSocket();
|
|
</script>
|
|
{% endblock scripts %} |