581 lines
20 KiB
Django/Jinja
581 lines
20 KiB
Django/Jinja
{% extends "base.html.jinja" %}
|
|
|
|
{% block title %}Paper Scraper Control Panel{% endblock title %}
|
|
|
|
{% block styles %}
|
|
{{ super() }}
|
|
<style>
|
|
.status-indicator {
|
|
width: 15px;
|
|
height: 15px;
|
|
border-radius: 50%;
|
|
display: inline-block;
|
|
margin-right: 5px;
|
|
}
|
|
|
|
.status-active {
|
|
background-color: #28a745;
|
|
}
|
|
|
|
.status-paused {
|
|
background-color: #ffc107;
|
|
}
|
|
|
|
.status-inactive {
|
|
background-color: #dc3545;
|
|
}
|
|
|
|
.stats-chart {
|
|
height: 400px;
|
|
}
|
|
|
|
.notification {
|
|
position: fixed;
|
|
bottom: 20px;
|
|
right: 20px;
|
|
max-width: 350px;
|
|
z-index: 1050;
|
|
}
|
|
|
|
.schedule-grid {
|
|
display: grid;
|
|
grid-template-columns: repeat(6, 1fr);
|
|
gap: 10px;
|
|
}
|
|
|
|
.hour-block {
|
|
padding: 10px;
|
|
border-radius: 5px;
|
|
text-align: center;
|
|
}
|
|
|
|
.weight-1 {
|
|
background-color: #d4edda;
|
|
}
|
|
|
|
.weight-0-7 {
|
|
background-color: #d1ecf1;
|
|
}
|
|
|
|
.weight-0-5 {
|
|
background-color: #fff3cd;
|
|
}
|
|
|
|
.weight-0-2 {
|
|
background-color: #f8d7da;
|
|
}
|
|
|
|
.weight-0-1 {
|
|
background-color: #f5c6cb;
|
|
}
|
|
</style>
|
|
{% endblock styles %}
|
|
|
|
{% block content %}
|
|
<div class="container mt-4">
|
|
<h1>Paper Scraper Control Panel</h1>
|
|
|
|
<div class="row mb-4">
|
|
<div class="col-md-6">
|
|
<div class="card">
|
|
<div class="card-header">
|
|
<h5>Scraper Status</h5>
|
|
</div>
|
|
<div class="card-body">
|
|
<div class="d-flex align-items-center mb-3">
|
|
<div id="statusIndicator" class="status-indicator status-inactive"></div>
|
|
<span id="statusText">Inactive</span>
|
|
</div>
|
|
|
|
<div class="btn-group" role="group">
|
|
<button id="startButton" class="btn btn-success">Start</button>
|
|
<button id="pauseButton" class="btn btn-warning" disabled>Pause</button>
|
|
<button id="stopButton" class="btn btn-danger" disabled>Stop</button>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<div class="col-md-6">
|
|
<div class="card">
|
|
<div class="card-header">
|
|
<h5>Volume Configuration</h5>
|
|
</div>
|
|
<div class="card-body">
|
|
<form id="volumeForm">
|
|
<div class="form-group">
|
|
<label for="volumeInput">Papers per day:</label>
|
|
<input type="number" class="form-control" id="volumeInput"
|
|
value="{{ volume_config.volume }}">
|
|
</div>
|
|
<button type="submit" class="btn btn-primary mt-2">Update Volume</button>
|
|
</form>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<div class="row mb-4">
|
|
<div class="col-12">
|
|
<div class="card">
|
|
<div class="card-header">
|
|
<h5>Schedule Configuration</h5>
|
|
<small class="text-muted">Weight factor for each hour (lower value = higher scraping rate)</small>
|
|
</div>
|
|
<div class="card-body">
|
|
<div class="schedule-grid">
|
|
{% for hour in range(24) %}
|
|
{% set weight = schedule_config.get(hour, 1.0) %}
|
|
{% set weight_class = "weight-1" %}
|
|
{% if weight == 0.1 %}
|
|
{% set weight_class = "weight-0-1" %}
|
|
{% elif weight == 0.2 %}
|
|
{% set weight_class = "weight-0-2" %}
|
|
{% elif weight == 0.5 %}
|
|
{% set weight_class = "weight-0-5" %}
|
|
{% elif weight == 0.7 %}
|
|
{% set weight_class = "weight-0-7" %}
|
|
{% endif %}
|
|
|
|
<div class="hour-block border {{ weight_class }}" data-hour="{{ hour }}">
|
|
<div class="hour-label">{{ "%02d:00"|format(hour) }}</div>
|
|
<select class="form-control hour-weight mt-1" data-hour="{{ hour }}">
|
|
<option value="0.1" {% if weight==0.1 %}selected{% endif %}>Very High</option>
|
|
<option value="0.2" {% if weight==0.2 %}selected{% endif %}>High</option>
|
|
<option value="0.5" {% if weight==0.5 %}selected{% endif %}>Medium</option>
|
|
<option value="0.7" {% if weight==0.7 %}selected{% endif %}>Low</option>
|
|
<option value="1.0" {% if weight==1.0 %}selected{% endif %}>Very Low</option>
|
|
</select>
|
|
</div>
|
|
{% endfor %}
|
|
</div>
|
|
<button id="updateScheduleButton" class="btn btn-primary mt-3">Update Schedule</button>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<div class="row mb-4">
|
|
<div class="col-12">
|
|
<div class="card">
|
|
<div class="card-header d-flex justify-content-between align-items-center">
|
|
<h5>Scraping Activity</h5>
|
|
<div>
|
|
<div class="form-check form-switch">
|
|
<input class="form-check-input" type="checkbox" id="notificationsToggle" checked>
|
|
<label class="form-check-label" for="notificationsToggle">Show Notifications</label>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="card-body">
|
|
<div class="btn-group mb-3">
|
|
<button class="btn btn-outline-secondary time-range-btn" data-hours="6">Last 6 hours</button>
|
|
<button class="btn btn-outline-secondary time-range-btn active" data-hours="24">Last 24
|
|
hours</button>
|
|
<button class="btn btn-outline-secondary time-range-btn" data-hours="72">Last 3 days</button>
|
|
</div>
|
|
<div class="stats-chart" id="activityChart"></div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<div class="row mb-4">
|
|
<div class="col-12">
|
|
<div class="card">
|
|
<div class="card-header">
|
|
<h5>Recent Activity</h5>
|
|
</div>
|
|
<div class="card-body">
|
|
<div class="table-responsive">
|
|
<table class="table table-striped">
|
|
<thead>
|
|
<tr>
|
|
<th>Time</th>
|
|
<th>Action</th>
|
|
<th>Status</th>
|
|
<th>Description</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody id="activityLog">
|
|
<tr>
|
|
<td colspan="4" class="text-center">Loading activities...</td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- Notification template -->
|
|
<div id="notificationContainer"></div>
|
|
{% endblock content %}
|
|
|
|
{% block scripts %}
|
|
{{ super() }}
|
|
<script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
|
|
<script>
|
|
// Global variables
|
|
let notificationsEnabled = true;
|
|
let activityChart = null;
|
|
let currentTimeRange = 24;
|
|
|
|
// DOM elements
|
|
const statusIndicator = document.getElementById('statusIndicator');
|
|
const statusText = document.getElementById('statusText');
|
|
const startButton = document.getElementById('startButton');
|
|
const pauseButton = document.getElementById('pauseButton');
|
|
const stopButton = document.getElementById('stopButton');
|
|
const notificationsToggle = document.getElementById('notificationsToggle');
|
|
const activityLog = document.getElementById('activityLog');
|
|
|
|
// Initialize the page
|
|
document.addEventListener('DOMContentLoaded', function () {
|
|
initStatusPolling();
|
|
loadActivityStats(currentTimeRange);
|
|
loadRecentActivity();
|
|
|
|
// Initialize event listeners
|
|
startButton.addEventListener('click', startScraper);
|
|
pauseButton.addEventListener('click', togglePauseScraper);
|
|
stopButton.addEventListener('click', stopScraper);
|
|
notificationsToggle.addEventListener('click', toggleNotifications);
|
|
|
|
document.getElementById('volumeForm').addEventListener('submit', function (e) {
|
|
e.preventDefault();
|
|
updateVolume();
|
|
});
|
|
|
|
document.getElementById('updateScheduleButton').addEventListener('click', updateSchedule);
|
|
|
|
document.querySelectorAll('.time-range-btn').forEach(btn => {
|
|
btn.addEventListener('click', function () {
|
|
document.querySelectorAll('.time-range-btn').forEach(b => b.classList.remove('active'));
|
|
this.classList.add('active');
|
|
currentTimeRange = parseInt(this.dataset.hours);
|
|
loadActivityStats(currentTimeRange);
|
|
});
|
|
});
|
|
});
|
|
|
|
// Status polling
|
|
function initStatusPolling() {
|
|
updateStatus();
|
|
setInterval(updateStatus, 5000); // Poll every 5 seconds
|
|
}
|
|
|
|
function updateStatus() {
|
|
fetch('/scraper/status')
|
|
.then(response => response.json())
|
|
.then(data => {
|
|
if (data.active) {
|
|
if (data.paused) {
|
|
statusIndicator.className = 'status-indicator status-paused';
|
|
statusText.textContent = 'Paused';
|
|
pauseButton.textContent = 'Resume';
|
|
} else {
|
|
statusIndicator.className = 'status-indicator status-active';
|
|
statusText.textContent = 'Active';
|
|
pauseButton.textContent = 'Pause';
|
|
}
|
|
startButton.disabled = true;
|
|
pauseButton.disabled = false;
|
|
stopButton.disabled = false;
|
|
} else {
|
|
statusIndicator.className = 'status-indicator status-inactive';
|
|
statusText.textContent = 'Inactive';
|
|
startButton.disabled = false;
|
|
pauseButton.disabled = true;
|
|
stopButton.disabled = true;
|
|
}
|
|
});
|
|
}
|
|
|
|
// Action functions
|
|
function startScraper() {
|
|
fetch('/scraper/start', { method: 'POST' })
|
|
.then(response => response.json())
|
|
.then(data => {
|
|
if (data.success) {
|
|
showNotification('Scraper started successfully', 'success');
|
|
updateStatus();
|
|
setTimeout(() => { loadRecentActivity(); }, 1000);
|
|
} else {
|
|
showNotification(data.message, 'danger');
|
|
}
|
|
});
|
|
}
|
|
|
|
function togglePauseScraper() {
|
|
fetch('/scraper/pause', { method: 'POST' })
|
|
.then(response => response.json())
|
|
.then(data => {
|
|
if (data.success) {
|
|
showNotification(data.message, 'info');
|
|
updateStatus();
|
|
setTimeout(() => { loadRecentActivity(); }, 1000);
|
|
} else {
|
|
showNotification(data.message, 'danger');
|
|
}
|
|
});
|
|
}
|
|
|
|
function stopScraper() {
|
|
fetch('/scraper/stop', { method: 'POST' })
|
|
.then(response => response.json())
|
|
.then(data => {
|
|
if (data.success) {
|
|
showNotification('Scraper stopped successfully', 'warning');
|
|
updateStatus();
|
|
setTimeout(() => { loadRecentActivity(); }, 1000);
|
|
} else {
|
|
showNotification(data.message, 'danger');
|
|
}
|
|
});
|
|
}
|
|
|
|
function updateVolume() {
|
|
const volume = document.getElementById('volumeInput').value;
|
|
|
|
fetch('/scraper/update_config', {
|
|
method: 'POST',
|
|
headers: {
|
|
'Content-Type': 'application/json'
|
|
},
|
|
body: JSON.stringify({ volume: volume })
|
|
})
|
|
.then(response => response.json())
|
|
.then(data => {
|
|
if (data.success) {
|
|
showNotification('Volume updated successfully', 'success');
|
|
} else {
|
|
showNotification(data.message, 'danger');
|
|
}
|
|
});
|
|
}
|
|
|
|
function updateSchedule() {
|
|
const schedule = {};
|
|
document.querySelectorAll('.hour-weight').forEach(select => {
|
|
const hour = select.dataset.hour;
|
|
const weight = select.value;
|
|
schedule[hour] = weight;
|
|
});
|
|
|
|
fetch('/scraper/update_config', {
|
|
method: 'POST',
|
|
headers: {
|
|
'Content-Type': 'application/json'
|
|
},
|
|
body: JSON.stringify({ schedule: schedule })
|
|
})
|
|
.then(response => response.json())
|
|
.then(data => {
|
|
if (data.success) {
|
|
showNotification('Schedule updated successfully', 'success');
|
|
} else {
|
|
showNotification(data.message, 'danger');
|
|
}
|
|
});
|
|
}
|
|
|
|
function toggleNotifications() {
|
|
notificationsEnabled = notificationsToggle.checked;
|
|
}
|
|
|
|
// Load data functions
|
|
function loadActivityStats(hours) {
|
|
fetch(`/scraper/stats?hours=${hours}`)
|
|
.then(response => response.json())
|
|
.then(data => {
|
|
renderActivityChart(data);
|
|
});
|
|
}
|
|
|
|
function loadRecentActivity() {
|
|
fetch('/api/activity_logs?category=scraper_activity&limit=20')
|
|
.then(response => response.json())
|
|
.then(data => {
|
|
renderActivityLog(data);
|
|
})
|
|
.catch(() => {
|
|
// If the API endpoint doesn't exist, just show a message
|
|
activityLog.innerHTML = '<tr><td colspan="4" class="text-center">Activity log API not available</td></tr>';
|
|
});
|
|
}
|
|
|
|
// Rendering functions
|
|
function renderActivityChart(data) {
|
|
const ctx = document.getElementById('activityChart').getContext('2d');
|
|
|
|
// Extract the data for the chart
|
|
const labels = data.map(item => `${item.hour}:00`);
|
|
const successData = data.map(item => item.success);
|
|
const errorData = data.map(item => item.error);
|
|
const pendingData = data.map(item => item.pending);
|
|
|
|
if (activityChart) {
|
|
activityChart.destroy();
|
|
}
|
|
|
|
activityChart = new Chart(ctx, {
|
|
type: 'bar',
|
|
data: {
|
|
labels: labels,
|
|
datasets: [
|
|
{
|
|
label: 'Success',
|
|
data: successData,
|
|
backgroundColor: '#28a745',
|
|
stack: 'Stack 0'
|
|
},
|
|
{
|
|
label: 'Error',
|
|
data: errorData,
|
|
backgroundColor: '#dc3545',
|
|
stack: 'Stack 0'
|
|
},
|
|
{
|
|
label: 'Pending',
|
|
data: pendingData,
|
|
backgroundColor: '#ffc107',
|
|
stack: 'Stack 0'
|
|
}
|
|
]
|
|
},
|
|
options: {
|
|
responsive: true,
|
|
maintainAspectRatio: false,
|
|
scales: {
|
|
x: {
|
|
stacked: true,
|
|
title: {
|
|
display: true,
|
|
text: 'Hour'
|
|
}
|
|
},
|
|
y: {
|
|
stacked: true,
|
|
beginAtZero: true,
|
|
title: {
|
|
display: true,
|
|
text: 'Papers Scraped'
|
|
}
|
|
}
|
|
}
|
|
}
|
|
});
|
|
}
|
|
|
|
function renderActivityLog(logs) {
|
|
activityLog.innerHTML = '';
|
|
|
|
if (!logs || logs.length === 0) {
|
|
activityLog.innerHTML = '<tr><td colspan="4" class="text-center">No recent activity</td></tr>';
|
|
return;
|
|
}
|
|
|
|
logs.forEach(log => {
|
|
const row = document.createElement('tr');
|
|
|
|
// Format timestamp
|
|
const date = new Date(log.timestamp);
|
|
const timeStr = date.toLocaleTimeString();
|
|
|
|
// Create status badge
|
|
let statusBadge = '';
|
|
if (log.status === 'success') {
|
|
statusBadge = '<span class="badge bg-success">Success</span>';
|
|
} else if (log.status === 'error') {
|
|
statusBadge = '<span class="badge bg-danger">Error</span>';
|
|
} else if (log.status === 'pending') {
|
|
statusBadge = '<span class="badge bg-warning text-dark">Pending</span>';
|
|
} else {
|
|
statusBadge = `<span class="badge bg-secondary">${log.status || 'Unknown'}</span>`;
|
|
}
|
|
|
|
row.innerHTML = `
|
|
<td>${timeStr}</td>
|
|
<td>${log.action}</td>
|
|
<td>${statusBadge}</td>
|
|
<td>${log.description || ''}</td>
|
|
`;
|
|
|
|
activityLog.appendChild(row);
|
|
});
|
|
}
|
|
|
|
// Notification functions
|
|
function showNotification(message, type) {
|
|
if (!notificationsEnabled && type !== 'danger') {
|
|
return;
|
|
}
|
|
|
|
const container = document.getElementById('notificationContainer');
|
|
const notification = document.createElement('div');
|
|
notification.className = `alert alert-${type} notification shadow-sm`;
|
|
notification.innerHTML = `
|
|
${message}
|
|
<button type="button" class="btn-close float-end" aria-label="Close"></button>
|
|
`;
|
|
|
|
container.appendChild(notification);
|
|
|
|
// Add close handler
|
|
notification.querySelector('.btn-close').addEventListener('click', () => {
|
|
notification.remove();
|
|
});
|
|
|
|
// Auto-close after 5 seconds
|
|
setTimeout(() => {
|
|
notification.classList.add('fade');
|
|
setTimeout(() => {
|
|
notification.remove();
|
|
}, 500);
|
|
}, 5000);
|
|
}
|
|
|
|
// WebSocket for real-time notifications
|
|
function setupWebSocket() {
|
|
// If WebSocket is available, implement it here
|
|
// For now we'll poll the server periodically for new papers
|
|
setInterval(checkForNewPapers, 10000); // Check every 10 seconds
|
|
}
|
|
|
|
let lastPaperTimestamp = new Date().toISOString();
|
|
|
|
function checkForNewPapers() {
|
|
fetch(`/api/activity_logs?category=scraper_activity&action=scrape_paper&after=${lastPaperTimestamp}&limit=5`)
|
|
.then(response => response.json())
|
|
.then(data => {
|
|
if (data && data.length > 0) {
|
|
// Update the timestamp
|
|
lastPaperTimestamp = new Date().toISOString();
|
|
|
|
// Show notifications for new papers
|
|
data.forEach(log => {
|
|
const extraData = log.extra_data ? JSON.parse(log.extra_data) : {};
|
|
if (log.status === 'success') {
|
|
showNotification(`New paper scraped: ${extraData.title || 'Unknown title'}`, 'success');
|
|
} else if (log.status === 'error') {
|
|
showNotification(`Failed to scrape paper: ${log.description}`, 'danger');
|
|
}
|
|
});
|
|
|
|
// Refresh the activity chart and log
|
|
loadActivityStats(currentTimeRange);
|
|
loadRecentActivity();
|
|
}
|
|
})
|
|
.catch(() => {
|
|
// If the API endpoint doesn't exist, do nothing
|
|
});
|
|
}
|
|
|
|
// Start checking for new papers
|
|
setupWebSocket();
|
|
</script>
|
|
{% endblock scripts %} |