986 lines
39 KiB
Django/Jinja
986 lines
39 KiB
Django/Jinja
{% extends "base.html.jinja" %}
|
|
|
|
{% block title %}Control Panel{% endblock title %}
|
|
|
|
{% block styles %}
|
|
{{ super() }}
|
|
<style>
|
|
.status-indicator {
|
|
width: 15px;
|
|
height: 15px;
|
|
border-radius: 50%;
|
|
display: inline-block;
|
|
margin-right: 5px;
|
|
}
|
|
|
|
.status-active {
|
|
background-color: #28a745;
|
|
}
|
|
|
|
.status-paused {
|
|
background-color: #ffc107;
|
|
}
|
|
|
|
.status-inactive {
|
|
background-color: #dc3545;
|
|
}
|
|
|
|
.stats-chart {
|
|
height: 400px;
|
|
}
|
|
|
|
.chart-wrapper {
|
|
position: relative;
|
|
height: 400px;
|
|
}
|
|
|
|
.search-results-container {
|
|
max-height: 300px;
|
|
overflow-y: auto;
|
|
}
|
|
|
|
/* Paper status badges */
|
|
.badge-new {
|
|
background-color: #17a2b8;
|
|
}
|
|
|
|
.badge-pending {
|
|
background-color: #ffc107;
|
|
}
|
|
|
|
.badge-done {
|
|
background-color: #28a745;
|
|
}
|
|
|
|
.badge-failed {
|
|
background-color: #dc3545;
|
|
}
|
|
</style>
|
|
{% endblock styles %}
|
|
|
|
{% block content %}
|
|
<div class="container mt-4">
|
|
<h1>Paper Scraper Control Panel</h1>
|
|
|
|
<!-- Include standardized flash messages -->
|
|
{% include "partials/flash_messages.html.jinja" %}
|
|
|
|
<div class="row mb-4">
|
|
<div class="col-md-6">
|
|
<div class="card">
|
|
<div class="card-header">
|
|
<h5>Scraper Status</h5>
|
|
</div>
|
|
<div class="card-body">
|
|
<div class="d-flex align-items-center mb-3">
|
|
<div id="statusIndicator" class="status-indicator status-inactive"></div>
|
|
<span id="statusText">Inactive</span>
|
|
</div>
|
|
|
|
<div class="btn-group" role="group">
|
|
<button id="startButton" class="btn btn-success">Start</button>
|
|
<button id="pauseButton" class="btn btn-warning" disabled>Pause</button>
|
|
<button id="stopButton" class="btn btn-danger" disabled>Stop</button>
|
|
<button id="resetButton" class="btn btn-secondary" disabled>Reset</button>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<div class="col-md-6">
|
|
<div class="card">
|
|
<div class="card-header">
|
|
<h5>Volume Configuration</h5>
|
|
</div>
|
|
<div class="card-body">
|
|
<form id="volumeForm">
|
|
<div class="form-group">
|
|
<label for="volumeInput">Papers per day:</label>
|
|
<input type="number" class="form-control" id="volumeInput"
|
|
value="{{ volume_config if volume_config else 100 }}" min="1" max="{{ max_volume }}">
|
|
<button type="submit" class="btn btn-primary mt-2">
|
|
<i class="fas fa-save"></i> Update Volume
|
|
</button>
|
|
</div>
|
|
<div class="form-text">Enter a value between 1 and {{ max_volume }}</div>
|
|
</div>
|
|
</form>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- New row for single paper processing -->
|
|
<div class="row mb-4">
|
|
<div class="col-12">
|
|
<div class="card">
|
|
<div class="card-header">
|
|
<h5>Process Single Paper</h5>
|
|
</div>
|
|
<div class="card-body">
|
|
<div class="row">
|
|
<div class="col-md-6">
|
|
<form id="searchPaperForm" class="mb-3">
|
|
<div class="input-group">
|
|
<input type="text" id="paperSearchInput" class="form-control"
|
|
placeholder="Search paper by title, DOI, or ID...">
|
|
<button class="btn btn-outline-secondary" type="submit">Search</button>
|
|
</div>
|
|
</form>
|
|
</div>
|
|
<div class="col-md-6">
|
|
<div class="form-group">
|
|
<label for="scraperSelect">Scraper Module:</label>
|
|
<select class="form-control" id="scraperSelect">
|
|
<option value="">Use default system scraper</option>
|
|
<!-- Available scrapers will be populated here -->
|
|
</select>
|
|
<div class="form-text">
|
|
Select which scraper to use for processing the paper
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<div id="searchResults" class="mt-3 search-results-container d-none">
|
|
<table class="table table-hover table-striped">
|
|
<thead>
|
|
<tr>
|
|
<th>ID</th>
|
|
<th>Title</th>
|
|
<th>DOI</th>
|
|
<th>Status</th>
|
|
<th>Actions</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody id="paperSearchResults">
|
|
<!-- Search results will be populated here -->
|
|
</tbody>
|
|
</table>
|
|
</div>
|
|
|
|
<div id="processingStatus" class="alert alert-info mt-3 d-none"></div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<div class="row mb-4">
|
|
<div class="col-12">
|
|
<div class="card">
|
|
<div class="card-header d-flex justify-content-between align-items-center">
|
|
<h5>Scraping Activity</h5>
|
|
<div>
|
|
<div class="form-check form-switch">
|
|
<input class="form-check-input" type="checkbox" id="notificationsToggle" checked>
|
|
<label class="form-check-label" for="notificationsToggle">Show Notifications</label>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="card-body">
|
|
<div class="btn-group mb-3">
|
|
<button class="btn btn-outline-secondary time-range-btn" data-hours="6">Last 6
|
|
hours</button>
|
|
<button class="btn btn-outline-secondary time-range-btn active" data-hours="24">Last 24
|
|
hours</button>
|
|
<button class="btn btn-outline-secondary time-range-btn" data-hours="72">Last 3
|
|
days</button>
|
|
</div>
|
|
<div class="chart-wrapper">
|
|
<canvas id="activityChart"></canvas>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<div class="row mb-4">
|
|
<div class="col-12">
|
|
<div class="card">
|
|
<div class="card-header">
|
|
<h5>Recent Activity</h5>
|
|
</div>
|
|
<div class="card-body">
|
|
<div class="table-responsive">
|
|
<table class="table table-striped">
|
|
<thead>
|
|
<tr>
|
|
<th>Time</th>
|
|
<th>Action</th>
|
|
<th>Status</th>
|
|
<th>Description</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody id="activityLog">
|
|
<tr>
|
|
<td colspan="4" class="text-center">Loading activities...</td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
{% endblock content %}
|
|
|
|
{% block scripts %}
|
|
{{ super() }}
|
|
<script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
|
|
<script>
|
|
// Global variables for the scraper dashboard
|
|
let notificationsEnabled = true;
|
|
let activityChart = null;
|
|
let currentTimeRange = 24;
|
|
|
|
// DOM elements
|
|
const statusIndicator = document.getElementById('statusIndicator');
|
|
const statusText = document.getElementById('statusText');
|
|
const startButton = document.getElementById('startButton');
|
|
const pauseButton = document.getElementById('pauseButton');
|
|
const stopButton = document.getElementById('stopButton');
|
|
const resetButton = document.getElementById('resetButton');
|
|
const notificationsToggle = document.getElementById('notificationsToggle');
|
|
const activityLog = document.getElementById('activityLog');
|
|
const searchForm = document.getElementById('searchPaperForm');
|
|
const searchInput = document.getElementById('paperSearchInput');
|
|
const searchResults = document.getElementById('searchResults');
|
|
const processingStatus = document.getElementById('processingStatus');
|
|
const paperSearchResults = document.getElementById('paperSearchResults');
|
|
const scraperSelect = document.getElementById('scraperSelect');
|
|
|
|
// Initialize the page
|
|
document.addEventListener('DOMContentLoaded', function () {
|
|
initStatusPolling();
|
|
loadRecentActivity();
|
|
loadAvailableScrapers();
|
|
|
|
// Load chart data after a short delay to ensure Chart.js is loaded
|
|
setTimeout(() => {
|
|
loadActivityStats(currentTimeRange);
|
|
}, 100);
|
|
|
|
// Initialize event listeners
|
|
startButton.addEventListener('click', startScraper);
|
|
pauseButton.addEventListener('click', togglePauseScraper);
|
|
stopButton.addEventListener('click', stopScraper);
|
|
resetButton.addEventListener('click', resetScraper);
|
|
notificationsToggle.addEventListener('click', toggleNotifications);
|
|
searchForm.addEventListener('submit', function (e) {
|
|
e.preventDefault();
|
|
searchPapers();
|
|
});
|
|
|
|
document.getElementById('volumeForm').addEventListener('submit', function (e) {
|
|
e.preventDefault();
|
|
updateVolume();
|
|
});
|
|
|
|
document.querySelectorAll('.time-range-btn').forEach(btn => {
|
|
btn.addEventListener('click', function () {
|
|
document.querySelectorAll('.time-range-btn').forEach(b => b.classList.remove('active'));
|
|
this.classList.add('active');
|
|
currentTimeRange = parseInt(this.dataset.hours);
|
|
loadActivityStats(currentTimeRange);
|
|
});
|
|
});
|
|
});
|
|
|
|
// Load available scraper modules
|
|
function loadAvailableScrapers() {
|
|
fetch('/scraper/available_scrapers')
|
|
.then(response => response.json())
|
|
.then(data => {
|
|
if (data.success && data.scrapers && data.scrapers.length > 0) {
|
|
// Clear previous options except the default one
|
|
while (scraperSelect.options.length > 1) {
|
|
scraperSelect.remove(1);
|
|
}
|
|
|
|
// Add each scraper as an option
|
|
data.scrapers.forEach(scraper => {
|
|
const option = document.createElement('option');
|
|
option.value = scraper.name;
|
|
option.textContent = `${scraper.name} - ${scraper.description.substring(0, 50)}${scraper.description.length > 50 ? '...' : ''}`;
|
|
if (scraper.is_current) {
|
|
option.textContent += ' (system default)';
|
|
}
|
|
scraperSelect.appendChild(option);
|
|
});
|
|
} else {
|
|
// If no scrapers or error, add a note
|
|
const option = document.createElement('option');
|
|
option.disabled = true;
|
|
option.textContent = 'No scrapers available';
|
|
scraperSelect.appendChild(option);
|
|
}
|
|
})
|
|
.catch(error => {
|
|
console.error('Error loading scrapers:', error);
|
|
const option = document.createElement('option');
|
|
option.disabled = true;
|
|
option.textContent = 'Error loading scrapers';
|
|
scraperSelect.appendChild(option);
|
|
});
|
|
}
|
|
|
|
// Search papers function
|
|
function searchPapers() {
|
|
const query = searchInput.value.trim();
|
|
|
|
if (!query) {
|
|
showFlashMessage('Please enter a search term', 'warning');
|
|
return;
|
|
}
|
|
|
|
// Show loading message
|
|
paperSearchResults.innerHTML = '<tr><td colspan="5" class="text-center">Searching papers...</td></tr>';
|
|
searchResults.classList.remove('d-none');
|
|
|
|
// Fetch papers from API
|
|
fetch(`/api/papers?query=${encodeURIComponent(query)}`)
|
|
.then(response => response.json())
|
|
.then(data => {
|
|
if (!data.papers || data.papers.length === 0) {
|
|
paperSearchResults.innerHTML = '<tr><td colspan="5" class="text-center">No papers found matching your search</td></tr>';
|
|
return;
|
|
}
|
|
|
|
paperSearchResults.innerHTML = '';
|
|
|
|
data.papers.forEach(paper => {
|
|
const row = document.createElement('tr');
|
|
|
|
// Create status badge
|
|
let statusBadge = '';
|
|
if (paper.status === 'New') {
|
|
statusBadge = '<span class="badge bg-info">New</span>';
|
|
} else if (paper.status === 'Pending') {
|
|
statusBadge = '<span class="badge bg-warning text-dark">Pending</span>';
|
|
} else if (paper.status === 'Done') {
|
|
statusBadge = '<span class="badge bg-success">Done</span>';
|
|
} else if (paper.status === 'Failed') {
|
|
statusBadge = '<span class="badge bg-danger">Failed</span>';
|
|
} else {
|
|
statusBadge = `<span class="badge bg-secondary">${paper.status}</span>`;
|
|
}
|
|
|
|
// Create process button (enabled only for papers not in 'Pending' status)
|
|
const processButtonDisabled = paper.status === 'Pending' ? 'disabled' : '';
|
|
|
|
// Truncate title if too long
|
|
const truncatedTitle = paper.title.length > 70 ? paper.title.substring(0, 70) + '...' : paper.title;
|
|
|
|
row.innerHTML = `
|
|
<td>${paper.id}</td>
|
|
<td title="${paper.title}">${truncatedTitle}</td>
|
|
<td>${paper.doi || 'N/A'}</td>
|
|
<td>${statusBadge}</td>
|
|
<td>
|
|
<button class="btn btn-sm btn-primary process-paper-btn"
|
|
data-paper-id="${paper.id}"
|
|
${processButtonDisabled}>
|
|
Process Now
|
|
</button>
|
|
</td>
|
|
`;
|
|
|
|
paperSearchResults.appendChild(row);
|
|
});
|
|
|
|
// Add event listeners to the process buttons
|
|
document.querySelectorAll('.process-paper-btn').forEach(btn => {
|
|
btn.addEventListener('click', function () {
|
|
processSinglePaper(this.getAttribute('data-paper-id'));
|
|
});
|
|
});
|
|
})
|
|
.catch(error => {
|
|
console.error('Error searching papers:', error);
|
|
paperSearchResults.innerHTML = '<tr><td colspan="5" class="text-center">Error searching papers</td></tr>';
|
|
});
|
|
}
|
|
|
|
// Process a single paper
|
|
function processSinglePaper(paperId) {
|
|
// Disable all process buttons to prevent multiple clicks
|
|
document.querySelectorAll('.process-paper-btn').forEach(btn => {
|
|
btn.disabled = true;
|
|
});
|
|
|
|
// Show processing status
|
|
processingStatus.textContent = 'Processing paper...';
|
|
processingStatus.classList.remove('d-none');
|
|
|
|
// Get selected scraper
|
|
const selectedScraper = scraperSelect.value;
|
|
|
|
// Send request to process the paper
|
|
fetch(`/scraper/process_single/${paperId}`, {
|
|
method: 'POST',
|
|
headers: {
|
|
'Content-Type': 'application/json'
|
|
},
|
|
body: JSON.stringify({
|
|
scraper_module: selectedScraper
|
|
})
|
|
})
|
|
.then(response => response.json())
|
|
.then(data => {
|
|
if (data.success) {
|
|
processingStatus.textContent = data.message;
|
|
processingStatus.className = 'alert alert-success mt-3';
|
|
|
|
// Update status in the search results
|
|
const row = document.querySelector(`.process-paper-btn[data-paper-id="${paperId}"]`).closest('tr');
|
|
const statusCell = row.querySelector('td:nth-child(4)');
|
|
statusCell.innerHTML = '<span class="badge bg-warning text-dark">Pending</span>';
|
|
|
|
// Show notification
|
|
showFlashMessage(data.message, 'success');
|
|
|
|
// Set up polling to check paper status and refresh activity
|
|
pollPaperStatus(paperId, 3000, 20);
|
|
} else {
|
|
processingStatus.textContent = data.message;
|
|
processingStatus.className = 'alert alert-danger mt-3';
|
|
showFlashMessage(data.message, 'error');
|
|
}
|
|
})
|
|
.catch(error => {
|
|
console.error('Error processing paper:', error);
|
|
processingStatus.textContent = 'Error: Could not process paper';
|
|
processingStatus.className = 'alert alert-danger mt-3';
|
|
showFlashMessage('Error processing paper', 'error');
|
|
})
|
|
.finally(() => {
|
|
// Re-enable the process buttons after a short delay
|
|
setTimeout(() => {
|
|
document.querySelectorAll('.process-paper-btn').forEach(btn => {
|
|
if (btn.getAttribute('data-paper-id') !== paperId) {
|
|
btn.disabled = false;
|
|
}
|
|
});
|
|
}, 1000);
|
|
});
|
|
}
|
|
|
|
// Status polling
|
|
function initStatusPolling() {
|
|
updateStatus();
|
|
setInterval(updateStatus, 5000); // Poll every 5 seconds
|
|
}
|
|
|
|
function updateStatus() {
|
|
fetch('/scraper/status')
|
|
.then(response => response.json())
|
|
.then(data => {
|
|
console.log('Status data received:', data); // Debug log
|
|
|
|
// Remove all status classes first
|
|
statusIndicator.classList.remove('status-active', 'status-paused', 'status-inactive');
|
|
|
|
// Handle the new JSON structure with scraper_state
|
|
const scraperState = data.scraper_state || data; // Fallback for old structure
|
|
|
|
if (scraperState.active) {
|
|
if (scraperState.paused) {
|
|
statusIndicator.classList.add('status-paused');
|
|
statusText.textContent = 'Paused';
|
|
pauseButton.textContent = 'Resume';
|
|
} else {
|
|
statusIndicator.classList.add('status-active');
|
|
statusText.textContent = 'Active';
|
|
pauseButton.textContent = 'Pause';
|
|
}
|
|
startButton.disabled = true;
|
|
pauseButton.disabled = false;
|
|
stopButton.disabled = false;
|
|
resetButton.disabled = false; // Enable reset when active
|
|
} else {
|
|
statusIndicator.classList.add('status-inactive');
|
|
statusText.textContent = 'Inactive';
|
|
startButton.disabled = false;
|
|
pauseButton.disabled = true;
|
|
stopButton.disabled = true;
|
|
resetButton.disabled = false; // Enable reset when inactive too
|
|
}
|
|
})
|
|
.catch(error => {
|
|
console.error('Error fetching status:', error);
|
|
// On error, show inactive state
|
|
statusIndicator.classList.remove('status-active', 'status-paused', 'status-inactive');
|
|
statusIndicator.classList.add('status-inactive');
|
|
statusText.textContent = 'Error';
|
|
});
|
|
}
|
|
|
|
// Action functions
|
|
function startScraper() {
|
|
console.log("Start button clicked - sending request to /scraper/start");
|
|
|
|
fetch('/scraper/start', {
|
|
method: 'POST',
|
|
headers: {
|
|
'Content-Type': 'application/json'
|
|
},
|
|
body: JSON.stringify({})
|
|
})
|
|
.then(response => {
|
|
console.log("Response received:", response);
|
|
return response.json();
|
|
})
|
|
.then(data => {
|
|
console.log("Data received:", data);
|
|
if (data.success) {
|
|
showFlashMessage('Scraper started successfully', 'success');
|
|
updateStatus();
|
|
setTimeout(() => { loadRecentActivity(); }, 1000);
|
|
} else {
|
|
showFlashMessage(data.message, 'error');
|
|
}
|
|
})
|
|
.catch(error => {
|
|
console.error("Error starting scraper:", error);
|
|
showFlashMessage('Error starting scraper: ' + error.message, 'error');
|
|
});
|
|
}
|
|
|
|
function togglePauseScraper() {
|
|
fetch('/scraper/pause', {
|
|
method: 'POST',
|
|
headers: {
|
|
'Content-Type': 'application/json'
|
|
},
|
|
body: JSON.stringify({})
|
|
})
|
|
.then(response => response.json())
|
|
.then(data => {
|
|
if (data.success) {
|
|
showFlashMessage(data.message, 'info');
|
|
updateStatus();
|
|
setTimeout(() => { loadRecentActivity(); }, 1000);
|
|
} else {
|
|
showFlashMessage(data.message, 'error');
|
|
}
|
|
});
|
|
}
|
|
|
|
function stopScraper() {
|
|
fetch('/scraper/stop', {
|
|
method: 'POST',
|
|
headers: {
|
|
'Content-Type': 'application/json'
|
|
},
|
|
body: JSON.stringify({})
|
|
})
|
|
.then(response => response.json())
|
|
.then(data => {
|
|
if (data.success) {
|
|
showFlashMessage('Scraper stopped successfully', 'warning');
|
|
updateStatus();
|
|
setTimeout(() => { loadRecentActivity(); }, 1000);
|
|
} else {
|
|
showFlashMessage(data.message, 'error');
|
|
}
|
|
});
|
|
}
|
|
|
|
function resetScraper() {
|
|
if (confirm("Are you sure you want to reset the scraper? This will stop all current tasks, optionally clear non-pending papers, and restart the scraper.")) {
|
|
// Disable button to prevent multiple clicks
|
|
resetButton.disabled = true;
|
|
|
|
// Show a loading message
|
|
showFlashMessage('Resetting scraper, please wait...', 'info');
|
|
|
|
fetch('/scraper/reset', {
|
|
method: 'POST',
|
|
headers: {
|
|
'Content-Type': 'application/json'
|
|
},
|
|
body: JSON.stringify({
|
|
clear_papers: true // You could make this configurable with a checkbox
|
|
})
|
|
})
|
|
.then(response => response.json())
|
|
.then(data => {
|
|
if (data.success) {
|
|
showFlashMessage('Scraper has been completely reset and restarted', 'success');
|
|
// Update everything
|
|
updateStatus();
|
|
loadActivityStats(currentTimeRange);
|
|
setTimeout(() => { loadRecentActivity(); }, 1000);
|
|
} else {
|
|
showFlashMessage(data.message || 'Error resetting scraper', 'error');
|
|
}
|
|
// Re-enable button
|
|
resetButton.disabled = false;
|
|
})
|
|
.catch(error => {
|
|
console.error("Error resetting scraper:", error);
|
|
showFlashMessage('Error resetting scraper: ' + error.message, 'error');
|
|
// Re-enable button
|
|
resetButton.disabled = false;
|
|
});
|
|
}
|
|
}
|
|
|
|
function updateVolume() {
|
|
const volumeInput = document.getElementById('volumeInput');
|
|
const volume = volumeInput.value;
|
|
const submitButton = document.querySelector('#volumeForm button[type="submit"]');
|
|
|
|
// Basic validation
|
|
if (!volume || volume < 1 || volume > {{ max_volume }} ) { // <-- Corrected line
|
|
showFlashMessage('Please enter a valid volume between 1 and {{ max_volume }}', 'warning');
|
|
volumeInput.focus();
|
|
return;
|
|
}
|
|
|
|
// Disable button and show loading state
|
|
submitButton.disabled = true;
|
|
const originalText = submitButton.innerHTML;
|
|
submitButton.innerHTML = '<i class="fas fa-spinner fa-spin"></i> Updating...';
|
|
|
|
fetch('/scraper/update_config', {
|
|
method: 'POST',
|
|
headers: {
|
|
'Content-Type': 'application/json'
|
|
},
|
|
body: JSON.stringify({ volume: volume })
|
|
})
|
|
.then(response => response.json())
|
|
.then(data => {
|
|
if (data.success) {
|
|
// Use the backend message for success
|
|
showFlashMessage(data.message || 'Volume updated successfully', 'success');
|
|
} else {
|
|
showFlashMessage(data.message || 'Failed to update volume', 'error');
|
|
}
|
|
})
|
|
.catch(error => {
|
|
console.error('Error updating volume:', error);
|
|
showFlashMessage('Network error while updating volume. Please try again.', 'error');
|
|
})
|
|
.finally(() => {
|
|
// Re-enable button and restore original text
|
|
submitButton.disabled = false;
|
|
submitButton.innerHTML = originalText;
|
|
});
|
|
}
|
|
|
|
function toggleNotifications() {
|
|
notificationsEnabled = notificationsToggle.checked;
|
|
}
|
|
|
|
// Poll paper status until it changes from Pending
|
|
function pollPaperStatus(paperId, interval = 3000, maxAttempts = 20) {
|
|
let attempts = 0;
|
|
|
|
// Immediately refresh activity log to show the initial pending status
|
|
loadRecentActivity();
|
|
|
|
const checkStatus = () => {
|
|
attempts++;
|
|
console.log(`Checking status of paper ${paperId}, attempt ${attempts}/${maxAttempts}`);
|
|
|
|
// Fetch the current paper status
|
|
fetch(`/api/papers/${paperId}`)
|
|
.then(response => response.json())
|
|
.then(data => {
|
|
if (data && data.paper) {
|
|
const paper = data.paper;
|
|
console.log(`Paper status: ${paper.status}`);
|
|
|
|
// Update the UI with the current status
|
|
const row = document.querySelector(`.process-paper-btn[data-paper-id="${paperId}"]`).closest('tr');
|
|
if (row) {
|
|
const statusCell = row.querySelector('td:nth-child(4)');
|
|
let statusBadge = '';
|
|
|
|
if (paper.status === 'New') {
|
|
statusBadge = '<span class="badge bg-info">New</span>';
|
|
} else if (paper.status === 'Pending') {
|
|
statusBadge = '<span class="badge bg-warning text-dark">Pending</span>';
|
|
} else if (paper.status === 'Done') {
|
|
statusBadge = '<span class="badge bg-success">Done</span>';
|
|
} else if (paper.status === 'Failed') {
|
|
statusBadge = '<span class="badge bg-danger">Failed</span>';
|
|
} else {
|
|
statusBadge = `<span class="badge bg-secondary">${paper.status}</span>`;
|
|
}
|
|
|
|
statusCell.innerHTML = statusBadge;
|
|
|
|
// Update processing status message if status changed
|
|
if (paper.status !== 'Pending') {
|
|
if (paper.status === 'Done') {
|
|
processingStatus.textContent = `Paper processed successfully: ${paper.title}`;
|
|
processingStatus.className = 'alert alert-success mt-3';
|
|
} else if (paper.status === 'Failed') {
|
|
processingStatus.textContent = `Paper processing failed: ${paper.error_msg || 'Unknown error'}`;
|
|
processingStatus.className = 'alert alert-danger mt-3';
|
|
}
|
|
}
|
|
}
|
|
|
|
// Always refresh activity log
|
|
loadRecentActivity();
|
|
|
|
// If status is still pending and we haven't reached max attempts, check again
|
|
if (paper.status === 'Pending' && attempts < maxAttempts) {
|
|
setTimeout(checkStatus, interval);
|
|
} else {
|
|
// If status changed or we reached max attempts, refresh chart data too
|
|
loadActivityStats(currentTimeRange);
|
|
|
|
// Show notification if status changed
|
|
if (paper.status !== 'Pending') {
|
|
const status = paper.status === 'Done' ? 'success' : 'error';
|
|
const message = paper.status === 'Done'
|
|
? `Paper processed successfully: ${paper.title}`
|
|
: `Paper processing failed: ${paper.error_msg || 'Unknown error'}`;
|
|
showFlashMessage(message, status);
|
|
}
|
|
|
|
// If we hit max attempts but status is still pending, show a message
|
|
if (paper.status === 'Pending' && attempts >= maxAttempts) {
|
|
processingStatus.textContent = 'Paper is still being processed. Check the activity log for updates.';
|
|
processingStatus.className = 'alert alert-info mt-3';
|
|
}
|
|
}
|
|
}
|
|
})
|
|
.catch(error => {
|
|
console.error(`Error polling paper status: ${error}`);
|
|
// If there's an error, we can still try again if under max attempts
|
|
if (attempts < maxAttempts) {
|
|
setTimeout(checkStatus, interval);
|
|
}
|
|
});
|
|
};
|
|
|
|
// Start checking
|
|
setTimeout(checkStatus, interval);
|
|
}
|
|
|
|
// Load data functions
|
|
function loadActivityStats(hours) {
|
|
fetch(`/scraper/stats?hours=${hours}`)
|
|
.then(response => {
|
|
if (!response.ok) {
|
|
throw new Error(`HTTP error! status: ${response.status}`);
|
|
}
|
|
return response.json();
|
|
})
|
|
.then(data => {
|
|
console.log('Stats data loaded:', data);
|
|
renderActivityChart(data);
|
|
})
|
|
.catch(error => {
|
|
console.error('Failed to load activity stats:', error);
|
|
// Hide the chart or show an error message
|
|
const chartContainer = document.getElementById('activityChart').parentElement;
|
|
if (chartContainer) {
|
|
chartContainer.innerHTML = '<p class="text-muted">Chart data unavailable</p>';
|
|
}
|
|
});
|
|
}
|
|
|
|
function loadRecentActivity() {
|
|
fetch('/api/activity_logs?category=scraper_activity&category=scraper_command&limit=50')
|
|
.then(response => response.json())
|
|
.then(data => {
|
|
renderActivityLog(data);
|
|
console.log("Activity log refreshed with latest data");
|
|
})
|
|
.catch((error) => {
|
|
console.error("Failed to load activity logs:", error);
|
|
// If the API endpoint doesn't exist, just show a message
|
|
activityLog.innerHTML = '<tr><td colspan="4" class="text-center">Activity log API not available</td></tr>';
|
|
});
|
|
}
|
|
|
|
// Rendering functions
|
|
function renderActivityChart(data) {
|
|
// Check if Chart.js is available
|
|
if (typeof Chart === 'undefined') {
|
|
console.error('Chart.js is not loaded');
|
|
return;
|
|
}
|
|
|
|
const chartElement = document.getElementById('activityChart');
|
|
if (!chartElement) {
|
|
console.error('Chart canvas element not found');
|
|
return;
|
|
}
|
|
|
|
const ctx = chartElement.getContext('2d');
|
|
|
|
// Extract the data for the chart
|
|
const labels = data.map(item => `${item.hour}:00`);
|
|
const successData = data.map(item => item.success);
|
|
const errorData = data.map(item => item.error);
|
|
const pendingData = data.map(item => item.pending);
|
|
|
|
if (activityChart) {
|
|
activityChart.destroy();
|
|
}
|
|
|
|
activityChart = new Chart(ctx, {
|
|
type: 'bar',
|
|
data: {
|
|
labels: labels,
|
|
datasets: [
|
|
{
|
|
label: 'Success',
|
|
data: successData,
|
|
backgroundColor: '#28a745',
|
|
stack: 'Stack 0'
|
|
},
|
|
{
|
|
label: 'Error',
|
|
data: errorData,
|
|
backgroundColor: '#dc3545',
|
|
stack: 'Stack 0'
|
|
},
|
|
{
|
|
label: 'Pending',
|
|
data: pendingData,
|
|
backgroundColor: '#ffc107',
|
|
stack: 'Stack 0'
|
|
}
|
|
]
|
|
},
|
|
options: {
|
|
responsive: true,
|
|
maintainAspectRatio: false,
|
|
scales: {
|
|
x: {
|
|
stacked: true,
|
|
title: {
|
|
display: true,
|
|
text: 'Hour'
|
|
}
|
|
},
|
|
y: {
|
|
stacked: true,
|
|
beginAtZero: true,
|
|
title: {
|
|
display: true,
|
|
text: 'Papers Scraped'
|
|
}
|
|
}
|
|
}
|
|
}
|
|
});
|
|
}
|
|
|
|
function renderActivityLog(logs) {
|
|
activityLog.innerHTML = '';
|
|
|
|
if (!logs || logs.length === 0) {
|
|
activityLog.innerHTML = '<tr><td colspan="4" class="text-center">No recent activity</td></tr>';
|
|
return;
|
|
}
|
|
|
|
logs.forEach(log => {
|
|
const row = document.createElement('tr');
|
|
|
|
// Format timestamp
|
|
const date = new Date(log.timestamp);
|
|
const timeStr = date.toLocaleTimeString();
|
|
|
|
// Create status badge
|
|
let statusBadge = '';
|
|
if (log.status === 'success') {
|
|
statusBadge = '<span class="badge bg-success">Success</span>';
|
|
} else if (log.status === 'error') {
|
|
statusBadge = '<span class="badge bg-danger">Error</span>';
|
|
} else if (log.status === 'pending') {
|
|
statusBadge = '<span class="badge bg-warning text-dark">Pending</span>';
|
|
} else {
|
|
statusBadge = `<span class="badge bg-secondary">${log.status || 'Unknown'}</span>`;
|
|
}
|
|
|
|
row.innerHTML = `
|
|
<td>${timeStr}</td>
|
|
<td>${log.action}</td>
|
|
<td>${statusBadge}</td>
|
|
<td>${log.description || ''}</td>
|
|
`;
|
|
|
|
activityLog.appendChild(row);
|
|
});
|
|
}
|
|
|
|
// Flash message function
|
|
function showFlashMessage(message, type) {
|
|
const flashContainer = document.createElement('div');
|
|
flashContainer.className = `alert alert-${type === 'error' ? 'danger' : type} alert-dismissible fade show`;
|
|
flashContainer.innerHTML = `
|
|
${message}
|
|
<button type="button" class="btn-close" data-bs-dismiss="alert" aria-label="Close"></button>
|
|
`;
|
|
|
|
// Use the existing client flash container at the top of the page
|
|
const clientFlashContainer = document.getElementById('clientFlashContainer');
|
|
if (clientFlashContainer) {
|
|
clientFlashContainer.appendChild(flashContainer);
|
|
} else {
|
|
// Fallback to body if container not found
|
|
document.body.appendChild(flashContainer);
|
|
}
|
|
|
|
// Auto dismiss after 5 seconds
|
|
setTimeout(() => {
|
|
flashContainer.classList.remove('show');
|
|
setTimeout(() => {
|
|
flashContainer.remove();
|
|
}, 150); // Remove after fade out animation
|
|
}, 5000);
|
|
}
|
|
|
|
// WebSocket for real-time notifications
|
|
function setupWebSocket() {
|
|
// If WebSocket is available, implement it here
|
|
// For now we'll poll the server periodically for new papers
|
|
setInterval(checkForNewPapers, 10000); // Check every 10 seconds
|
|
}
|
|
|
|
let lastPaperTimestamp = new Date().toISOString();
|
|
|
|
function checkForNewPapers() {
|
|
fetch(`/api/activity_logs?category=scraper_activity&category=scraper_command&action=scrape_paper&after=${lastPaperTimestamp}&limit=5`)
|
|
.then(response => response.json())
|
|
.then(data => {
|
|
if (data && data.length > 0) {
|
|
// Update the timestamp
|
|
lastPaperTimestamp = new Date().toISOString();
|
|
|
|
// Show notifications for new papers
|
|
data.forEach(log => {
|
|
const extraData = log.extra_data ? JSON.parse(log.extra_data) : {};
|
|
if (log.status === 'success') {
|
|
showFlashMessage(`New paper scraped: ${extraData.title || 'Unknown title'}`, 'success');
|
|
} else if (log.status === 'error') {
|
|
showFlashMessage(`Failed to scrape paper: ${log.description}`, 'error');
|
|
}
|
|
});
|
|
|
|
// Refresh the activity chart and log
|
|
loadActivityStats(currentTimeRange);
|
|
loadRecentActivity();
|
|
}
|
|
})
|
|
.catch(() => {
|
|
// If the API endpoint doesn't exist, do nothing
|
|
});
|
|
}
|
|
|
|
// Start checking for new papers
|
|
setupWebSocket();
|
|
</script>
|
|
{% endblock scripts %} |