249 lines
12 KiB
Django/Jinja
249 lines
12 KiB
Django/Jinja
<!-- Scraper Overview Modal -->
|
|
<div class="modal fade" id="scraperOverviewModal" tabindex="-1" role="dialog"
|
|
aria-labelledby="scraperOverviewModalLabel" aria-hidden="true">
|
|
<div class="modal-dialog modal-xl" role="document">
|
|
<div class="modal-content">
|
|
<div class="modal-header">
|
|
<h5 class="modal-title" id="scraperOverviewModalLabel">
|
|
<i class="fas fa-cogs"></i> Scraper Modules Overview
|
|
</h5>
|
|
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
|
|
</div>
|
|
<div class="modal-body">
|
|
<!-- Loading state -->
|
|
<div id="scraperOverviewLoading" class="text-center py-4">
|
|
<div class="spinner-border text-primary" role="status">
|
|
<span class="visually-hidden">Loading...</span>
|
|
</div>
|
|
<p class="mt-2 text-muted">Loading scraper information...</p>
|
|
</div>
|
|
|
|
<!-- Error state -->
|
|
<div id="scraperOverviewError" class="alert alert-danger d-none" role="alert">
|
|
<h6 class="alert-heading">Error Loading Scrapers</h6>
|
|
<p id="scraperOverviewErrorMessage"></p>
|
|
<button class="btn btn-outline-danger btn-sm" onclick="loadScraperOverview()">
|
|
<i class="fas fa-redo"></i> Retry
|
|
</button>
|
|
</div>
|
|
|
|
<!-- Content -->
|
|
<div id="scraperOverviewContent" class="d-none">
|
|
<!-- Scraper Architecture Overview -->
|
|
<div class="card mb-4">
|
|
<div class="card-header">
|
|
<h6 class="mb-0">
|
|
<i class="fas fa-info-circle"></i> How Scraper Modules Work
|
|
</h6>
|
|
</div>
|
|
<div class="card-body">
|
|
<p class="mb-3">
|
|
SciPaperLoader uses a modular scraper architecture where each scraper module handles
|
|
specific paper processing stages. Papers flow through different statuses as they are
|
|
processed by various scrapers.
|
|
</p>
|
|
|
|
<div class="row">
|
|
<div class="col-md-6">
|
|
<h6>Key Concepts:</h6>
|
|
<ul class="small">
|
|
<li><strong>Input Statuses:</strong> Paper statuses this scraper can process
|
|
</li>
|
|
<li><strong>Output Statuses:</strong> Statuses papers get after processing</li>
|
|
<li><strong>Processing Status:</strong> Temporary status while scraper works
|
|
</li>
|
|
<li><strong>Pipeline:</strong> Scrapers can be chained together</li>
|
|
</ul>
|
|
</div>
|
|
<div class="col-md-6">
|
|
<h6>Status Flow Example:</h6>
|
|
<div class="d-flex align-items-center small">
|
|
<span class="badge bg-info">New</span>
|
|
<i class="fas fa-arrow-right mx-2"></i>
|
|
<span class="badge bg-warning">Processing</span>
|
|
<i class="fas fa-arrow-right mx-2"></i>
|
|
<span class="badge bg-success">Done</span>
|
|
</div>
|
|
<div class="text-muted mt-1">Papers transition through these statuses</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- Current System Configuration -->
|
|
<div class="card mb-4">
|
|
<div class="card-header">
|
|
<h6 class="mb-0">
|
|
<i class="fas fa-server"></i> System Configuration
|
|
</h6>
|
|
</div>
|
|
<div class="card-body">
|
|
<div class="row">
|
|
<div class="col-md-4">
|
|
<p><strong>Active Scraper Module:</strong> <span id="currentScraperModule"
|
|
class="badge bg-primary">Loading...</span></p>
|
|
<p><strong>Daily Volume Limit:</strong> <span
|
|
id="currentVolumeLimit">Loading...</span> papers</p>
|
|
</div>
|
|
<div class="col-md-4">
|
|
<p><strong>Total Available Modules:</strong> <span
|
|
id="totalScraperModules">Loading...</span></p>
|
|
<p><strong>Processing Pipeline:</strong> <span
|
|
id="processingPipeline">Multi-stage</span></p>
|
|
</div>
|
|
<div class="col-md-4">
|
|
<p><strong>Current Paper Counts:</strong></p>
|
|
<div id="paperCountsSummary" class="small">
|
|
<!-- Will be populated by JavaScript -->
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- Available Scrapers Table -->
|
|
<div class="card">
|
|
<div class="card-header">
|
|
<h6 class="mb-0">
|
|
<i class="fas fa-list"></i> Available Scraper Modules
|
|
</h6>
|
|
</div>
|
|
<div class="card-body">
|
|
<div class="table-responsive">
|
|
<table class="table table-hover">
|
|
<thead>
|
|
<tr>
|
|
<th>Module Name</th>
|
|
<th>Description</th>
|
|
<th>Input Statuses</th>
|
|
<th>Success Output</th>
|
|
<th>Failure Output</th>
|
|
<th>Processing Status</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody id="scrapersTableBody">
|
|
<!-- Table content will be populated by JavaScript -->
|
|
</tbody>
|
|
</table>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- Publisher Parser Overview -->
|
|
<div class="card mt-4">
|
|
<div class="card-header">
|
|
<h6 class="mb-0">
|
|
<i class="fas fa-building"></i> Publisher Parser Overview
|
|
</h6>
|
|
</div>
|
|
<div class="card-body">
|
|
<div class="row mb-3">
|
|
<div class="col-md-12">
|
|
<p class="text-muted mb-2">
|
|
<i class="fas fa-info-circle"></i>
|
|
Publishers are detected from paper URLs and mapped to specific parser modules
|
|
for content extraction.
|
|
</p>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- Publisher Statistics -->
|
|
<div class="row mb-4" id="publisherStats">
|
|
<!-- Will be populated by JavaScript -->
|
|
</div>
|
|
|
|
<!-- Publishers Table -->
|
|
<div class="table-responsive">
|
|
<table class="table table-hover table-sm">
|
|
<thead>
|
|
<tr>
|
|
<th>Publisher</th>
|
|
<th>Papers</th>
|
|
<th>Parser Status</th>
|
|
<th>Parser Available</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody id="publishersTableBody">
|
|
<!-- Table content will be populated by JavaScript -->
|
|
</tbody>
|
|
</table>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- Status Flow Diagram -->
|
|
<div class="card mt-4">
|
|
<div class="card-header">
|
|
<h6 class="mb-0">
|
|
<i class="fas fa-project-diagram"></i> Paper Status Flow Diagram
|
|
</h6>
|
|
</div>
|
|
<div class="card-body">
|
|
<div id="statusFlowDiagram" class="text-center py-4">
|
|
<!-- This will be populated by JavaScript -->
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="modal-footer">
|
|
<div class="d-flex justify-content-between w-100">
|
|
<small class="text-muted">
|
|
<i class="fas fa-lightbulb"></i>
|
|
Tip: Scrapers can be chained to create complex processing pipelines
|
|
</small>
|
|
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Close</button>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<style>
|
|
/* Custom styles for the scraper overview modal */
|
|
#scraperOverviewModal .modal-xl {
|
|
max-width: 1200px;
|
|
}
|
|
|
|
#scraperOverviewModal .table th {
|
|
font-size: 0.9rem;
|
|
background-color: #f8f9fa;
|
|
}
|
|
|
|
#scraperOverviewModal .badge {
|
|
font-size: 0.75rem;
|
|
}
|
|
|
|
#scraperOverviewModal .status-badge {
|
|
margin: 2px;
|
|
display: inline-block;
|
|
}
|
|
|
|
.status-flow-node {
|
|
display: inline-block;
|
|
padding: 8px 16px;
|
|
margin: 4px;
|
|
border-radius: 20px;
|
|
font-size: 0.9rem;
|
|
font-weight: 500;
|
|
}
|
|
|
|
.status-flow-arrow {
|
|
color: #6c757d;
|
|
margin: 0 8px;
|
|
}
|
|
|
|
.scraper-description {
|
|
max-width: 300px;
|
|
word-break: break-word;
|
|
}
|
|
|
|
.input-status-list {
|
|
max-width: 150px;
|
|
}
|
|
|
|
.status-output {
|
|
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', monospace;
|
|
font-size: 0.8rem;
|
|
}
|
|
</style> |