adds pagination to scraper and improves timestamp formatting
This commit is contained in:
parent
7a1ab3d7e6
commit
676a3c96eb
@ -238,10 +238,51 @@ def get_status():
|
||||
|
||||
@bp.route("/logs")
|
||||
def get_logs():
|
||||
"""Get recent activity logs."""
|
||||
"""Get recent activity logs with pagination support."""
|
||||
try:
|
||||
limit = request.args.get('limit', 50, type=int)
|
||||
logs = ActivityLog.query.order_by(ActivityLog.timestamp.desc()).limit(limit).all()
|
||||
# Pagination parameters
|
||||
page = request.args.get('page', 1, type=int)
|
||||
per_page = request.args.get('per_page', 20, type=int)
|
||||
|
||||
# Legacy limit parameter for backward compatibility
|
||||
limit = request.args.get('limit', type=int)
|
||||
if limit and not request.args.get('page'):
|
||||
# Legacy mode: use limit without pagination
|
||||
logs = ActivityLog.query.order_by(ActivityLog.timestamp.desc()).limit(limit).all()
|
||||
return jsonify({
|
||||
"success": True,
|
||||
"logs": [{
|
||||
"id": log.id,
|
||||
"timestamp": log.timestamp.isoformat(),
|
||||
"action": log.action,
|
||||
"status": log.status,
|
||||
"description": log.description,
|
||||
"category": log.category
|
||||
} for log in logs]
|
||||
})
|
||||
|
||||
# Ensure reasonable per_page limits
|
||||
per_page = min(per_page, 100) # Cap at 100 items per page
|
||||
|
||||
# Build query with optional filtering
|
||||
query = ActivityLog.query
|
||||
|
||||
# Filter by categories if specified
|
||||
categories = request.args.getlist('category')
|
||||
if categories:
|
||||
query = query.filter(ActivityLog.category.in_(categories))
|
||||
|
||||
# Filter by status if specified
|
||||
status = request.args.get('status')
|
||||
if status:
|
||||
query = query.filter(ActivityLog.status == status)
|
||||
|
||||
# Order by most recent first and paginate
|
||||
pagination = query.order_by(ActivityLog.timestamp.desc()).paginate(
|
||||
page=page,
|
||||
per_page=per_page,
|
||||
error_out=False
|
||||
)
|
||||
|
||||
return jsonify({
|
||||
"success": True,
|
||||
@ -251,8 +292,18 @@ def get_logs():
|
||||
"action": log.action,
|
||||
"status": log.status,
|
||||
"description": log.description,
|
||||
"category": log.category.name if log.category else None
|
||||
} for log in logs]
|
||||
"category": log.category
|
||||
} for log in pagination.items],
|
||||
"pagination": {
|
||||
"page": pagination.page,
|
||||
"pages": pagination.pages,
|
||||
"per_page": pagination.per_page,
|
||||
"total": pagination.total,
|
||||
"has_next": pagination.has_next,
|
||||
"has_prev": pagination.has_prev,
|
||||
"next_num": pagination.next_num if pagination.has_next else None,
|
||||
"prev_num": pagination.prev_num if pagination.has_prev else None
|
||||
}
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
|
||||
@ -9,6 +9,22 @@ class ActivityMonitor {
|
||||
this.notificationsEnabled = true;
|
||||
this.lastPaperTimestamp = new Date().toISOString();
|
||||
|
||||
// Pagination state
|
||||
this.currentPage = 1;
|
||||
this.perPage = 20;
|
||||
this.statusFilter = "";
|
||||
this.totalPages = 1;
|
||||
this.totalEntries = 0;
|
||||
|
||||
// Pagination elements
|
||||
this.paginationContainer = document.getElementById("activityPagination");
|
||||
this.paginationInfo = document.getElementById("activityPaginationInfo");
|
||||
this.prevPageBtn = document.getElementById("activityPrevPage");
|
||||
this.nextPageBtn = document.getElementById("activityNextPage");
|
||||
this.currentPageSpan = document.getElementById("activityCurrentPage");
|
||||
this.pageSizeSelect = document.getElementById("activityPageSize");
|
||||
this.statusFilterSelect = document.getElementById("activityStatusFilter");
|
||||
|
||||
this.initEventListeners();
|
||||
this.setupWebSocket();
|
||||
}
|
||||
@ -38,6 +54,45 @@ class ActivityMonitor {
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// Pagination event listeners
|
||||
if (this.prevPageBtn) {
|
||||
this.prevPageBtn.addEventListener("click", (e) => {
|
||||
e.preventDefault();
|
||||
if (this.currentPage > 1) {
|
||||
this.currentPage--;
|
||||
this.loadRecentActivity();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
if (this.nextPageBtn) {
|
||||
this.nextPageBtn.addEventListener("click", (e) => {
|
||||
e.preventDefault();
|
||||
if (this.currentPage < this.totalPages) {
|
||||
this.currentPage++;
|
||||
this.loadRecentActivity();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Page size change
|
||||
if (this.pageSizeSelect) {
|
||||
this.pageSizeSelect.addEventListener("change", () => {
|
||||
this.perPage = parseInt(this.pageSizeSelect.value);
|
||||
this.currentPage = 1; // Reset to first page
|
||||
this.loadRecentActivity();
|
||||
});
|
||||
}
|
||||
|
||||
// Status filter change
|
||||
if (this.statusFilterSelect) {
|
||||
this.statusFilterSelect.addEventListener("change", () => {
|
||||
this.statusFilter = this.statusFilterSelect.value;
|
||||
this.currentPage = 1; // Reset to first page
|
||||
this.loadRecentActivity();
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@ -47,16 +102,35 @@ class ActivityMonitor {
|
||||
if (!this.activityLog) return;
|
||||
|
||||
try {
|
||||
const data = await apiRequest(
|
||||
"/api/activity_logs?category=scraper_activity&category=scraper_command&limit=50"
|
||||
);
|
||||
this.renderActivityLog(data);
|
||||
console.log("Activity log refreshed with latest data");
|
||||
// Build query parameters for pagination
|
||||
const params = new URLSearchParams({
|
||||
page: this.currentPage,
|
||||
per_page: this.perPage,
|
||||
});
|
||||
|
||||
// Add multiple category parameters
|
||||
params.append("category", "scraper_activity");
|
||||
params.append("category", "scraper_command");
|
||||
|
||||
if (this.statusFilter) {
|
||||
params.append("status", this.statusFilter);
|
||||
}
|
||||
|
||||
const data = await apiRequest(`/scraper/logs?${params.toString()}`);
|
||||
|
||||
if (data.success) {
|
||||
this.renderActivityLog(data.logs);
|
||||
this.updatePagination(data.pagination);
|
||||
console.log("Activity log refreshed with latest data");
|
||||
} else {
|
||||
throw new Error(data.message || "Failed to load logs");
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Failed to load activity logs:", error);
|
||||
// If the API endpoint doesn't exist, just show a message
|
||||
this.activityLog.innerHTML =
|
||||
'<tr><td colspan="4" class="text-center">Activity log API not available</td></tr>';
|
||||
this.hidePagination();
|
||||
}
|
||||
}
|
||||
|
||||
@ -95,6 +169,80 @@ class ActivityMonitor {
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Update pagination controls based on API response
|
||||
* @param {Object} pagination - Pagination data from API
|
||||
*/
|
||||
updatePagination(pagination) {
|
||||
if (!pagination || !this.paginationContainer) return;
|
||||
|
||||
this.currentPage = pagination.page;
|
||||
this.totalPages = pagination.pages;
|
||||
this.totalEntries = pagination.total;
|
||||
|
||||
// Show pagination container
|
||||
this.paginationContainer.classList.remove("d-none");
|
||||
|
||||
// Update pagination info
|
||||
const startEntry = (pagination.page - 1) * pagination.per_page + 1;
|
||||
const endEntry = Math.min(
|
||||
pagination.page * pagination.per_page,
|
||||
pagination.total
|
||||
);
|
||||
|
||||
if (this.paginationInfo) {
|
||||
this.paginationInfo.textContent = `Showing ${startEntry} - ${endEntry} of ${pagination.total} entries`;
|
||||
}
|
||||
|
||||
// Update current page display
|
||||
if (this.currentPageSpan) {
|
||||
this.currentPageSpan.textContent = `${pagination.page} of ${pagination.pages}`;
|
||||
}
|
||||
|
||||
// Update previous button
|
||||
if (this.prevPageBtn) {
|
||||
if (pagination.has_prev) {
|
||||
this.prevPageBtn.classList.remove("disabled");
|
||||
this.prevPageBtn.querySelector("a").removeAttribute("tabindex");
|
||||
this.prevPageBtn
|
||||
.querySelector("a")
|
||||
.setAttribute("aria-disabled", "false");
|
||||
} else {
|
||||
this.prevPageBtn.classList.add("disabled");
|
||||
this.prevPageBtn.querySelector("a").setAttribute("tabindex", "-1");
|
||||
this.prevPageBtn
|
||||
.querySelector("a")
|
||||
.setAttribute("aria-disabled", "true");
|
||||
}
|
||||
}
|
||||
|
||||
// Update next button
|
||||
if (this.nextPageBtn) {
|
||||
if (pagination.has_next) {
|
||||
this.nextPageBtn.classList.remove("disabled");
|
||||
this.nextPageBtn.querySelector("a").removeAttribute("tabindex");
|
||||
this.nextPageBtn
|
||||
.querySelector("a")
|
||||
.setAttribute("aria-disabled", "false");
|
||||
} else {
|
||||
this.nextPageBtn.classList.add("disabled");
|
||||
this.nextPageBtn.querySelector("a").setAttribute("tabindex", "-1");
|
||||
this.nextPageBtn
|
||||
.querySelector("a")
|
||||
.setAttribute("aria-disabled", "true");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Hide pagination controls when not needed
|
||||
*/
|
||||
hidePagination() {
|
||||
if (this.paginationContainer) {
|
||||
this.paginationContainer.classList.add("d-none");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Setup WebSocket for real-time notifications
|
||||
*/
|
||||
@ -111,6 +259,7 @@ class ActivityMonitor {
|
||||
if (!this.notificationsEnabled) return;
|
||||
|
||||
try {
|
||||
// Use the API endpoint for checking new papers, with limit for efficiency
|
||||
const data = await apiRequest(
|
||||
`/api/activity_logs?category=scraper_activity&category=scraper_command&action=scrape_paper&after=${this.lastPaperTimestamp}&limit=5`
|
||||
);
|
||||
@ -139,7 +288,10 @@ class ActivityMonitor {
|
||||
if (this.onChartRefresh) {
|
||||
this.onChartRefresh();
|
||||
}
|
||||
this.loadRecentActivity();
|
||||
// Only reload if we're on page 1 to avoid disrupting user navigation
|
||||
if (this.currentPage === 1) {
|
||||
this.loadRecentActivity();
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
// If the API endpoint doesn't exist, do nothing
|
||||
@ -153,4 +305,24 @@ class ActivityMonitor {
|
||||
setChartRefreshCallback(callback) {
|
||||
this.onChartRefresh = callback;
|
||||
}
|
||||
|
||||
/**
|
||||
* Refresh activity log manually (useful for external triggers)
|
||||
*/
|
||||
refresh() {
|
||||
this.loadRecentActivity();
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset pagination to first page
|
||||
*/
|
||||
resetToFirstPage() {
|
||||
this.currentPage = 1;
|
||||
this.loadRecentActivity();
|
||||
}
|
||||
}
|
||||
|
||||
// Export for use in other modules
|
||||
if (typeof window !== "undefined") {
|
||||
window.ActivityMonitor = ActivityMonitor;
|
||||
}
|
||||
|
||||
@ -68,7 +68,14 @@ function createStatusBadge(status) {
|
||||
*/
|
||||
function formatTimestamp(timestamp) {
|
||||
const date = new Date(timestamp);
|
||||
return date.toLocaleTimeString();
|
||||
return date.toLocaleTimeString("de-DE", {
|
||||
year: "2-digit",
|
||||
month: "numeric",
|
||||
day: "numeric",
|
||||
hour: "2-digit",
|
||||
minute: "2-digit",
|
||||
second: "2-digit",
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@ -75,6 +75,11 @@
|
||||
.badge-failed {
|
||||
background-color: #dc3545;
|
||||
}
|
||||
|
||||
.activity-controls {
|
||||
width: auto;
|
||||
display: inline-block;
|
||||
}
|
||||
</style>
|
||||
{% endblock styles %}
|
||||
|
||||
@ -232,8 +237,29 @@
|
||||
<div class="row mb-4">
|
||||
<div class="col-12">
|
||||
<div class="card">
|
||||
<div class="card-header">
|
||||
<div class="card-header d-flex justify-content-between align-items-center">
|
||||
<h5>Recent Activity</h5>
|
||||
<div class="d-flex align-items-center gap-3">
|
||||
<div class="form-group mb-0">
|
||||
<label for="activityPageSize" class="form-label mb-0 me-2">Show:</label>
|
||||
<select id="activityPageSize" class="form-select form-select-sm activity-controls">
|
||||
<option value="10">10</option>
|
||||
<option value="20" selected>20</option>
|
||||
<option value="50">50</option>
|
||||
<option value="100">100</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="form-group mb-0">
|
||||
<label for="activityStatusFilter" class="form-label mb-0 me-2">Status:</label>
|
||||
<select id="activityStatusFilter" class="form-select form-select-sm activity-controls">
|
||||
<option value="">All</option>
|
||||
<option value="success">Success</option>
|
||||
<option value="error">Error</option>
|
||||
<option value="pending">Pending</option>
|
||||
<option value="info">Info</option>
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="card-body">
|
||||
<div class="table-responsive">
|
||||
@ -253,6 +279,30 @@
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<!-- Pagination Controls -->
|
||||
<nav id="activityPagination" aria-label="Activity log pagination" class="mt-3 d-none">
|
||||
<div class="d-flex justify-content-between align-items-center">
|
||||
<div class="text-muted">
|
||||
<span id="activityPaginationInfo">Showing 0 - 0 of 0 entries</span>
|
||||
</div>
|
||||
<ul class="pagination pagination-sm mb-0">
|
||||
<li class="page-item" id="activityPrevPage">
|
||||
<a class="page-link" href="#" aria-label="Previous">
|
||||
<span aria-hidden="true">«</span>
|
||||
</a>
|
||||
</li>
|
||||
<li class="page-item active" id="activityCurrentPage">
|
||||
<span class="page-link">1</span>
|
||||
</li>
|
||||
<li class="page-item" id="activityNextPage">
|
||||
<a class="page-link" href="#" aria-label="Next">
|
||||
<span aria-hidden="true">»</span>
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</nav>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user