diff --git a/scipaperloader/blueprints/scraper.py b/scipaperloader/blueprints/scraper.py
index b73e17b..2fae038 100644
--- a/scipaperloader/blueprints/scraper.py
+++ b/scipaperloader/blueprints/scraper.py
@@ -238,10 +238,51 @@ def get_status():
@bp.route("/logs")
def get_logs():
- """Get recent activity logs."""
+ """Get recent activity logs with pagination support."""
try:
- limit = request.args.get('limit', 50, type=int)
- logs = ActivityLog.query.order_by(ActivityLog.timestamp.desc()).limit(limit).all()
+ # Pagination parameters
+ page = request.args.get('page', 1, type=int)
+ per_page = request.args.get('per_page', 20, type=int)
+
+ # Legacy limit parameter for backward compatibility
+ limit = request.args.get('limit', type=int)
+ if limit and not request.args.get('page'):
+ # Legacy mode: use limit without pagination
+ logs = ActivityLog.query.order_by(ActivityLog.timestamp.desc()).limit(limit).all()
+ return jsonify({
+ "success": True,
+ "logs": [{
+ "id": log.id,
+ "timestamp": log.timestamp.isoformat(),
+ "action": log.action,
+ "status": log.status,
+ "description": log.description,
+ "category": log.category
+ } for log in logs]
+ })
+
+ # Ensure reasonable per_page limits
+ per_page = min(per_page, 100) # Cap at 100 items per page
+
+ # Build query with optional filtering
+ query = ActivityLog.query
+
+ # Filter by categories if specified
+ categories = request.args.getlist('category')
+ if categories:
+ query = query.filter(ActivityLog.category.in_(categories))
+
+ # Filter by status if specified
+ status = request.args.get('status')
+ if status:
+ query = query.filter(ActivityLog.status == status)
+
+ # Order by most recent first and paginate
+ pagination = query.order_by(ActivityLog.timestamp.desc()).paginate(
+ page=page,
+ per_page=per_page,
+ error_out=False
+ )
return jsonify({
"success": True,
@@ -251,8 +292,18 @@ def get_logs():
"action": log.action,
"status": log.status,
"description": log.description,
- "category": log.category.name if log.category else None
- } for log in logs]
+ "category": log.category
+ } for log in pagination.items],
+ "pagination": {
+ "page": pagination.page,
+ "pages": pagination.pages,
+ "per_page": pagination.per_page,
+ "total": pagination.total,
+ "has_next": pagination.has_next,
+ "has_prev": pagination.has_prev,
+ "next_num": pagination.next_num if pagination.has_next else None,
+ "prev_num": pagination.prev_num if pagination.has_prev else None
+ }
})
except Exception as e:
diff --git a/scipaperloader/static/js/activity-monitor.js b/scipaperloader/static/js/activity-monitor.js
index 120cc72..8831f29 100644
--- a/scipaperloader/static/js/activity-monitor.js
+++ b/scipaperloader/static/js/activity-monitor.js
@@ -9,6 +9,22 @@ class ActivityMonitor {
this.notificationsEnabled = true;
this.lastPaperTimestamp = new Date().toISOString();
+ // Pagination state
+ this.currentPage = 1;
+ this.perPage = 20;
+ this.statusFilter = "";
+ this.totalPages = 1;
+ this.totalEntries = 0;
+
+ // Pagination elements
+ this.paginationContainer = document.getElementById("activityPagination");
+ this.paginationInfo = document.getElementById("activityPaginationInfo");
+ this.prevPageBtn = document.getElementById("activityPrevPage");
+ this.nextPageBtn = document.getElementById("activityNextPage");
+ this.currentPageSpan = document.getElementById("activityCurrentPage");
+ this.pageSizeSelect = document.getElementById("activityPageSize");
+ this.statusFilterSelect = document.getElementById("activityStatusFilter");
+
this.initEventListeners();
this.setupWebSocket();
}
@@ -38,6 +54,45 @@ class ActivityMonitor {
}
});
});
+
+ // Pagination event listeners
+ if (this.prevPageBtn) {
+ this.prevPageBtn.addEventListener("click", (e) => {
+ e.preventDefault();
+ if (this.currentPage > 1) {
+ this.currentPage--;
+ this.loadRecentActivity();
+ }
+ });
+ }
+
+ if (this.nextPageBtn) {
+ this.nextPageBtn.addEventListener("click", (e) => {
+ e.preventDefault();
+ if (this.currentPage < this.totalPages) {
+ this.currentPage++;
+ this.loadRecentActivity();
+ }
+ });
+ }
+
+ // Page size change
+ if (this.pageSizeSelect) {
+ this.pageSizeSelect.addEventListener("change", () => {
+ this.perPage = parseInt(this.pageSizeSelect.value);
+ this.currentPage = 1; // Reset to first page
+ this.loadRecentActivity();
+ });
+ }
+
+ // Status filter change
+ if (this.statusFilterSelect) {
+ this.statusFilterSelect.addEventListener("change", () => {
+ this.statusFilter = this.statusFilterSelect.value;
+ this.currentPage = 1; // Reset to first page
+ this.loadRecentActivity();
+ });
+ }
}
/**
@@ -47,16 +102,35 @@ class ActivityMonitor {
if (!this.activityLog) return;
try {
- const data = await apiRequest(
- "/api/activity_logs?category=scraper_activity&category=scraper_command&limit=50"
- );
- this.renderActivityLog(data);
- console.log("Activity log refreshed with latest data");
+ // Build query parameters for pagination
+ const params = new URLSearchParams({
+ page: this.currentPage,
+ per_page: this.perPage,
+ });
+
+ // Add multiple category parameters
+ params.append("category", "scraper_activity");
+ params.append("category", "scraper_command");
+
+ if (this.statusFilter) {
+ params.append("status", this.statusFilter);
+ }
+
+ const data = await apiRequest(`/scraper/logs?${params.toString()}`);
+
+ if (data.success) {
+ this.renderActivityLog(data.logs);
+ this.updatePagination(data.pagination);
+ console.log("Activity log refreshed with latest data");
+ } else {
+ throw new Error(data.message || "Failed to load logs");
+ }
} catch (error) {
console.error("Failed to load activity logs:", error);
// If the API endpoint doesn't exist, just show a message
this.activityLog.innerHTML =
'| Activity log API not available |
';
+ this.hidePagination();
}
}
@@ -95,6 +169,80 @@ class ActivityMonitor {
});
}
+ /**
+ * Update pagination controls based on API response
+ * @param {Object} pagination - Pagination data from API
+ */
+ updatePagination(pagination) {
+ if (!pagination || !this.paginationContainer) return;
+
+ this.currentPage = pagination.page;
+ this.totalPages = pagination.pages;
+ this.totalEntries = pagination.total;
+
+ // Show pagination container
+ this.paginationContainer.classList.remove("d-none");
+
+ // Update pagination info
+ const startEntry = (pagination.page - 1) * pagination.per_page + 1;
+ const endEntry = Math.min(
+ pagination.page * pagination.per_page,
+ pagination.total
+ );
+
+ if (this.paginationInfo) {
+ this.paginationInfo.textContent = `Showing ${startEntry} - ${endEntry} of ${pagination.total} entries`;
+ }
+
+ // Update current page display
+ if (this.currentPageSpan) {
+ this.currentPageSpan.textContent = `${pagination.page} of ${pagination.pages}`;
+ }
+
+ // Update previous button
+ if (this.prevPageBtn) {
+ if (pagination.has_prev) {
+ this.prevPageBtn.classList.remove("disabled");
+ this.prevPageBtn.querySelector("a").removeAttribute("tabindex");
+ this.prevPageBtn
+ .querySelector("a")
+ .setAttribute("aria-disabled", "false");
+ } else {
+ this.prevPageBtn.classList.add("disabled");
+ this.prevPageBtn.querySelector("a").setAttribute("tabindex", "-1");
+ this.prevPageBtn
+ .querySelector("a")
+ .setAttribute("aria-disabled", "true");
+ }
+ }
+
+ // Update next button
+ if (this.nextPageBtn) {
+ if (pagination.has_next) {
+ this.nextPageBtn.classList.remove("disabled");
+ this.nextPageBtn.querySelector("a").removeAttribute("tabindex");
+ this.nextPageBtn
+ .querySelector("a")
+ .setAttribute("aria-disabled", "false");
+ } else {
+ this.nextPageBtn.classList.add("disabled");
+ this.nextPageBtn.querySelector("a").setAttribute("tabindex", "-1");
+ this.nextPageBtn
+ .querySelector("a")
+ .setAttribute("aria-disabled", "true");
+ }
+ }
+ }
+
+ /**
+ * Hide pagination controls when not needed
+ */
+ hidePagination() {
+ if (this.paginationContainer) {
+ this.paginationContainer.classList.add("d-none");
+ }
+ }
+
/**
* Setup WebSocket for real-time notifications
*/
@@ -111,6 +259,7 @@ class ActivityMonitor {
if (!this.notificationsEnabled) return;
try {
+ // Use the API endpoint for checking new papers, with limit for efficiency
const data = await apiRequest(
`/api/activity_logs?category=scraper_activity&category=scraper_command&action=scrape_paper&after=${this.lastPaperTimestamp}&limit=5`
);
@@ -139,7 +288,10 @@ class ActivityMonitor {
if (this.onChartRefresh) {
this.onChartRefresh();
}
- this.loadRecentActivity();
+ // Only reload if we're on page 1 to avoid disrupting user navigation
+ if (this.currentPage === 1) {
+ this.loadRecentActivity();
+ }
}
} catch (error) {
// If the API endpoint doesn't exist, do nothing
@@ -153,4 +305,24 @@ class ActivityMonitor {
setChartRefreshCallback(callback) {
this.onChartRefresh = callback;
}
+
+ /**
+ * Refresh activity log manually (useful for external triggers)
+ */
+ refresh() {
+ this.loadRecentActivity();
+ }
+
+ /**
+ * Reset pagination to first page
+ */
+ resetToFirstPage() {
+ this.currentPage = 1;
+ this.loadRecentActivity();
+ }
+}
+
+// Export for use in other modules
+if (typeof window !== "undefined") {
+ window.ActivityMonitor = ActivityMonitor;
}
diff --git a/scipaperloader/static/js/common.js b/scipaperloader/static/js/common.js
index 3dd742c..9647292 100644
--- a/scipaperloader/static/js/common.js
+++ b/scipaperloader/static/js/common.js
@@ -68,7 +68,14 @@ function createStatusBadge(status) {
*/
function formatTimestamp(timestamp) {
const date = new Date(timestamp);
- return date.toLocaleTimeString();
+ return date.toLocaleTimeString("de-DE", {
+ year: "2-digit",
+ month: "numeric",
+ day: "numeric",
+ hour: "2-digit",
+ minute: "2-digit",
+ second: "2-digit",
+ });
}
/**
diff --git a/scipaperloader/templates/scraper.html.jinja b/scipaperloader/templates/scraper.html.jinja
index c79444a..e49c935 100644
--- a/scipaperloader/templates/scraper.html.jinja
+++ b/scipaperloader/templates/scraper.html.jinja
@@ -75,6 +75,11 @@
.badge-failed {
background-color: #dc3545;
}
+
+ .activity-controls {
+ width: auto;
+ display: inline-block;
+ }
{% endblock styles %}
@@ -232,8 +237,29 @@