adds pagination to scraper and improves timestamp formatting

scipaperloader/blueprints/scraper.py

@@ -238,10 +238,51 @@ def get_status():
 
 @bp.route("/logs")
 def get_logs():
-    """Get recent activity logs."""
+    """Get recent activity logs with pagination support."""
     try:
-        limit = request.args.get('limit', 50, type=int)
+        # Pagination parameters
-        logs = ActivityLog.query.order_by(ActivityLog.timestamp.desc()).limit(limit).all()
+        page = request.args.get('page', 1, type=int)
+        per_page = request.args.get('per_page', 20, type=int)
+        
+        # Legacy limit parameter for backward compatibility
+        limit = request.args.get('limit', type=int)
+        if limit and not request.args.get('page'):
+            # Legacy mode: use limit without pagination
+            logs = ActivityLog.query.order_by(ActivityLog.timestamp.desc()).limit(limit).all()
+            return jsonify({
+                "success": True,
+                "logs": [{
+                    "id": log.id,
+                    "timestamp": log.timestamp.isoformat(),
+                    "action": log.action,
+                    "status": log.status,
+                    "description": log.description,
+                    "category": log.category
+                } for log in logs]
+            })
+        
+        # Ensure reasonable per_page limits
+        per_page = min(per_page, 100)  # Cap at 100 items per page
+        
+        # Build query with optional filtering
+        query = ActivityLog.query
+        
+        # Filter by categories if specified
+        categories = request.args.getlist('category')
+        if categories:
+            query = query.filter(ActivityLog.category.in_(categories))
+        
+        # Filter by status if specified
+        status = request.args.get('status')
+        if status:
+            query = query.filter(ActivityLog.status == status)
+        
+        # Order by most recent first and paginate
+        pagination = query.order_by(ActivityLog.timestamp.desc()).paginate(
+            page=page, 
+            per_page=per_page, 
+            error_out=False
+        )
         
         return jsonify({
             "success": True,
@@ -251,8 +292,18 @@ def get_logs():
                 "action": log.action,
                 "status": log.status,
                 "description": log.description,
-                "category": log.category.name if log.category else None
+                "category": log.category
-            } for log in logs]
+            } for log in pagination.items],
+            "pagination": {
+                "page": pagination.page,
+                "pages": pagination.pages,
+                "per_page": pagination.per_page,
+                "total": pagination.total,
+                "has_next": pagination.has_next,
+                "has_prev": pagination.has_prev,
+                "next_num": pagination.next_num if pagination.has_next else None,
+                "prev_num": pagination.prev_num if pagination.has_prev else None
+            }
         })
         
     except Exception as e:

scipaperloader/static/js/activity-monitor.js

@@ -9,6 +9,22 @@ class ActivityMonitor {
     this.notificationsEnabled = true;
     this.lastPaperTimestamp = new Date().toISOString();
 
+    // Pagination state
+    this.currentPage = 1;
+    this.perPage = 20;
+    this.statusFilter = "";
+    this.totalPages = 1;
+    this.totalEntries = 0;
+
+    // Pagination elements
+    this.paginationContainer = document.getElementById("activityPagination");
+    this.paginationInfo = document.getElementById("activityPaginationInfo");
+    this.prevPageBtn = document.getElementById("activityPrevPage");
+    this.nextPageBtn = document.getElementById("activityNextPage");
+    this.currentPageSpan = document.getElementById("activityCurrentPage");
+    this.pageSizeSelect = document.getElementById("activityPageSize");
+    this.statusFilterSelect = document.getElementById("activityStatusFilter");
+
     this.initEventListeners();
     this.setupWebSocket();
   }
@@ -38,6 +54,45 @@ class ActivityMonitor {
         }
       });
     });
+
+    // Pagination event listeners
+    if (this.prevPageBtn) {
+      this.prevPageBtn.addEventListener("click", (e) => {
+        e.preventDefault();
+        if (this.currentPage > 1) {
+          this.currentPage--;
+          this.loadRecentActivity();
+        }
+      });
+    }
+
+    if (this.nextPageBtn) {
+      this.nextPageBtn.addEventListener("click", (e) => {
+        e.preventDefault();
+        if (this.currentPage < this.totalPages) {
+          this.currentPage++;
+          this.loadRecentActivity();
+        }
+      });
+    }
+
+    // Page size change
+    if (this.pageSizeSelect) {
+      this.pageSizeSelect.addEventListener("change", () => {
+        this.perPage = parseInt(this.pageSizeSelect.value);
+        this.currentPage = 1; // Reset to first page
+        this.loadRecentActivity();
+      });
+    }
+
+    // Status filter change
+    if (this.statusFilterSelect) {
+      this.statusFilterSelect.addEventListener("change", () => {
+        this.statusFilter = this.statusFilterSelect.value;
+        this.currentPage = 1; // Reset to first page
+        this.loadRecentActivity();
+      });
+    }
   }
 
   /**
@@ -47,16 +102,35 @@ class ActivityMonitor {
     if (!this.activityLog) return;
 
     try {
-      const data = await apiRequest(
+      // Build query parameters for pagination
-        "/api/activity_logs?category=scraper_activity&category=scraper_command&limit=50"
+      const params = new URLSearchParams({
-      );
+        page: this.currentPage,
-      this.renderActivityLog(data);
+        per_page: this.perPage,
-      console.log("Activity log refreshed with latest data");
+      });
+
+      // Add multiple category parameters
+      params.append("category", "scraper_activity");
+      params.append("category", "scraper_command");
+
+      if (this.statusFilter) {
+        params.append("status", this.statusFilter);
+      }
+
+      const data = await apiRequest(`/scraper/logs?${params.toString()}`);
+
+      if (data.success) {
+        this.renderActivityLog(data.logs);
+        this.updatePagination(data.pagination);
+        console.log("Activity log refreshed with latest data");
+      } else {
+        throw new Error(data.message || "Failed to load logs");
+      }
     } catch (error) {
       console.error("Failed to load activity logs:", error);
       // If the API endpoint doesn't exist, just show a message
       this.activityLog.innerHTML =
         '<tr><td colspan="4" class="text-center">Activity log API not available</td></tr>';
+      this.hidePagination();
     }
   }
 
@@ -95,6 +169,80 @@ class ActivityMonitor {
     });
   }
 
+  /**
+   * Update pagination controls based on API response
+   * @param {Object} pagination - Pagination data from API
+   */
+  updatePagination(pagination) {
+    if (!pagination || !this.paginationContainer) return;
+
+    this.currentPage = pagination.page;
+    this.totalPages = pagination.pages;
+    this.totalEntries = pagination.total;
+
+    // Show pagination container
+    this.paginationContainer.classList.remove("d-none");
+
+    // Update pagination info
+    const startEntry = (pagination.page - 1) * pagination.per_page + 1;
+    const endEntry = Math.min(
+      pagination.page * pagination.per_page,
+      pagination.total
+    );
+
+    if (this.paginationInfo) {
+      this.paginationInfo.textContent = `Showing ${startEntry} - ${endEntry} of ${pagination.total} entries`;
+    }
+
+    // Update current page display
+    if (this.currentPageSpan) {
+      this.currentPageSpan.textContent = `${pagination.page} of ${pagination.pages}`;
+    }
+
+    // Update previous button
+    if (this.prevPageBtn) {
+      if (pagination.has_prev) {
+        this.prevPageBtn.classList.remove("disabled");
+        this.prevPageBtn.querySelector("a").removeAttribute("tabindex");
+        this.prevPageBtn
+          .querySelector("a")
+          .setAttribute("aria-disabled", "false");
+      } else {
+        this.prevPageBtn.classList.add("disabled");
+        this.prevPageBtn.querySelector("a").setAttribute("tabindex", "-1");
+        this.prevPageBtn
+          .querySelector("a")
+          .setAttribute("aria-disabled", "true");
+      }
+    }
+
+    // Update next button
+    if (this.nextPageBtn) {
+      if (pagination.has_next) {
+        this.nextPageBtn.classList.remove("disabled");
+        this.nextPageBtn.querySelector("a").removeAttribute("tabindex");
+        this.nextPageBtn
+          .querySelector("a")
+          .setAttribute("aria-disabled", "false");
+      } else {
+        this.nextPageBtn.classList.add("disabled");
+        this.nextPageBtn.querySelector("a").setAttribute("tabindex", "-1");
+        this.nextPageBtn
+          .querySelector("a")
+          .setAttribute("aria-disabled", "true");
+      }
+    }
+  }
+
+  /**
+   * Hide pagination controls when not needed
+   */
+  hidePagination() {
+    if (this.paginationContainer) {
+      this.paginationContainer.classList.add("d-none");
+    }
+  }
+
   /**
    * Setup WebSocket for real-time notifications
    */
@@ -111,6 +259,7 @@ class ActivityMonitor {
     if (!this.notificationsEnabled) return;
 
     try {
+      // Use the API endpoint for checking new papers, with limit for efficiency
       const data = await apiRequest(
         `/api/activity_logs?category=scraper_activity&category=scraper_command&action=scrape_paper&after=${this.lastPaperTimestamp}&limit=5`
       );
@@ -139,7 +288,10 @@ class ActivityMonitor {
         if (this.onChartRefresh) {
           this.onChartRefresh();
         }
-        this.loadRecentActivity();
+        // Only reload if we're on page 1 to avoid disrupting user navigation
+        if (this.currentPage === 1) {
+          this.loadRecentActivity();
+        }
       }
     } catch (error) {
       // If the API endpoint doesn't exist, do nothing
@@ -153,4 +305,24 @@ class ActivityMonitor {
   setChartRefreshCallback(callback) {
     this.onChartRefresh = callback;
   }
+
+  /**
+   * Refresh activity log manually (useful for external triggers)
+   */
+  refresh() {
+    this.loadRecentActivity();
+  }
+
+  /**
+   * Reset pagination to first page
+   */
+  resetToFirstPage() {
+    this.currentPage = 1;
+    this.loadRecentActivity();
+  }
+}
+
+// Export for use in other modules
+if (typeof window !== "undefined") {
+  window.ActivityMonitor = ActivityMonitor;
 }

scipaperloader/static/js/common.js

@@ -68,7 +68,14 @@ function createStatusBadge(status) {
  */
 function formatTimestamp(timestamp) {
   const date = new Date(timestamp);
-  return date.toLocaleTimeString();
+  return date.toLocaleTimeString("de-DE", {
+    year: "2-digit",
+    month: "numeric",
+    day: "numeric",
+    hour: "2-digit",
+    minute: "2-digit",
+    second: "2-digit",
+  });
 }
 
 /**

scipaperloader/templates/scraper.html.jinja

@@ -75,6 +75,11 @@
     .badge-failed {
         background-color: #dc3545;
     }
+
+    .activity-controls {
+        width: auto;
+        display: inline-block;
+    }
 </style>
 {% endblock styles %}
 
@@ -232,8 +237,29 @@
     <div class="row mb-4">
         <div class="col-12">
             <div class="card">
-                <div class="card-header">
+                <div class="card-header d-flex justify-content-between align-items-center">
                     <h5>Recent Activity</h5>
+                    <div class="d-flex align-items-center gap-3">
+                        <div class="form-group mb-0">
+                            <label for="activityPageSize" class="form-label mb-0 me-2">Show:</label>
+                            <select id="activityPageSize" class="form-select form-select-sm activity-controls">
+                                <option value="10">10</option>
+                                <option value="20" selected>20</option>
+                                <option value="50">50</option>
+                                <option value="100">100</option>
+                            </select>
+                        </div>
+                        <div class="form-group mb-0">
+                            <label for="activityStatusFilter" class="form-label mb-0 me-2">Status:</label>
+                            <select id="activityStatusFilter" class="form-select form-select-sm activity-controls">
+                                <option value="">All</option>
+                                <option value="success">Success</option>
+                                <option value="error">Error</option>
+                                <option value="pending">Pending</option>
+                                <option value="info">Info</option>
+                            </select>
+                        </div>
+                    </div>
                 </div>
                 <div class="card-body">
                     <div class="table-responsive">
@@ -253,6 +279,30 @@
                             </tbody>
                         </table>
                     </div>
+
+                    <!-- Pagination Controls -->
+                    <nav id="activityPagination" aria-label="Activity log pagination" class="mt-3 d-none">
+                        <div class="d-flex justify-content-between align-items-center">
+                            <div class="text-muted">
+                                <span id="activityPaginationInfo">Showing 0 - 0 of 0 entries</span>
+                            </div>
+                            <ul class="pagination pagination-sm mb-0">
+                                <li class="page-item" id="activityPrevPage">
+                                    <a class="page-link" href="#" aria-label="Previous">
+                                        <span aria-hidden="true">«</span>
+                                    </a>
+                                </li>
+                                <li class="page-item active" id="activityCurrentPage">
+                                    <span class="page-link">1</span>
+                                </li>
+                                <li class="page-item" id="activityNextPage">
+                                    <a class="page-link" href="#" aria-label="Next">
+                                        <span aria-hidden="true">»</span>
+                                    </a>
+                                </li>
+                            </ul>
+                        </div>
+                    </nav>
                 </div>
             </div>
         </div>