SciPaperLoader/scipaperloader/static/js/scraper-control.js

/**
 * Scraper control functionality
 */

class ScraperController {
  constructor(options = {}) {
    this.maxVolume = options.maxVolume || 1000;
    this.volumeConfig = options.volumeConfig || 100;

    // DOM elements
    this.statusIndicator = document.getElementById("statusIndicator");
    this.statusText = document.getElementById("statusText");
    this.startButton = document.getElementById("startButton");
    this.pauseButton = document.getElementById("pauseButton");
    this.stopButton = document.getElementById("stopButton");
    this.resetButton = document.getElementById("resetButton");

    this.initEventListeners();
    this.initStatusPolling();
  }

  /**
   * Initialize event listeners for scraper controls
   */
  initEventListeners() {
    if (this.startButton) {
      this.startButton.addEventListener("click", () => this.startScraper());
    }
    if (this.pauseButton) {
      this.pauseButton.addEventListener("click", () =>
        this.togglePauseScraper()
      );
    }
    if (this.stopButton) {
      this.stopButton.addEventListener("click", () => this.stopScraper());
    }
    if (this.resetButton) {
      this.resetButton.addEventListener("click", () => this.resetScraper());
    }

    // Configuration form (handles both volume and scraper module)
    const configForm = document.getElementById("volumeForm");
    if (configForm) {
      configForm.addEventListener("submit", (e) => {
        e.preventDefault();
        this.updateConfiguration();
      });
    }
  }

  /**
   * Initialize status polling
   */
  initStatusPolling() {
    this.updateStatus();
    setInterval(() => this.updateStatus(), 5000); // Poll every 5 seconds
  }

  /**
   * Update scraper status display
   */
  async updateStatus() {
    try {
      const data = await apiRequest("/scraper/status");
      console.log("Status data received:", data);

      // Remove all status classes first
      if (this.statusIndicator) {
        this.statusIndicator.classList.remove(
          "status-active",
          "status-paused",
          "status-inactive"
        );
      }

      // Handle the new JSON structure with scraper_state
      const scraperState = data.scraper_state || data; // Fallback for old structure

      if (scraperState.active) {
        if (scraperState.paused) {
          this.statusIndicator?.classList.add("status-paused");
          if (this.statusText) this.statusText.textContent = "Paused";
          if (this.pauseButton) this.pauseButton.textContent = "Resume";
        } else {
          this.statusIndicator?.classList.add("status-active");
          if (this.statusText) this.statusText.textContent = "Active";
          if (this.pauseButton) this.pauseButton.textContent = "Pause";
        }
        if (this.startButton) this.startButton.disabled = true;
        if (this.pauseButton) this.pauseButton.disabled = false;
        if (this.stopButton) this.stopButton.disabled = false;
        if (this.resetButton) this.resetButton.disabled = false;
      } else {
        this.statusIndicator?.classList.add("status-inactive");
        if (this.statusText) this.statusText.textContent = "Inactive";
        if (this.startButton) this.startButton.disabled = false;
        if (this.pauseButton) this.pauseButton.disabled = true;
        if (this.stopButton) this.stopButton.disabled = true;
        if (this.resetButton) this.resetButton.disabled = false;
      }
    } catch (error) {
      console.error("Error fetching status:", error);
      // On error, show inactive state
      if (this.statusIndicator) {
        this.statusIndicator.classList.remove(
          "status-active",
          "status-paused",
          "status-inactive"
        );
        this.statusIndicator.classList.add("status-inactive");
      }
      if (this.statusText) this.statusText.textContent = "Error";
    }
  }

  /**
   * Start the scraper
   */
  async startScraper() {
    console.log("Start button clicked - sending request to /scraper/start");

    try {
      const data = await apiRequest("/scraper/start", {
        method: "POST",
        body: JSON.stringify({}),
      });
      console.log("Data received:", data);

      if (data.success) {
        showFlashMessage("Scraper started successfully", "success");
        this.updateStatus();
        // Trigger activity refresh if callback is provided
        if (this.onActivityRefresh) {
          setTimeout(() => this.onActivityRefresh(), 1000);
        }
      } else {
        showFlashMessage(data.message, "error");
      }
    } catch (error) {
      console.error("Error starting scraper:", error);
      showFlashMessage("Error starting scraper: " + error.message, "error");
    }
  }

  /**
   * Toggle pause/resume scraper
   */
  async togglePauseScraper() {
    try {
      const data = await apiRequest("/scraper/pause", {
        method: "POST",
        body: JSON.stringify({}),
      });

      if (data.success) {
        showFlashMessage(data.message, "info");
        this.updateStatus();
        if (this.onActivityRefresh) {
          setTimeout(() => this.onActivityRefresh(), 1000);
        }
      } else {
        showFlashMessage(data.message, "error");
      }
    } catch (error) {
      console.error("Error toggling pause:", error);
      showFlashMessage("Error controlling scraper: " + error.message, "error");
    }
  }

  /**
   * Stop the scraper
   */
  async stopScraper() {
    try {
      const data = await apiRequest("/scraper/stop", {
        method: "POST",
        body: JSON.stringify({}),
      });

      if (data.success) {
        showFlashMessage("Scraper stopped successfully", "warning");
        this.updateStatus();
        if (this.onActivityRefresh) {
          setTimeout(() => this.onActivityRefresh(), 1000);
        }
      } else {
        showFlashMessage(data.message, "error");
      }
    } catch (error) {
      console.error("Error stopping scraper:", error);
      showFlashMessage("Error stopping scraper: " + error.message, "error");
    }
  }

  /**
   * Reset the scraper
   */
  async resetScraper() {
    if (
      !confirm(
        "Are you sure you want to reset the scraper? This will stop all current tasks, optionally clear non-pending papers, and restart the scraper."
      )
    ) {
      return;
    }

    // Disable button to prevent multiple clicks
    if (this.resetButton) this.resetButton.disabled = true;

    // Show a loading message
    showFlashMessage("Resetting scraper, please wait...", "info");

    try {
      const data = await apiRequest("/scraper/reset", {
        method: "POST",
        body: JSON.stringify({
          clear_papers: true, // You could make this configurable with a checkbox
        }),
      });

      if (data.success) {
        showFlashMessage(
          "Scraper has been completely reset and restarted",
          "success"
        );
        // Update everything
        this.updateStatus();
        if (this.onActivityRefresh) {
          this.onActivityRefresh();
          setTimeout(() => this.onActivityRefresh(), 1000);
        }
        if (this.onChartRefresh) {
          this.onChartRefresh();
        }
      } else {
        showFlashMessage(data.message || "Error resetting scraper", "error");
      }
    } catch (error) {
      console.error("Error resetting scraper:", error);
      showFlashMessage("Error resetting scraper: " + error.message, "error");
    } finally {
      // Re-enable button
      if (this.resetButton) this.resetButton.disabled = false;
    }
  }

  /**
   * Update configuration (volume and/or scraper module)
   */
  async updateConfiguration() {
    const volumeInput = document.getElementById("volumeInput");
    const scraperSelect = document.getElementById("mainScraperSelect");
    const submitButton = document.querySelector(
      '#volumeForm button[type="submit"]'
    );

    if (!submitButton) return;

    const updates = {};
    let hasChanges = false;

    // Check volume changes
    if (volumeInput) {
      const volume = volumeInput.value;

      // Basic validation
      if (!volume || volume < 1 || volume > this.maxVolume) {
        showFlashMessage(
          `Please enter a valid volume between 1 and ${this.maxVolume}`,
          "warning"
        );
        volumeInput.focus();
        return;
      }

      updates.volume = volume;
      hasChanges = true;
    }

    // Check scraper module changes
    if (scraperSelect && scraperSelect.value) {
      updates.scraper_module = scraperSelect.value;
      hasChanges = true;
    }

    if (!hasChanges) {
      showFlashMessage("No changes to save", "info");
      return;
    }

    // Toggle loading state
    toggleButtonLoading(submitButton, true, "Updating...");

    try {
      const data = await apiRequest("/scraper/update_config", {
        method: "POST",
        body: JSON.stringify(updates),
      });

      if (data.success) {
        showFlashMessage(
          data.message || "Configuration updated successfully",
          "success"
        );
      } else {
        showFlashMessage(
          data.message || "Failed to update configuration",
          "error"
        );
      }
    } catch (error) {
      console.error("Error updating configuration:", error);
      showFlashMessage(
        "Network error while updating configuration. Please try again.",
        "error"
      );
    } finally {
      toggleButtonLoading(submitButton, false);
    }
  }

  /**
   * Set callback for activity refresh
   */
  setActivityRefreshCallback(callback) {
    this.onActivityRefresh = callback;
  }

  /**
   * Set callback for chart refresh
   */
  setChartRefreshCallback(callback) {
    this.onChartRefresh = callback;
  }
}