SciPaperLoader/scipaperloader/static/js/paper-processor.js

/**
 * Paper search and processing functionality
 */

class PaperProcessor {
  constructor() {
    // DOM elements
    this.searchForm = document.getElementById("searchPaperForm");
    this.searchInput = document.getElementById("paperSearchInput");
    this.searchResults = document.getElementById("searchResults");
    this.paperSearchResults = document.getElementById("paperSearchResults");
    this.scraperSelect = document.getElementById("scraperSelect");

    this.initEventListeners();
    this.loadAvailableScrapers();
  }

  /**
   * Initialize event listeners
   */
  initEventListeners() {
    if (this.searchForm) {
      this.searchForm.addEventListener("submit", (e) => {
        e.preventDefault();
        this.searchPapers();
      });
    }
  }

  /**
   * Load available scraper modules
   */
  async loadAvailableScrapers() {
    if (!this.scraperSelect) return;

    try {
      const data = await apiRequest("/scraper/available_scrapers");

      if (data.success && data.scrapers && data.scrapers.length > 0) {
        // Clear previous options except the default one
        while (this.scraperSelect.options.length > 1) {
          this.scraperSelect.remove(1);
        }

        // Add each scraper as an option
        data.scrapers.forEach((scraper) => {
          const option = document.createElement("option");
          option.value = scraper.name;
          option.textContent = `${
            scraper.name
          } - ${scraper.description.substring(0, 50)}${
            scraper.description.length > 50 ? "..." : ""
          }`;
          if (scraper.is_current) {
            option.textContent += " (system default)";
          }
          this.scraperSelect.appendChild(option);
        });
      } else {
        // If no scrapers or error, add a note
        const option = document.createElement("option");
        option.disabled = true;
        option.textContent = "No scrapers available";
        this.scraperSelect.appendChild(option);
      }
    } catch (error) {
      console.error("Error loading scrapers:", error);
      const option = document.createElement("option");
      option.disabled = true;
      option.textContent = "Error loading scrapers";
      this.scraperSelect.appendChild(option);
    }
  }

  /**
   * Search for papers
   */
  async searchPapers() {
    if (!this.searchInput || !this.paperSearchResults || !this.searchResults)
      return;

    const query = this.searchInput.value.trim();

    if (!query) {
      showFlashMessage("Please enter a search term", "warning");
      return;
    }

    // Show loading message
    this.paperSearchResults.innerHTML =
      '<tr><td colspan="5" class="text-center">Searching papers...</td></tr>';
    this.searchResults.classList.remove("d-none");

    try {
      const data = await apiRequest(
        `/api/papers?query=${encodeURIComponent(query)}`
      );

      if (!data.papers || data.papers.length === 0) {
        this.paperSearchResults.innerHTML =
          '<tr><td colspan="5" class="text-center">No papers found matching your search</td></tr>';
        return;
      }

      this.paperSearchResults.innerHTML = "";

      data.papers.forEach((paper) => {
        const row = document.createElement("tr");

        // Create status badge
        const statusBadge = createStatusBadge(paper.status);

        // Create process button (enabled only for papers not in 'Pending' status)
        const processButtonDisabled =
          paper.status === "Pending" ? "disabled" : "";

        // Truncate title if too long
        const truncatedTitle = truncateText(paper.title, 70);

        row.innerHTML = `
                    <td>${paper.id}</td>
                    <td title="${paper.title}">${truncatedTitle}</td>
                    <td>${paper.doi || "N/A"}</td>
                    <td>${statusBadge}</td>
                    <td>
                        <button class="btn btn-sm btn-primary process-paper-btn" 
                            data-paper-id="${paper.id}" 
                            ${processButtonDisabled}>
                            Process Now
                        </button>
                    </td>
                `;

        this.paperSearchResults.appendChild(row);
      });

      // Add event listeners to the process buttons
      document.querySelectorAll(".process-paper-btn").forEach((btn) => {
        btn.addEventListener("click", () => {
          this.processSinglePaper(btn.getAttribute("data-paper-id"));
        });
      });
    } catch (error) {
      console.error("Error searching papers:", error);
      this.paperSearchResults.innerHTML =
        '<tr><td colspan="5" class="text-center">Error searching papers</td></tr>';
    }
  }

  /**
   * Process a single paper
   * @param {string} paperId - The ID of the paper to process
   */
  async processSinglePaper(paperId) {
    if (!this.scraperSelect) return;

    // Disable all process buttons to prevent multiple clicks
    document.querySelectorAll(".process-paper-btn").forEach((btn) => {
      btn.disabled = true;
    });

    // Show processing status via flash message
    showFlashMessage("Processing paper...", "info");

    // Get selected scraper
    const selectedScraper = this.scraperSelect.value;

    try {
      const data = await apiRequest(`/scraper/process_single/${paperId}`, {
        method: "POST",
        body: JSON.stringify({
          scraper_module: selectedScraper,
        }),
      });

      if (data.success) {
        // Update status in the search results
        const row = document
          .querySelector(`.process-paper-btn[data-paper-id="${paperId}"]`)
          ?.closest("tr");
        if (row) {
          const statusCell = row.querySelector("td:nth-child(4)");
          if (statusCell) {
            statusCell.innerHTML = createStatusBadge("Pending");
          }
        }

        // Show success notification
        showFlashMessage(data.message, "success");

        // Set up polling to check paper status and refresh activity
        this.pollPaperStatus(paperId, 3000, 20);
      } else {
        showFlashMessage(data.message, "error");
      }
    } catch (error) {
      console.error("Error processing paper:", error);
      showFlashMessage("Error processing paper", "error");
    } finally {
      // Re-enable the process buttons after a short delay
      setTimeout(() => {
        document.querySelectorAll(".process-paper-btn").forEach((btn) => {
          if (btn.getAttribute("data-paper-id") !== paperId) {
            btn.disabled = false;
          }
        });
      }, 1000);
    }
  }

  /**
   * Poll paper status until it changes from Pending
   * @param {string} paperId - The paper ID to poll
   * @param {number} interval - Polling interval in milliseconds
   * @param {number} maxAttempts - Maximum number of polling attempts
   */
  pollPaperStatus(paperId, interval = 3000, maxAttempts = 20) {
    let attempts = 0;

    // Immediately refresh activity log to show the initial pending status
    if (this.onActivityRefresh) {
      this.onActivityRefresh();
    }

    const checkStatus = async () => {
      attempts++;
      console.log(
        `Checking status of paper ${paperId}, attempt ${attempts}/${maxAttempts}`
      );

      try {
        const data = await apiRequest(`/api/papers/${paperId}`);

        if (data && data.paper) {
          const paper = data.paper;
          console.log(`Paper status: ${paper.status}`);

          // Update the UI with the current status
          const row = document
            .querySelector(`.process-paper-btn[data-paper-id="${paperId}"]`)
            ?.closest("tr");
          if (row) {
            const statusCell = row.querySelector("td:nth-child(4)");
            if (statusCell) {
              statusCell.innerHTML = createStatusBadge(paper.status);
            }

            // Update processing status message if status changed
            if (paper.status !== "Pending") {
              if (paper.status === "Done") {
                showFlashMessage(
                  `Paper processed successfully: ${paper.title}`,
                  "success"
                );
              } else if (paper.status === "Failed") {
                showFlashMessage(
                  `Paper processing failed: ${
                    paper.error_msg || "Unknown error"
                  }`,
                  "error"
                );
              }
            }
          }

          // Always refresh activity log
          if (this.onActivityRefresh) {
            this.onActivityRefresh();
          }

          // If status is still pending and we haven't reached max attempts, check again
          if (paper.status === "Pending" && attempts < maxAttempts) {
            setTimeout(checkStatus, interval);
          } else {
            // If status changed or we reached max attempts, refresh chart data too
            if (this.onChartRefresh) {
              this.onChartRefresh();
            }

            // If we hit max attempts but status is still pending, show a message
            if (paper.status === "Pending" && attempts >= maxAttempts) {
              showFlashMessage(
                "Paper is still being processed. Check the activity log for updates.",
                "info"
              );
            }
          }
        }
      } catch (error) {
        console.error(`Error polling paper status: ${error}`);
        // If there's an error, we can still try again if under max attempts
        if (attempts < maxAttempts) {
          setTimeout(checkStatus, interval);
        }
      }
    };

    // Start checking
    setTimeout(checkStatus, interval);
  }

  /**
   * Set callback for activity refresh
   */
  setActivityRefreshCallback(callback) {
    this.onActivityRefresh = callback;
  }

  /**
   * Set callback for chart refresh
   */
  setChartRefreshCallback(callback) {
    this.onChartRefresh = callback;
  }
}