diff --git a/.gitignore b/.gitignore index fe67c43..9c0daba 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,7 @@ dist/ *.egg-info/ .pytest_cache/ .mypy_cache/ + +*.db + +*.R \ No newline at end of file diff --git a/scipaperloader/blueprints/main.py b/scipaperloader/blueprints/main.py index be4cd58..b7f2827 100644 --- a/scipaperloader/blueprints/main.py +++ b/scipaperloader/blueprints/main.py @@ -6,14 +6,14 @@ bp = Blueprint("main", __name__) @bp.route("/") def index(): - return render_template("index.html") + return render_template("index.html.jina") @bp.route("/logs") def logs(): - return render_template("logs.html", app_title="PaperScraper") + return render_template("logs.html.jina", app_title="PaperScraper") @bp.route("/about") def about(): - return render_template("about.html", app_title="PaperScraper") \ No newline at end of file + return render_template("about.html.jina", app_title="PaperScraper") \ No newline at end of file diff --git a/scipaperloader/blueprints/papers.py b/scipaperloader/blueprints/papers.py index 890271e..fba87d3 100644 --- a/scipaperloader/blueprints/papers.py +++ b/scipaperloader/blueprints/papers.py @@ -72,7 +72,7 @@ def list_papers(): status_counts = {status: count for status, count in status_counts} return render_template( - "papers.html", + "papers.html.jina", papers=pagination.items, pagination=pagination, total_papers=total_papers, @@ -137,4 +137,4 @@ def export_papers(): @bp.route("//detail") def paper_detail(paper_id): paper = PaperMetadata.query.get_or_404(paper_id) - return render_template("partials/paper_detail_modal.html", paper=paper) \ No newline at end of file + return render_template("partials/paper_detail_modal.html.jina", paper=paper) \ No newline at end of file diff --git a/scipaperloader/blueprints/schedule.py b/scipaperloader/blueprints/schedule.py index 7d4f878..f83bb18 100644 --- a/scipaperloader/blueprints/schedule.py +++ b/scipaperloader/blueprints/schedule.py @@ -72,7 +72,7 @@ def schedule(): } volume = VolumeConfig.query.first() return render_template( - "schedule.html", + "schedule.html.jina", schedule=schedule, volume=volume.volume if volume else 0, app_title="PaperScraper", diff --git a/scipaperloader/blueprints/upload.py b/scipaperloader/blueprints/upload.py index 07bc7d1..73fcf48 100644 --- a/scipaperloader/blueprints/upload.py +++ b/scipaperloader/blueprints/upload.py @@ -32,19 +32,19 @@ def upload(): duplicate_strategy = request.form.get("duplicate_strategy", "skip") if not file: - return render_template("upload.html", error="No file selected.") + return render_template("upload.html.jina", error="No file selected.") try: stream = codecs.iterdecode(file.stream, "utf-8") content = "".join(stream) df = pd.read_csv(StringIO(content), delimiter=delimiter) except Exception as e: - return render_template("upload.html", error=f"Failed to read CSV file: {e}") + return render_template("upload.html.jina", error=f"Failed to read CSV file: {e}") missing = REQUIRED_COLUMNS - set(df.columns) if missing: return render_template( - "upload.html", error=f"Missing required columns: {', '.join(missing)}" + "upload.html.jina", error=f"Missing required columns: {', '.join(missing)}" ) # Optional: parse 'published_online' to date @@ -126,7 +126,7 @@ def upload(): except Exception as e: db.session.rollback() return render_template( - "upload.html", error=f"Failed to save data to database: {e}" + "upload.html.jina", error=f"Failed to save data to database: {e}" ) # Prepare error samples for display @@ -145,13 +145,13 @@ def upload(): session["error_data"] = error_csv.getvalue() return render_template( - "upload.html", + "upload.html.jina", success=f"File processed! Added: {added_count}, Updated: {updated_count}, Skipped: {skipped_count}, Errors: {error_count}", error_message=error_message, error_samples=error_samples ) - return render_template("upload.html") + return render_template("upload.html.jina") @bp.route("/download_error_log") diff --git a/scipaperloader/templates/about.html b/scipaperloader/templates/about.html.jinja similarity index 98% rename from scipaperloader/templates/about.html rename to scipaperloader/templates/about.html.jinja index 2a3b168..58706ea 100644 --- a/scipaperloader/templates/about.html +++ b/scipaperloader/templates/about.html.jinja @@ -1,4 +1,4 @@ -{% extends 'base.html' %} {% block content %} +{% extends "base.html" %} {% block content %}

📘 About This App

@@ -107,4 +107,4 @@

  • Anyone needing a structured way to fetch and track papers in bulk
    • -{% endblock %} +{% endblock content %} \ No newline at end of file diff --git a/scipaperloader/templates/base.html b/scipaperloader/templates/base.html deleted file mode 100644 index 4992010..0000000 --- a/scipaperloader/templates/base.html +++ /dev/null @@ -1,22 +0,0 @@ - - - - - {{ app_title }} - - - - - - - {% include 'nav.html' %} -
    {% block content %}{% endblock %}
    - {% include 'footer.html' %} - - diff --git a/scipaperloader/templates/base.html.jinja b/scipaperloader/templates/base.html.jinja new file mode 100644 index 0000000..9c2ec9e --- /dev/null +++ b/scipaperloader/templates/base.html.jinja @@ -0,0 +1,21 @@ + + + + + + + + {{ app_title }} + + + + + + + + {% include "nav.html" %} +
    {% block content %}{% endblock content %}
    + {% include "footer.html" %} + + + \ No newline at end of file diff --git a/scipaperloader/templates/footer.html b/scipaperloader/templates/footer.html.jinja similarity index 100% rename from scipaperloader/templates/footer.html rename to scipaperloader/templates/footer.html.jinja diff --git a/scipaperloader/templates/index.html b/scipaperloader/templates/index.html.jinja similarity index 88% rename from scipaperloader/templates/index.html rename to scipaperloader/templates/index.html.jinja index e9d28a6..16c9cbd 100644 --- a/scipaperloader/templates/index.html +++ b/scipaperloader/templates/index.html.jinja @@ -1,4 +1,5 @@ -{% extends 'base.html' %} {% block content %} +{% extends "base.html" %} +{% block content %}

    Welcome to SciPaperLoader

    @@ -29,7 +30,7 @@ A daemon process runs hourly to fetch papers using Zotero API. Downloads are randomized to mimic human behavior and avoid detection.

    - View Logs + View Logs
    @@ -43,9 +44,7 @@ inspect errors. Files are stored on disk in structured folders per DOI.

    - Browse Papers + Browse Papers @@ -59,11 +58,9 @@ volume (e.g. 2/hour at daytime, 0 at night) to match your bandwidth or usage pattern.

    - Adjust Schedule + Adjust Schedule -{% endblock %} +{% endblock content %} \ No newline at end of file diff --git a/scipaperloader/templates/nav.html b/scipaperloader/templates/nav.html.jinja similarity index 52% rename from scipaperloader/templates/nav.html rename to scipaperloader/templates/nav.html.jinja index bd95753..85e3809 100644 --- a/scipaperloader/templates/nav.html +++ b/scipaperloader/templates/nav.html.jinja @@ -1,17 +1,8 @@ + \ No newline at end of file diff --git a/scipaperloader/templates/papers.html b/scipaperloader/templates/papers.html deleted file mode 100644 index 27ed35f..0000000 --- a/scipaperloader/templates/papers.html +++ /dev/null @@ -1,273 +0,0 @@ -{% extends "base.html" %} -{% block title %}Papers{% endblock %} -{% block content %} - -{# --- Sort direction logic for each column --- #} -{% set title_sort = 'asc' if sort_by != 'title' or sort_dir == 'desc' else 'desc' %} -{% set journal_sort = 'asc' if sort_by != 'journal' or sort_dir == 'desc' else 'desc' %} -{% set doi_sort = 'asc' if sort_by != 'doi' or sort_dir == 'desc' else 'desc' %} -{% set issn_sort = 'asc' if sort_by != 'issn' or sort_dir == 'desc' else 'desc' %} -{% set status_sort = 'asc' if sort_by != 'status' or sort_dir == 'desc' else 'desc' %} -{% set created_sort = 'asc' if sort_by != 'created_at' or sort_dir == 'desc' else 'desc' %} -{% set updated_sort = 'asc' if sort_by != 'updated_at' or sort_dir == 'desc' else 'desc' %} - -
    -
    - - -
    -
    - - -
    -
    - - -
    -
    - - -
    -
    - - -
    -
    - -
    -
    - - - -
    - -
    -
    -
    - Total Papers - {{ total_papers }} -
    - {% for status, count in status_counts.items() %} -
    - {{ status }}: - {{ count }} -
    - {% endfor %} -
    -
    - - - - - -
    - Export CSV -
    -
    - - - - - - - - - - - - - - {% for paper in papers %} - - - - - - - - - - {% endfor %} - -
    - {% set params = request.args.to_dict() %} - {% set params = params.update({'sort_by': 'title', 'sort_dir': title_sort}) or params %} - Title - - {% set params = request.args.to_dict() %} - {% set params = params.update({'sort_by': 'journal', 'sort_dir': journal_sort}) or params %} - Journal - - {% set params = request.args.to_dict() %} - {% set params = params.update({'sort_by': 'doi', 'sort_dir': doi_sort}) or params %} - DOI - - {% set params = request.args.to_dict() %} - {% set params = params.update({'sort_by': 'issn', 'sort_dir': issn_sort}) or params %} - ISSN - - {% set params = request.args.to_dict() %} - {% set params = params.update({'sort_by': 'status', 'sort_dir': status_sort}) or params %} - Status - - {% set params = request.args.to_dict() %} - {% set params = params.update({'sort_by': 'created_at', 'sort_dir': created_sort}) or params %} - Created - - {% set params = request.args.to_dict() %} - {% set params = params.update({'sort_by': 'updated_at', 'sort_dir': updated_sort}) or params %} - Updated -
    - - - {{ paper.title }} - - {{ paper.journal }} - - {{ paper.doi }} - - - {{ paper.issn }}{{ paper.status }}{{ paper.created_at.strftime('%Y-%m-%d %H:%M:%S') }}{{ paper.updated_at.strftime('%Y-%m-%d %H:%M:%S') }}
    - - - - - -{% endblock %} diff --git a/scipaperloader/templates/papers.html.jinja b/scipaperloader/templates/papers.html.jinja new file mode 100644 index 0000000..c7187af --- /dev/null +++ b/scipaperloader/templates/papers.html.jinja @@ -0,0 +1,281 @@ +{% extends "base.html" %} +{% block title %}Papers{% endblock title %} +{% block content %} + +{# --- Sort direction logic for each column --- #} +{% set title_sort = 'asc' if sort_by != 'title' or sort_dir == 'desc' else 'desc' %} +{% set journal_sort = 'asc' if sort_by != 'journal' or sort_dir == 'desc' else 'desc' %} +{% set doi_sort = 'asc' if sort_by != 'doi' or sort_dir == 'desc' else 'desc' %} +{% set issn_sort = 'asc' if sort_by != 'issn' or sort_dir == 'desc' else 'desc' %} +{% set status_sort = 'asc' if sort_by != 'status' or sort_dir == 'desc' else 'desc' %} +{% set created_sort = 'asc' if sort_by != 'created_at' or sort_dir == 'desc' else 'desc' %} +{% set updated_sort = 'asc' if sort_by != 'updated_at' or sort_dir == 'desc' else 'desc' %} + +
    +
    + + +
    +
    + + +
    +
    + + +
    +
    + + +
    +
    + + +
    +
    + +
    +
    + + + +
    + +
    +
    +
    + Total Papers + {{ total_papers }} +
    + {% for status, count in status_counts.items() %} +
    + {{ status }}: + {{ count }} +
    + {% endfor %} +
    +
    + + + + + +
    + Export CSV +
    +
    + + + + + + + + + + + + + + {% for paper in papers %} + + + + + + + + + + {% endfor %} + +
    + {% set params = request.args.to_dict() %} + {% set params = params.update({'sort_by': 'title', 'sort_dir': title_sort}) or params %} + Title + + {% set params = request.args.to_dict() %} + {% set params = params.update({'sort_by': 'journal', 'sort_dir': journal_sort}) or params %} + Journal + + {% set params = request.args.to_dict() %} + {% set params = params.update({'sort_by': 'doi', 'sort_dir': doi_sort}) or params %} + DOI + + {% set params = request.args.to_dict() %} + {% set params = params.update({'sort_by': 'issn', 'sort_dir': issn_sort}) or params %} + ISSN + + {% set params = request.args.to_dict() %} + {% set params = params.update({'sort_by': 'status', 'sort_dir': status_sort}) or params %} + Status + + {% set params = request.args.to_dict() %} + {% set params = params.update({'sort_by': 'created_at', 'sort_dir': created_sort}) or params %} + Created + + {% set params = request.args.to_dict() %} + {% set params = params.update({'sort_by': 'updated_at', 'sort_dir': updated_sort}) or params %} + Updated +
    + + + {{ paper.title }} + + {{ paper.journal }} + + {{ paper.doi }} + + + {{ paper.issn }}{{ paper.status }}{{ paper.created_at.strftime('%Y-%m-%d %H:%M:%S') }}{{ paper.updated_at.strftime('%Y-%m-%d %H:%M:%S') }}
    + + + + +{% endblock content %} \ No newline at end of file diff --git a/scipaperloader/templates/partials/paper_detail_modal.html b/scipaperloader/templates/partials/paper_detail_modal.html.jinja similarity index 80% rename from scipaperloader/templates/partials/paper_detail_modal.html rename to scipaperloader/templates/partials/paper_detail_modal.html.jinja index a1b24d9..4b71eb5 100644 --- a/scipaperloader/templates/partials/paper_detail_modal.html +++ b/scipaperloader/templates/partials/paper_detail_modal.html.jinja @@ -12,18 +12,13 @@

    ISSN: {% for issn in value.split(',') %} - {{ issn.strip() }}{% if not loop.last %}, {% endif %} {% endfor %} + {{ issn.strip() }}{% if not + loop.last %}, {% endif %} {% endfor %}

    {% else %}

    ISSN: - {{ value }} + {{ value }}

    {% endif %} {% endif %} {% if not key.startswith('_') and key != 'metadata' and key != 'doi' and key != 'issn' %} @@ -34,4 +29,4 @@ - + \ No newline at end of file diff --git a/scipaperloader/templates/schedule.html b/scipaperloader/templates/schedule.html.jinja similarity index 84% rename from scipaperloader/templates/schedule.html rename to scipaperloader/templates/schedule.html.jinja index f60cff8..f51c3dd 100644 --- a/scipaperloader/templates/schedule.html +++ b/scipaperloader/templates/schedule.html.jinja @@ -1,15 +1,17 @@ -{% extends 'base.html' %} {% block content %} +{% extends "base.html" %} {% block content %}