mischbeck/SciPaperLoader
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

adds path config checks in config and dummy scraper

Browse Source
This commit is contained in:
Michael Beck 2025-05-23 17:21:41 +02:00
parent 012163ba3f
commit 987c76969b
2 changed files with 81 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
scipaperloader/blueprints/config.py
View File

@ -70,25 +70,31 @@ def _update_download_path(new_path):
# Try to create it if it doesn't exist
try:
os.makedirs(new_path, exist_ok=True)
ActivityLog.log_system_activity(
ActivityLog.log_scraper_activity(
action="create_directory",
status="info",
description=f"Created download directory: {new_path}"
)
except OSError as e:
ActivityLog.log_system_activity(
action="create_directory",
status="error",
description=f"Failed to create download directory: {new_path}, Error: {str(e)}"
ActivityLog.log_error(
error_message=f"Failed to create download directory: {new_path}, Error: {str(e)}",
source="update_download_path"
)
return False, f"Path '{new_path}' is not a valid directory and could not be created: {e}", None
# Check if the path is readable
if not os.access(new_path, os.R_OK):
ActivityLog.log_error(
error_message=f"Download path '{new_path}' is not readable.",
source="check_directory_permissions"
)
return False, f"Path '{new_path}' exists but is not readable by the application.", None
# Check if the path is writable
if not os.access(new_path, os.W_OK):
ActivityLog.log_system_activity(
action="check_directory_permissions",
status="error",
description=f"Download path '{new_path}' is not writable."
ActivityLog.log_error(
error_message=f"Download path '{new_path}' is not writable.",
source="check_directory_permissions"
)
return False, f"Path '{new_path}' exists but is not writable by the application.", None
# --- End of validation ---

68
scipaperloader/scrapers/dummy.py
View File

@ -37,8 +37,72 @@ class Scraper(BaseScraper):
file_name = f"{doi.replace('/', '_')}.pdf"
file_path = f"{download_path}/{file_name}"
# Create directory if it doesn't exist
os.makedirs(download_path, exist_ok=True)
# Check if the path is readable and writable
if not os.path.exists(download_path):
try:
# Create directory if it doesn't exist
os.makedirs(download_path, exist_ok=True)
except OSError as e:
error_msg = f"Failed to create download directory: {str(e)}"
paper.status = "Failed"
paper.error_msg = error_msg
ActivityLog.log_scraper_activity(
action="dummy_scrape_path_error",
status="error",
description=error_msg,
paper_id=paper.id
)
return ScrapeResult(
status="error",
message=error_msg,
data={"error_code": "path_creation_error"},
duration=time.time() - start_time,
timestamp=datetime.utcnow()
)
# Check if the path is readable
if not os.access(download_path, os.R_OK):
error_msg = f"Download path '{download_path}' is not readable"
paper.status = "Failed"
paper.error_msg = error_msg
ActivityLog.log_scraper_activity(
action="dummy_scrape_path_error",
status="error",
description=error_msg,
paper_id=paper.id
)
return ScrapeResult(
status="error",
message=error_msg,
data={"error_code": "path_read_error"},
duration=time.time() - start_time,
timestamp=datetime.utcnow()
)
# Check if the path is writable
if not os.access(download_path, os.W_OK):
error_msg = f"Download path '{download_path}' is not writable"
paper.status = "Failed"
paper.error_msg = error_msg
ActivityLog.log_scraper_activity(
action="dummy_scrape_path_error",
status="error",
description=error_msg,
paper_id=paper.id
)
return ScrapeResult(
status="error",
message=error_msg,
data={"error_code": "path_write_error"},
duration=time.time() - start_time,
timestamp=datetime.utcnow()
)
# Create a simple dummy PDF file
try: