Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ jobs:

steps:
- name: Checkout
uses: actions/checkout@v4
uses: actions/checkout@v6

- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v6
Expand Down Expand Up @@ -86,7 +86,7 @@ jobs:
PIP_DISABLE_PIP_VERSION_CHECK: "1"
steps:
- name: Checkout
uses: actions/checkout@v4
uses: actions/checkout@v6

- name: Set up Python 3.12
uses: actions/setup-python@v6
Expand Down Expand Up @@ -134,7 +134,7 @@ jobs:
SKIP_CF_TEST: "0"
steps:
- name: Checkout
uses: actions/checkout@v4
uses: actions/checkout@v6

- name: Set up Python 3.12
uses: actions/setup-python@v6
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/publish.yml
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ jobs:
PIP_DISABLE_PIP_VERSION_CHECK: "1"
steps:
- name: Checkout
uses: actions/checkout@v4
uses: actions/checkout@v6

- name: Set up Python
uses: actions/setup-python@v6
Expand Down
22 changes: 11 additions & 11 deletions run_server.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
# ./run_server.py
"""
Shim for local development.
Please use 'web-scraper-server' (or 'python -m web_scraper_toolkit.server.mcp_server')
when installed.
"""
from web_scraper_toolkit.server.mcp_server import main
if __name__ == "__main__":
main()
# ./run_server.py
"""
Shim for local development.
Please use 'web-scraper-server' (or 'python -m web_scraper_toolkit.server.mcp_server')
when installed.
"""

from web_scraper_toolkit.server.mcp_server import main

if __name__ == "__main__":
main()
224 changes: 112 additions & 112 deletions src/web_scraper_toolkit/core/diagnostics.py
Original file line number Diff line number Diff line change
@@ -1,112 +1,112 @@
# ./src/web_scraper_toolkit/core/diagnostics.py
"""
Diagnostics Module
==================
Provides system self-checks to ensure the environment is correctly configured.
Checks Playwright installation, browser binaries, and network connectivity.
Usage:
print_diagnostics()
Outputs:
- Printed report of system status to stdout.
"""
import asyncio
import logging
import sys
from typing import Dict, Any
from playwright.async_api import async_playwright
logger = logging.getLogger(__name__)
async def verify_environment() -> Dict[str, Any]:
"""
Checks the scraping environment for necessary dependencies and functionality.
Returns:
Dict containing status of:
- python_version
- playwright_installed
- browser_launch_successful
- browsers_found
"""
report: Dict[str, Any] = {
"python_version": sys.version,
"playwright_installed": False,
"browser_launch_successful": False,
"browsers_found": [],
"errors": [],
}
# 1. Check Playwright Import
try:
from importlib.util import find_spec
if find_spec("playwright"):
report["playwright_installed"] = True
else:
report["errors"].append("Playwright package not found.")
except ImportError:
report["errors"].append("Playwright package not installed.")
return report
# 2. Check Browsers
try:
async with async_playwright() as p:
# Try launching Chromium
try:
browser = await p.chromium.launch(headless=True)
report["browsers_found"].append("chromium")
report["browser_launch_successful"] = True
await browser.close()
except Exception as e:
report["errors"].append(f"Chromium launch failed: {e}")
# Try launching Firefox (optional but good to know)
try:
browser = await p.firefox.launch(headless=True)
report["browsers_found"].append("firefox")
await browser.close()
except Exception:
pass # Firefox might not be installed, that's okay
except Exception as e:
report["errors"].append(f"Playwright runtime error: {e}")
return report
def print_diagnostics() -> None:
"""Runs the verification and prints a human-readable report."""
print("Running WebScraperToolkit Diagnostics...")
try:
results = asyncio.run(verify_environment())
print(f"\nPython Version: {results['python_version'].split()[0]}")
print(
f"Playwright Installed: {'✅' if results['playwright_installed'] else '❌'}"
)
if results["playwright_installed"]:
if results["browser_launch_successful"]:
print("Browser Launch: ✅ (Chromium)")
else:
print("Browser Launch: ❌")
print(f"Browsers Detected: {', '.join(results['browsers_found'])}")
if results["errors"]:
print("\n⚠️ Issues Found:")
for err in results["errors"]:
print(f" - {err}")
if "Executable doesn't exist" in str(results["errors"]):
print(
"\nSuggestion: Run `playwright install` to download necessary browsers."
)
except Exception as e:
print(f"Diagnostics failed to run: {e}")
# ./src/web_scraper_toolkit/core/diagnostics.py
"""
Diagnostics Module
==================

Provides system self-checks to ensure the environment is correctly configured.
Checks Playwright installation, browser binaries, and network connectivity.

Usage:
print_diagnostics()

Outputs:
- Printed report of system status to stdout.
"""

import asyncio
import logging
import sys
from typing import Dict, Any

from playwright.async_api import async_playwright

logger = logging.getLogger(__name__)


async def verify_environment() -> Dict[str, Any]:
"""
Checks the scraping environment for necessary dependencies and functionality.

Returns:
Dict containing status of:
- python_version
- playwright_installed
- browser_launch_successful
- browsers_found
"""
report: Dict[str, Any] = {
"python_version": sys.version,
"playwright_installed": False,
"browser_launch_successful": False,
"browsers_found": [],
"errors": [],
}

# 1. Check Playwright Import
try:
from importlib.util import find_spec

if find_spec("playwright"):
report["playwright_installed"] = True
else:
report["errors"].append("Playwright package not found.")
except ImportError:
report["errors"].append("Playwright package not installed.")
return report

# 2. Check Browsers
try:
async with async_playwright() as p:
# Try launching Chromium
try:
browser = await p.chromium.launch(headless=True)
report["browsers_found"].append("chromium")
report["browser_launch_successful"] = True
await browser.close()
except Exception as e:
report["errors"].append(f"Chromium launch failed: {e}")

# Try launching Firefox (optional but good to know)
try:
browser = await p.firefox.launch(headless=True)
report["browsers_found"].append("firefox")
await browser.close()
except Exception:
pass # Firefox might not be installed, that's okay

except Exception as e:
report["errors"].append(f"Playwright runtime error: {e}")

return report


def print_diagnostics() -> None:
"""Runs the verification and prints a human-readable report."""
print("Running WebScraperToolkit Diagnostics...")
try:
results = asyncio.run(verify_environment())

print(f"\nPython Version: {results['python_version'].split()[0]}")
print(
f"Playwright Installed: {'✅' if results['playwright_installed'] else '❌'}"
)

if results["playwright_installed"]:
if results["browser_launch_successful"]:
print("Browser Launch: ✅ (Chromium)")
else:
print("Browser Launch: ❌")

print(f"Browsers Detected: {', '.join(results['browsers_found'])}")

if results["errors"]:
print("\n⚠️ Issues Found:")
for err in results["errors"]:
print(f" - {err}")

if "Executable doesn't exist" in str(results["errors"]):
print(
"\nSuggestion: Run `playwright install` to download necessary browsers."
)
except Exception as e:
print(f"Diagnostics failed to run: {e}")
Loading
Loading