Initial commit with translated description
This commit is contained in:
117
scripts/crawl.py
Normal file
117
scripts/crawl.py
Normal file
@@ -0,0 +1,117 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Firecrawl crawl script for crawling entire websites."""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
from urllib.error import HTTPError
|
||||
|
||||
|
||||
def start_crawl(url: str, max_pages: int = 50, exclude_paths: list = None):
|
||||
"""Start a crawl job."""
|
||||
api_key = os.environ.get("FIRECRAWL_API_KEY")
|
||||
if not api_key:
|
||||
print("Error: FIRECRAWL_API_KEY not set", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
req_url = "https://api.firecrawl.dev/v1/crawl"
|
||||
|
||||
payload = {
|
||||
"url": url,
|
||||
"limit": max_pages,
|
||||
"scrapeOptions": {
|
||||
"formats": ["markdown"],
|
||||
"onlyMainContent": True
|
||||
}
|
||||
}
|
||||
|
||||
if exclude_paths:
|
||||
payload["excludePaths"] = exclude_paths
|
||||
|
||||
data = json.dumps(payload).encode()
|
||||
|
||||
req = urllib.request.Request(
|
||||
req_url,
|
||||
data=data,
|
||||
headers={
|
||||
"Authorization": f"Bearer {api_key}",
|
||||
"Content-Type": "application/json"
|
||||
},
|
||||
method="POST"
|
||||
)
|
||||
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
return json.loads(resp.read().decode())
|
||||
except HTTPError as e:
|
||||
print(f"Error: {e.code} - {e.reason}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def check_crawl_status(job_id: str):
|
||||
"""Check crawl job status."""
|
||||
api_key = os.environ.get("FIRECRAWL_API_KEY")
|
||||
req_url = f"https://api.firecrawl.dev/v1/crawl/{job_id}"
|
||||
|
||||
req = urllib.request.Request(
|
||||
req_url,
|
||||
headers={"Authorization": f"Bearer {api_key}"}
|
||||
)
|
||||
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
return json.loads(resp.read().decode())
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Crawl a website via Firecrawl")
|
||||
parser.add_argument("url", help="URL to start crawling from")
|
||||
parser.add_argument("--max-pages", type=int, default=50, help="Max pages to crawl")
|
||||
parser.add_argument("--wait", action="store_true", help="Wait for completion")
|
||||
parser.add_argument("--json", action="store_true", help="Output raw JSON")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Start crawl
|
||||
result = start_crawl(args.url, args.max_pages)
|
||||
|
||||
if not result.get("success"):
|
||||
print("Error: Failed to start crawl", file=sys.stderr)
|
||||
print(json.dumps(result, indent=2))
|
||||
sys.exit(1)
|
||||
|
||||
job_id = result.get("id")
|
||||
print(f"Crawl started: {job_id}")
|
||||
|
||||
if not args.wait:
|
||||
print(f"Check status with: firecrawl_crawl_status {job_id}")
|
||||
return
|
||||
|
||||
# Poll for completion
|
||||
print("Waiting for completion...")
|
||||
while True:
|
||||
status = check_crawl_status(job_id)
|
||||
|
||||
if status.get("status") in ["completed", "failed", "cancelled"]:
|
||||
break
|
||||
|
||||
print(f"Status: {status.get('status')}...")
|
||||
time.sleep(2)
|
||||
|
||||
if args.json:
|
||||
print(json.dumps(status, indent=2))
|
||||
else:
|
||||
print(f"\nCrawl {status.get('status')}")
|
||||
if "data" in status:
|
||||
print(f"Pages crawled: {len(status['data'])}")
|
||||
for page in status["data"]:
|
||||
print(f"\n{'='*60}")
|
||||
print(f"URL: {page.get('metadata', {}).get('sourceURL', 'N/A')}")
|
||||
if "markdown" in page:
|
||||
preview = page["markdown"][:300] + "..." if len(page["markdown"]) > 300 else page["markdown"]
|
||||
print(f"Preview: {preview}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
89
scripts/scrape.py
Normal file
89
scripts/scrape.py
Normal file
@@ -0,0 +1,89 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Firecrawl scrape script for single URLs."""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import urllib.request
|
||||
from urllib.error import HTTPError
|
||||
|
||||
|
||||
def scrape(url: str, formats: list = None, only_main: bool = True):
|
||||
"""Scrape a URL using Firecrawl."""
|
||||
api_key = os.environ.get("FIRECRAWL_API_KEY")
|
||||
if not api_key:
|
||||
print("Error: FIRECRAWL_API_KEY not set", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
formats = formats or ["markdown"]
|
||||
|
||||
req_url = "https://api.firecrawl.dev/v1/scrape"
|
||||
|
||||
data = json.dumps({
|
||||
"url": url,
|
||||
"formats": formats,
|
||||
"onlyMainContent": only_main
|
||||
}).encode()
|
||||
|
||||
req = urllib.request.Request(
|
||||
req_url,
|
||||
data=data,
|
||||
headers={
|
||||
"Authorization": f"Bearer {api_key}",
|
||||
"Content-Type": "application/json"
|
||||
},
|
||||
method="POST"
|
||||
)
|
||||
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=60) as resp:
|
||||
result = json.loads(resp.read().decode())
|
||||
return result
|
||||
except HTTPError as e:
|
||||
print(f"Error: {e.code} - {e.reason}", file=sys.stderr)
|
||||
print(e.read().decode(), file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Scrape a URL via Firecrawl")
|
||||
parser.add_argument("url", help="URL to scrape")
|
||||
parser.add_argument("--html", action="store_true", help="Include HTML format")
|
||||
parser.add_argument("--markdown", action="store_true", default=True, help="Include markdown format")
|
||||
parser.add_argument("--screenshot", action="store_true", help="Include screenshot")
|
||||
parser.add_argument("--json", action="store_true", help="Output raw JSON")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
formats = []
|
||||
if args.html:
|
||||
formats.append("html")
|
||||
if args.markdown:
|
||||
formats.append("markdown")
|
||||
if args.screenshot:
|
||||
formats.append("screenshot")
|
||||
|
||||
result = scrape(args.url, formats)
|
||||
|
||||
if args.json:
|
||||
print(json.dumps(result, indent=2))
|
||||
return
|
||||
|
||||
# Pretty print results
|
||||
if result.get("success") and "data" in result:
|
||||
data = result["data"]
|
||||
print(f"Title: {data.get('metadata', {}).get('title', 'N/A')}")
|
||||
print(f"URL: {data.get('metadata', {}).get('sourceURL', args.url)}")
|
||||
print(f"\n{'='*60}\n")
|
||||
|
||||
if "markdown" in data:
|
||||
print(data["markdown"])
|
||||
elif "html" in data:
|
||||
print(data["html"][:5000])
|
||||
else:
|
||||
print("Error: Failed to scrape")
|
||||
print(json.dumps(result, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
77
scripts/search.py
Normal file
77
scripts/search.py
Normal file
@@ -0,0 +1,77 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Firecrawl web search script."""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import urllib.request
|
||||
from urllib.error import HTTPError
|
||||
|
||||
|
||||
def search(query: str, limit: int = 10):
|
||||
"""Search the web using Firecrawl."""
|
||||
api_key = os.environ.get("FIRECRAWL_API_KEY")
|
||||
if not api_key:
|
||||
print("Error: FIRECRAWL_API_KEY not set", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
url = "https://api.firecrawl.dev/v1/search"
|
||||
|
||||
data = json.dumps({
|
||||
"query": query,
|
||||
"limit": limit,
|
||||
"lang": "en",
|
||||
"country": "us"
|
||||
}).encode()
|
||||
|
||||
req = urllib.request.Request(
|
||||
url,
|
||||
data=data,
|
||||
headers={
|
||||
"Authorization": f"Bearer {api_key}",
|
||||
"Content-Type": "application/json"
|
||||
},
|
||||
method="POST"
|
||||
)
|
||||
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
result = json.loads(resp.read().decode())
|
||||
return result
|
||||
except HTTPError as e:
|
||||
print(f"Error: {e.code} - {e.reason}", file=sys.stderr)
|
||||
print(e.read().decode(), file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Search the web via Firecrawl")
|
||||
parser.add_argument("query", help="Search query")
|
||||
parser.add_argument("--limit", type=int, default=10, help="Max results")
|
||||
parser.add_argument("--json", action="store_true", help="Output raw JSON")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
result = search(args.query, args.limit)
|
||||
|
||||
if args.json:
|
||||
print(json.dumps(result, indent=2))
|
||||
return
|
||||
|
||||
# Pretty print results
|
||||
if result.get("success") and "data" in result:
|
||||
for item in result["data"]:
|
||||
print(f"\n{'='*60}")
|
||||
print(f"Title: {item.get('title', 'N/A')}")
|
||||
print(f"URL: {item.get('url', 'N/A')}")
|
||||
print(f"Description: {item.get('description', 'N/A')}")
|
||||
if "markdown" in item:
|
||||
content = item["markdown"][:500] + "..." if len(item["markdown"]) > 500 else item["markdown"]
|
||||
print(f"Content preview:\n{content}")
|
||||
else:
|
||||
print("No results found")
|
||||
print(json.dumps(result, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user