| |
| """ |
| GODDESS CRAWLER - High-performance async web scraper |
| Sellable tool - £20 one-time |
| |
| Usage: python goddess_crawler.py seeds.txt output_dir/ --workers 100 |
| """ |
| import asyncio |
| import aiohttp |
| import argparse |
| import os |
| import re |
| from pathlib import Path |
| from typing import Set |
| import time |
|
|
| class GoddessCrawler: |
| def __init__(self, output_dir: str, workers: int = 100, timeout: int = 15): |
| self.output = Path(output_dir) |
| self.output.mkdir(exist_ok=True) |
| self.workers = workers |
| self.timeout = aiohttp.ClientTimeout(total=timeout) |
| self.visited: Set[str] = set() |
| self.count = 0 |
| self.bytes = 0 |
| self.errors = 0 |
| self.start = time.time() |
| |
| def strip_html(self, html: str) -> str: |
| html = re.sub(r'<script[^>]*>.*?</script>', '', html, flags=re.DOTALL|re.I) |
| html = re.sub(r'<style[^>]*>.*?</style>', '', html, flags=re.DOTALL|re.I) |
| html = re.sub(r'<[^>]+>', ' ', html) |
| html = re.sub(r'\s+', ' ', html) |
| return html.strip() |
| |
| async def fetch(self, session: aiohttp.ClientSession, url: str) -> str | None: |
| if url in self.visited: |
| return None |
| self.visited.add(url) |
| try: |
| async with session.get(url) as r: |
| if r.status == 200 and 'text' in r.content_type: |
| html = await r.text() |
| return self.strip_html(html) |
| except: |
| self.errors += 1 |
| return None |
| |
| async def process(self, session: aiohttp.ClientSession, url: str): |
| text = await self.fetch(session, url) |
| if text and len(text) > 200: |
| self.count += 1 |
| path = self.output / f"p_{self.count:08d}.txt" |
| content = f"URL: {url}\n\n{text}" |
| path.write_text(content) |
| self.bytes += len(content) |
| |
| async def crawl(self, seeds: list[str]): |
| headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/121.0.0.0"} |
| conn = aiohttp.TCPConnector(limit=self.workers, limit_per_host=10) |
| async with aiohttp.ClientSession(connector=conn, timeout=self.timeout, headers=headers) as session: |
| sem = asyncio.Semaphore(self.workers) |
| async def bounded(url): |
| async with sem: |
| await self.process(session, url) |
| |
| |
| async def status(): |
| while True: |
| await asyncio.sleep(15) |
| elapsed = time.time() - self.start |
| rate = self.bytes / elapsed / 1e6 |
| print(f"[{elapsed:.0f}s] {self.count} pages | {self.bytes/1e9:.2f}GB | {self.errors} err | {rate:.1f}MB/s") |
| |
| status_task = asyncio.create_task(status()) |
| await asyncio.gather(*[bounded(url) for url in seeds]) |
| status_task.cancel() |
| |
| print(f"\nDone! {self.count} pages, {self.bytes/1e9:.2f}GB") |
|
|
| def main(): |
| p = argparse.ArgumentParser(description="High-performance web crawler") |
| p.add_argument("seeds", help="File with URLs, one per line") |
| p.add_argument("output", help="Output directory") |
| p.add_argument("--workers", type=int, default=100, help="Concurrent connections") |
| p.add_argument("--timeout", type=int, default=15, help="Request timeout") |
| args = p.parse_args() |
| |
| seeds = Path(args.seeds).read_text().strip().split('\n') |
| print(f"Loaded {len(seeds)} seeds, {args.workers} workers") |
| |
| crawler = GoddessCrawler(args.output, args.workers, args.timeout) |
| asyncio.run(crawler.crawl(seeds)) |
|
|
| if __name__ == "__main__": |
| main() |
|
|
