aioscrapy 0.1.2

Python asynchronous library for web scraping

Python async library for web scrapping

Installing

pip install aioscrapy

Usage

from aioscrapy import Client, WebClient, SingleSessionPool, Dispatcher, SimpleWorker

class CustomClient(Client[str, dict]):
    def __init__(self, client: WebClient):
        self._client = client

    async def fetch(self, key: str) -> dict:
        data = self._client.fetch(key)
        # Processing data, getting result 
        return {"result": "some_result"}

async def main():
    pool = SingleSessionPool()
    dispatcher = Dispatcher(['https://start-url.example.com'])
    client = CustomClient(WebClient(pool))
    worker = SimpleWorker(dispatcher, client)

    result = await worker.run()
    return result