##Example.py
# -- coding: utf-8 -- from pyquery import PyQuery as pq
from newspider.interfaces import * from newspider.spider import Newspider
- class DemoFetcher(IntFetcher):
- def __init__(self):
self.next_page = []
- def fetch_detail_urls(self,html):
d = pq(html) list = []
- for a in d(‘.post-title a’):
url = d(a).attr(‘href’) extras = {“category”: “Test for %s” % url} list.append((url,extras))
- for l in d(‘.page-navigator a’):
self.next_page.append(d(l).attr(‘href’))
return list
- def start_page(self):
return [’http://www.typechodev.com/’,’http://www.typechodev.com/index.php/category/questions/’]
- def next_pages(self):
return self.next_page
- class DemoParser(IntParser):
- def parse(self,tag,html,extras):
print “Receive content from url %s for tag %s|%s” % (extras.get(‘_url’),extras.get(‘category’),tag)
- if __name__ == ‘__main__’:
sp = Newspider() sp.config(‘GUARD_INTERVAL’, 0)
sp.add_parser(DemoParser()) sp.add_fetcher(DemoFetcher())
sp.run()
1 maintainer
Metadata
1 maintainer
Release files for newspider 0.9.9
For a detailed explanation of source distributions (sdists) and built distributions (wheels), please see the package formats documentation.
Source distribution (sdist)
| File | Size | Uploaded | |
|---|---|---|---|
| newspider-0.9.9.tar.gz | 8.0 kB | Details |
Release files / newspider-0.9.9.tar.gz
| Download URL | newspider-0.9.9.tar.gz |
|---|---|
| Size | 8.0 kB |
| Tags | Source |
|
SHA-256 checksum How to use checksums |
ec8f628be4c38c9b4aa9e13a38d6ab1e462f9532695262c36b47ae3418a2d109
|
|
BLAKE2b-256 checksum How to use checksums |
5fb386af61ea05bccee02f2658de289ef087cbd71c26a0f7b35e31223414bd4c
|
| Upload date | |
|
Uploaded using Trusted Publishing? What is trusted publishing? |
No |
