No project description provided
Project description
fastapi_crawler_scheduler
使用
from fastapi_crawler_scheduler import TaskScheduler
from fastapi import FastAPI
import uuid
app = FastAPI()
task_scheduler = TaskScheduler(
app=app,
ssl=True,
project_name="project_name",
uuid_number=uuid.uuid4().__str__(),
redis_username='redis_username',
redis_password='redis_password',
redis_host="redis_host",
redis_port=6379,
)
def add_spider(**crawler_info):
pass
print(f"add_spider = {crawler_info}")
print("add_spider")
crawler_info = {
"topic": "website_washingtonpost",
"name": "华盛顿邮报",
"base_url": "https://www.washingtonpost.com/arcio/news-sitemap/",
"news_node_tag": "url",
"title_tag": "news:title",
"url_tag": "loc",
"need_translation": 1,
"special_language_code": None,
"language_tag": "news:language",
"title_handler_name": "remove_cdata"
}
# 新增任务
task_scheduler.insert_task(func=add_spider, interval=4, job_id="job_1", crawler_info=crawler_info)
# 更新任务
task_scheduler.update_task(func=add_spider, interval=4, job_id="job_1", crawler_info=crawler_info)
# 删除任务
task_scheduler.delete_task(job_id="job_1")
参数介绍
insert_task
# func:Callable 定时任务函数
# interval: int 任务间隔
# job_id: str 任务id
# crawler_info: Dict = None 任务参数
# trigger: str = "interval" 任务类型
update_task
# func:Callable 定时任务函数
# interval: int 任务间隔
# job_id: str 任务id
# crawler_info: Dict = None 任务参数
# trigger: str = "interval" 任务类型
delete_task
# job_id: str 任务id
安装
Pypi
$ pip install fastapi-crawler-scheduler
Project details
Release history Release notifications | RSS feed
Download files
Download the file for your platform. If you're not sure which to choose, learn more about installing packages.
Source Distribution
Built Distribution
Close
Hashes for fastapi-crawler-scheduler-2.0.3.tar.gz
Algorithm | Hash digest | |
---|---|---|
SHA256 | c597cfe2b6daf46147e95a71dd1f72033a25b667db89c51a314f5572b6216e77 |
|
MD5 | 0c9798260a4a54c7e8b35af3ebfce65f |
|
BLAKE2b-256 | ba5996efddf5aeff5e35f52e9aac189539479fc939a733038481aa30e829081b |
Close
Hashes for fastapi_crawler_scheduler-2.0.3-py3-none-any.whl
Algorithm | Hash digest | |
---|---|---|
SHA256 | e709d582f745ad28336457829442ec90ba0860310c7aa2c9c81bcb8584f55b47 |
|
MD5 | ac8bfb961aa5452520e682f5c46c0259 |
|
BLAKE2b-256 | d53c3d431f7e8b12b260c22aeb4396496f8a32626e22ba456dab8dbf97bc07b7 |