diff --git a/.gitignore b/.gitignore index b32a1a3..d749713 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ dbs/* venv/* __pycache__/ -*.pyc \ No newline at end of file +*.pyc +twistd.pid \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 0f103ee..578c6c6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ scrapy==2.8.0 scrapyd==1.4.1 -scrapyd-client==1.2.3 \ No newline at end of file +scrapyd-client==1.2.3 +html2text==2020.1.16 \ No newline at end of file diff --git a/zcspider/settings.py b/zcspider/settings.py index 383e700..fccac85 100644 --- a/zcspider/settings.py +++ b/zcspider/settings.py @@ -91,6 +91,7 @@ ROBOTSTXT_OBEY = True REQUEST_FINGERPRINTER_IMPLEMENTATION = "2.7" TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor" FEED_EXPORT_ENCODING = "utf-8" +LOG_LEVEL = 'INFO' ITEM_PIPELINES = { 'zcspider.pipelines.ZcspiderPipeline': 300,