From 8888e19fc3a368e9b71f9d844ba4e7fd93aecbc6 Mon Sep 17 00:00:00 2001 From: shijing Date: Fri, 25 Aug 2023 16:30:44 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20web3=20scrapy=20cmd=20=E5=9C=B0=E5=9D=80?= =?UTF-8?q?=E4=BF=AE=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- web3.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web3.py b/web3.py index bad4f66..94d0006 100644 --- a/web3.py +++ b/web3.py @@ -12,7 +12,7 @@ from mycode.main import make_simple_csv_from_db, make_wechat_articles_full, ana_ from mycode.crawl_chrome import chrome_main, failed_sites_file python_exe = os.path.join(BASE_DIR, 'runtime/python.exe') -scrapy_exe = os.path.join(BASE_DIR, 'runtime/Scripts/scrapy.exe') +# scrapy_exe = os.path.join(BASE_DIR, 'runtime/Scripts/scrapy.exe') def save_info_to_excel(info_list, output_filename): df = pd.DataFrame(info_list, columns=['单位', '主办' , '地址']) @@ -51,7 +51,7 @@ if __name__ == '__main__': url = row['地址'] domain = urlparse(url).netloc.replace('www.', '') # output = os.path.join(BASE_DIR, f'web_dir/{name}_{domain}.xlsx') - cmd = [scrapy_exe, 'crawl', 'basespider', '-a', f'domain={domain}', '-a', f'start_url={url}', '-a', f'name={name}', '-a', f'group={group}', '-o', f'web_dir/{name}_{domain}.xlsx'] + cmd = [python_exe, '-m', 'scrapy', 'crawl', 'basespider', '-a', f'domain={domain}', '-a', f'start_url={url}', '-a', f'name={name}', '-a', f'group={group}', '-o', f'web_dir/{name}_{domain}.xlsx'] # cmd = ['scrapy', 'crawl', 'basespider', '-a', f'domain={domain}', '-a', f'start_url={url}', '-a', f'name={name}', '-a', f'group={group}', '-a', f'output={output}'] process = subprocess.Popen(cmd) processes.append(process)