import subprocess
import pandas as pd
from urllib.parse import urlparse
import signal
import sys

df = pd.read_excel('web_sites_full.xlsx', sheet_name='Sheet1')

processes = []
# 定义 SIGINT 信号处理函数
def sigint_handler(signal, frame):
    print('收到 Ctrl-C 信号，正在关闭子进程...')
    for process in processes:
        process.terminate()
    print('子进程已关闭，程序退出。')
    sys.exit(0)
# 注册 SIGINT 信号处理函数
signal.signal(signal.SIGINT, sigint_handler)


ind = 0
for ind, row in df.iterrows():
        group = row['单位']
        name = row['主办']
        url = row['地址']
        if 'http' in url:
            sx = row['地址'].split('http')
            ename = sx[0].strip()
            if ename:
                name = ename
            url = 'http' + sx[1]
        elif 'www' in url:
            sx = row['地址'].split('www')
            ename = sx[0].strip()
            if ename:
                name = ename
            url = 'http://www' + sx[1]
        domain = urlparse(url).netloc
        cmd = ['scrapy', 'crawl', 'basespider', '-a', f'domain={domain}', '-a', f'start_url={url}', '-a', f'name={name}', '-a', f'group={group}', '-o', f'web_dir/{name}_{domain}.xlsx']
        process = subprocess.Popen(cmd)
        processes.append(process)
        ind +=1
        # if ind > 0:
        #     break