import scrapy
from urllib.parse import urlparse
import html2text
import requests
import re
from urllib.parse import urljoin

class BaseSpider(scrapy.Spider):
    name = "basespider"
    start_urls = ["http://ctc.ac.cn/"]
    visited_urls = set()

    def  __init__(self, domain: str, start_url: str, name='',  group='', **kwargs):
         super().__init__(name, **kwargs)
         self.headers = {
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
            'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
            'Cache-Control': 'max-age=0',
            'Proxy-Connection': 'keep-alive',
            'Referer': 'https://www.baidu.com/',
            'Upgrade-Insecure-Requests': '1',
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36 Edg/112.0.1722.48',
        }
         self.domain = domain
         self.start_urls = [start_url]
         self.name = name
         self.group = group
         self.ext = tuple(['.png', '.jpg', '.jpeg', '.pdf', '.doc', '.docx', '.xls', '.xlsx', '.ppt', '.pptx', '.rar', '.zip', '.ico', '.dat', '.css', '.js', '.mp4', '.m3u', '.flv'])
         self.resource_content_types = ['image/', 'text/css', 'application/javascript', 'application/octet-stream']
         print(f"爬取开始: {name}_{domain}")

    def fix_url_scheme(self, url, default_scheme='http'):
        # 检查URL是否包含方案
        if not url.startswith('http://') and not url.startswith('https://'):
            # 如果没有方案，添加默认方案
            url = f'{default_scheme}://{url}'
        return url
    
    def start_requests(self):
        for url in self.start_urls:
            url = self.fix_url_scheme(url)
            self.visited_urls.add(url)
            r = scrapy.Request(url, dont_filter=True, headers=self.headers, callback=self.parse, meta={'download_timeout': 30})
            yield r

    def is_file_url(self, url):
        for item in self.ext:
            if url.lower().endswith(item):
                return True
        return False

    def is_file_res(self, res):
        content_type = res.headers['content-type']
        if isinstance(content_type, bytes):
            content_type = str(content_type, encoding = "utf-8")
        return any(content_type.startswith(prefix) for prefix in self.resource_content_types)

    def request2(self, fail):
        rurl = fail.request.url
        self.logger.info(f'{rurl} 使用requests继续请求')
        r = requests.get(url=fail.request.url, headers=self.headers, timeout=20)
        if r.status_code < 400:
            rtext = r.text
            h = html2text.HTML2Text()
            h.ignore_links = True  # 忽略所有链接
            text = h.handle(rtext)
            yield {
                    'group': self.group,
                    'name': self.name,
                    'domain': self.domain,
                    'url': rurl,
                    'text': text,
                }
            self.visited_urls.add(rurl)
            links = re.findall(r'href=["\']?([^"\'>]+)', r.text)
            for link in links:
                full_link = urljoin(r.url, link)
                if not full_link.startswith('http'):
                    continue
                if full_link not in self.visited_urls and (self.is_file_url(full_link) is False):
                    if urlparse(full_link).netloc.replace('www.', '') == self.domain:
                        yield scrapy.Request(full_link, callback=self.parse, headers=self.headers, errback=self.request2, meta={'download_timeout': 30})
         
    def parse(self, response):
        if response.status >= 500:
            return
        h = html2text.HTML2Text()
        h.ignore_links = True  # 忽略所有链接
        # 提取纯文本内容
        # try:
        text = h.handle(response.text)
        # except:
        #     text = h.handle(response.body.decode(encoding='gb18030'))
        if response.status < 400:
            yield {
                    'group': self.group,
                    'name': self.name,
                    'domain': self.domain,
                    'url': response.url,
                    'text': text,
                }
            links = re.findall(r'href=["\']?([^"\'>]+)', response.text)
            for link in links:
                full_link = response.urljoin(link)
                if not full_link.startswith('http'):
                    continue
                if full_link not in self.visited_urls and (self.is_file_url(full_link) is False):
                    if urlparse(full_link).netloc.replace('www.', '') == self.domain:
                        self.visited_urls.add(response.url)
                        # try:
                        yield scrapy.Request(full_link, callback=self.parse, headers=self.headers, meta={'download_timeout': 30})
                        # except ValueError:
                        #     import traceback
                        #     print(traceback.format_exc())
                        #     print(full_link)

    def closed(self, reason):
        # This method will be called when the Spider is about to close
        print(f'爬取完成: {self.name}_{self.domain}')