3 层抓取架构:从 L1 到 L3 兜底

3-Layer Scraping: L1 Requests → L2 Selenium → L3 JS Eval

真实项目里,目标站反爬强度参差不齐。最好的工程方案是分层兜底 — 简单的用 Requests,Requests 不行用 Selenium,Selenium 也不行用 JS 执行兜底。

1. 3 层架构设计

def smart_scrape(url, target_selector):


    try:


        html = requests.get(url, headers=make_headers(), timeout=10).text


        if has_content(html, target_selector):


            return parse_with_bs4(html, target_selector)


    except Exception as e:


        print(f'L1 失败: {e}')





    try:


        driver = create_stealth_driver()


        driver.get(url)


        for _ in range(5):


            driver.execute_script('window.scrollTo(0, document.body.scrollHeight);')


            time.sleep(1.5)


        if has_content(driver.page_source, target_selector):


            return parse_with_selenium(driver, target_selector)


    except Exception as e:


        print(f'L2 失败: {e}')





    try:


        data = driver.execute_script('''


            return fetch("/api/reviews", {credentials: "include"})


                .then(r => r.json())


        ''')


        return data


    except Exception as e:


        print(f'L3 失败: {e}')





    return None

2. 域名预检(避免浪费时间)

HARD_CRAWL_DOMAINS = {


    'mbd.baidu.com',


    'douyin.com',


    'xiaohongshu.com',


}





def should_skip_l1(url):


    from urllib.parse import urlparse


    domain = urlparse(url).netloc


    return any(d in domain for d in HARD_CRAWL_DOMAINS)

3. 数据入库 + 去重

import hashlib


import sqlite3





db = sqlite3.connect('reviews.db')


db.execute('''


    CREATE TABLE IF NOT EXISTS reviews (


        id INTEGER PRIMARY KEY,


        hash TEXT UNIQUE,


        text TEXT,


        rating REAL,


        source TEXT,


        crawled_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP


    )


''')





def save_review(text, rating, source):


    h = hashlib.md5(text.encode('utf-8')).hexdigest()


    try:


        db.execute(


            'INSERT INTO reviews (hash, text, rating, source) VALUES (?, ?, ?, ?)',


            (h, text, rating, source)


        )


        db.commit()


        return True


    except sqlite3.IntegrityError:


        return False

这 3 节 = 3 层抓取架构的完整教程。配合我们的评论采集软件,你不用写代码也能享受这套架构。