4K 优先 + 滚动翻页:更精致的抓取

4K-First & Infinite Scroll: Polished Crawling

老板经常提图要高清。普通采集可能下到 800x600 的缩略图,这一节我们让脚本优先选 4K 大图。

1. 解析图片多种尺寸

import re


from urllib.parse import urlparse





def upgrade_to_original(url):


    if '?' in url:


        base, query = url.split('?', 1)


        upgraded = re.sub(r'w=\d+', 'w=4000', query)


        return f'{base}?{upgraded}'


    return url

2. 滚动翻页 + 懒加载触发

from selenium import webdriver


import time





def scrape_infinite_scroll(url, target_count=100):


    driver = create_stealth_driver()


    driver.get(url)





    collected = []


    last_count = 0


    scroll_attempt = 0





    while len(collected) < target_count and scroll_attempt < 50:


        driver.execute_script('window.scrollTo(0, document.body.scrollHeight);')


        time.sleep(2)





        imgs = driver.find_elements('css selector', 'img[data-src], img[src]')


        for img in imgs:


            src = img.get_attribute('data-src') or img.get_attribute('src')


            if src and src not in collected:


                collected.append(upgrade_to_original(src))





        if len(collected) == last_count:


            scroll_attempt += 1


        else:


            scroll_attempt = 0


            last_count = len(collected)


            print(f'已收集 {len(collected)} 张')





    driver.quit()


    return collected[:target_count]

3. 完整工程化

image_urls = scrape_infinite_scroll(


    'https://example.com/gallery?keyword=小米',


    target_count=500


)





tasks = [(url, Path(f'downloads/{i:04d}.jpg')) for i, url in enumerate(image_urls)]


with ThreadPoolExecutor(max_workers=20) as executor:


    list(executor.map(lambda t: download_with_retry(*t), tasks))





print(f'共下载 {len(image_urls)} 张 4K 图')

这套流水线能搞定 90% 图片采集需求。我们的图片爬虫软件把这套封装成 GUI — 输关键词、点开始,剩下的全自动。