Requests + BeautifulSoup 入门:电商评论

Requests + BeautifulSoup: First Scraper

采集是文本分析的水源 — 没有数据,后续分析全免谈。最简单的入口是 requests + BeautifulSoup。

1. 抓一个商品页(伪代码示例)

import requests


from bs4 import BeautifulSoup


import time





headers = {


    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) '


                  'AppleWebKit/537.36 (KHTML, like Gecko) '


                  'Chrome/120.0.0.0 Safari/537.36',


    'Accept-Language': 'zh-CN,zh;q=0.9',


}





url = 'https://example.com/products/123/reviews?page=1'


resp = requests.get(url, headers=headers, timeout=10)


resp.raise_for_status()





soup = BeautifulSoup(resp.text, 'html.parser')


reviews = []


for item in soup.select('.review-item'):


    text = item.select_one('.review-text').get_text(strip=True)


    rating = item.select_one('.rating').get('data-score')


    reviews.append({'text': text, 'rating': rating})





print(f'抓到 {len(reviews)} 条评论')

2. 翻页循环 + 失败重试

import time


from tenacity import retry, stop_after_attempt, wait_exponential





@retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=2, max=10))


def fetch_page(page_num):


    url = f'https://example.com/products/123/reviews?page={page_num}'


    resp = requests.get(url, headers=headers, timeout=10)


    resp.raise_for_status()


    return resp.text





all_reviews = []


for page in range(1, 101):


    try:


        html = fetch_page(page)


        soup = BeautifulSoup(html, 'html.parser')


        for item in soup.select('.review-item'):


            all_reviews.append({


                'text': item.select_one('.review-text').get_text(strip=True),


                'rating': item.select_one('.rating').get('data-score'),


                'page': page,


            })


    except Exception as e:


        print(f'第 {page} 页失败: {e}')


    time.sleep(2)


    if page % 10 == 0:


        print(f'已抓 {page} 页,共 {len(all_reviews)} 条')





import pandas as pd


pd.DataFrame(all_reviews).to_csv('reviews.csv', index=False)


print(f'总共 {len(all_reviews)} 条已保存')

3. 反爬初步:加 Referer + 随机 UA

import random





UA_LIST = [


    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/120.0.0.0 Safari/537.36',


    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/120.0.0.0 Safari/537.36',


    'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/120.0.0.0 Safari/537.36',


]





def make_headers(referer=None):


    return {


        'User-Agent': random.choice(UA_LIST),


        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',


        'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',


        'Referer': referer or 'https://www.google.com/',


    }

Requests 方案能搞定 50% 公开页,剩下 50% 需要 Selenium 渲染。下一节我们进反检测。