爬虫学习案例4

news/2024/12/17 20:15:00/

爬取猪八戒网站数据：2024-12-12

使用xpath解析元素，安装依赖库

pip install lxml

使用selenium步骤我的上篇博客有提到，这里就不重复了
selenium使用博客导航

# 安装pip install lxml,使用xpath
from lxml import etree
import time
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options# 设置Chrome选项
chrome_options = Options()
chrome_options.add_argument("--headless")  # 无头模式，不打开浏览器窗口
chrome_options.add_argument("--disable-gpu")
chrome_options.add_argument("--no-sandbox")# 设置ChromeDriver路径
service = Service('D:\\env\\python3\\chromedriver.exe')
keyword = "微信小程序"
url = f"https://www.zbj.com/fw/?k={keyword}"
# 初始化WebDriver
driver = webdriver.Chrome(service=service, options=chrome_options)
driver.get(url)
# 等待页面加载
time.sleep(2)  # 等待内容加载html = driver.page_source # 原页面
# 使用xpath提取元素
tree = etree.HTML(html)
divList = tree.xpath("/html/body/div[2]/div/div/div[3]/div[1]/div[4]/div/div[2]/div[1]/div[2]/div")
for divItem in divList:price_elements = divItem.xpath("./div/div[3]/div[1]/span/text()")title_elements = divItem.xpath("./div/div[3]/div[2]/a/span/text()")company_elements = divItem.xpath("./div/div[5]/div/div/div/text()")sales_elements = divItem.xpath("./div/div[3]/div[3]/div[1]/div/span[2]/text()")good_elements = divItem.xpath("./div/div[3]/div[3]/div[2]/div/span[2]/text()")price = price_elements[0].strip("¥") if price_elements else "N/A"title = keyword.join(title_elements) if title_elements else "N/A"company = company_elements[0] if company_elements else "N/A"sales = sales_elements[0] if sales_elements else "N/A"good = good_elements[0] if good_elements else "N/A"print(f"价格: {price}")print(f"标题: {title}")print(f"商铺名: {company}")print(f"销量: {sales}")print(f"好评: {good}")print("下一家***********************")
driver.quit()  # 关闭浏览器