使用requests爬取电影天堂数据，用lxml和Xpath-CSDN博客

本文介绍了一种使用Python和XPath语法从电影天堂网站抓取最新电影及其详细信息的方法，包括电影标题、封面、年份、产地、类别等关键数据。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

完整代码

import requests
from lxml import etree

BASE_DOMAIN = 'http://www.ygdy8.net'
# url = 'http://www.ygdy8.net/html/gndy/dyzz/list_23_1.html'

HEADERS = {
   
   
    'Referer': 'http://www.ygdy8.net/html/gndy/dyzz/index.html',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36'
}


# 得到详情页面的url
def get_detail_urls(url):
    response = requests.get(url, headers=HEADERS)
    text = response.text
    html = etree.HTML(text)
    detail_urls = html.xpath('//table[@class="tbspan"]//a/@href')
    detail_urls = map(lambda url: BASE_DOMAIN + url, detail_urls)
    return detail_urls


# 得到页面详情里面的数据
def parse_detail_page(url):
    movie = {
   
   }
    respoonse = requests.get(url, headers=HEADERS)
    text = respoonse.content.decode('gbk')
    html = etree.HTML(text)
    title = html.xpath('//div[@class="title_all"]//font[@color="#07519a"]/text()')[0]
    movie['title'] = title
    zoomE = html.xpath("//div[@id='Zoom']")[0]
    cover = zoomE.xpath(".//img/@src")[0]
    movie['cover'] = cover
    infos = zoomE.xpath('.//text()')
    for index,info in enumerate(infos):
        if info.startswith('◎年　　代'):
            info = info.replace('◎年　　代', "").strip()  # strip是去前后空格
            movie['year'] = info
        elif info.startswith('◎产　　地',):
            info = info.replace('◎产　　地', "").strip()
            movie[