requests运用----爬取大主宰小说

这是小说网址http://www.biququ.info/html/3486/43571.html
检查元素–抓取文本内容在Elements的具体位置

可以看到都是在

== $0下，那就意味着我们只要抓取该标签下所有的p标签中的文本就行

etree.HTML():构造了一个XPath解析对象并对HTML文本进行自动修正。
etree.tostring()：输出修正后的结果，类型是bytes
我们从网页请求到的数据并不一定能直接使用xpath定位，所以需要使用etree.HTML()转换成能用xpath的格式
同样我们可以抓取到屏幕上下一站所在的标签，可以看到其中有下一章的地址，我们就可以将它爬取作为下一次的url地址

import requests
import time
from lxml import etree

class DazhuzaiSpider:
    def __init__(self):
        self.header={"ser-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36"}
        self.tmep_url="http://www.biququ.info/html/3486/43571.html"

    def parse(self,url):
        response = requests.get(url,headers=self.header)
        content = response.content.decode()
        return content

    def get_content(self,html_str):
        html = etree.HTML(html_str)
        content = html.xpath("//div[@id='content']/p")
        next_url = html.xpath("//div[@class='bottem1']//a[@id='pager_next']/@href")
        next_url ="http://www.biququ.info"+"".join(next_url)
        #print(next_url)
        #print(content)
        content_list = []
        for i in content:
            p_content = i.xpath("./text()")
            # p_content = [i.replace("['","") for i in p_content]
            # p_content = [i.replace("']", "") for i in p_content]
            content_list.append(p_content)
        #print(content_list)
        return content_list,next_url

    def save_content(self,content):
        with open("大主宰.txt","a") as f:
            for i in content:
                #print(i)
                #print(type(i))
                #print("".join(i))
                str = "".join(i)
                # print(str)
                # str =str+'\n'
                str = str.strip("/n")
                f.write(str)
                f.write("\n")
        f.close()

    def run(self):
        start = time.clock()
        print(start)
        next_url = self.tmep_url
        while True:
            #获取start_url
            #发送请求，获得数据
            html_str = self.parse(next_url)
            #print(html_str)
            #提取数据
            content,next_url= self.get_content(html_str)
            #print(content)
            if next_url=="http://www.biququ.info/html/3486/":
                break
            #获得下一个url地址
            self.save_content(content)
            #发送请求
        end = time.clock()
        print(end-start)

if __name__ == "__main__":
    dzz = DazhuzaiSpider()
    dzz.run()