本文详细介绍使用Scrapy框架爬取全书网小说数据的过程，包括分类、小说信息、章节及内容的抓取，并演示如何下载图片及存储至本地。
首先我们来看看代码吧,有些重点,我会单独列出来
1: 创建项目scrapy startproject books
2: 创建spider文件 scrapy genspider book quanshuwang.com 注: 本次项目案例爬取的是全书网
3: 更换目标的完整地址 http://www.quanshuwang.com
4: 我先说一下我这次的思路,本次爬取建立了数据库,数据库中有四张表
分别是: 分类表 小说介绍表 章节表 内容表
我们来看看都是怎么实现的吧
import scrapy
from ..items import BooksItem
class BookSpider(scrapy.Spider):
name = 'book'
allowed_domains = ['quanshuwang.com']
start_urls = ['http://www.quanshuwang.com']
def __init__(self):
self.count = 0
def parse(self, response):
# 获取所有分类地址,目的所有分类下的所有小说
classify_list_title = response.xpath("//nav[@class='channel-nav']//li/a/text()").extract()[:-1]
for classify_title in classify_list_title:
classify_title = classify_title # 获取分类名称
classify_list_url = response.xpath("//nav[@class='channel-nav']//li/a/@href").extract()[:-1]
for classify_url in classify_list_url:
url = classify_url
books_id = url.split("/")[-1].split("_")[-2] # 获取分类id，目的关联每部小说对应的分类
yield scrapy.Request(url=url, callback=self.parse2, dont_filter=True, meta={'each_url': url})
第一部分主要获取,分类地址,以及分类的id 目的:一对多,一个分类下有很多小说,我们拿到这个分类主要是对应每个分类下的所有小说,这里没有使用外键
def parse2(self, response):
# 获取小说地址,目的所有小说详情信息
books_list_url = response.xpath('//ul[@class="seeWell cf"]/li/a/@href').extract()
for books_url in books_list_url:
url = books_url
yield scrapy.Request(url=url, callback=self.parse3, dont_filter=True, meta={'each_url': url})
next_page = response.xpath('//a[@class="next"]/@href').extract_first()
if next_page is not None:
next_page = response.urljoin(next_page)
yield scrapy.Request(next_page, callback=self.parse2)
第二部分,我们只获取了小说详情地址,因为我们不在这里拿小说信息
def parse3(self, response):
item = BooksItem()
# 获取自己所需要的数据
item['books_title'] = response.xpath("//div[@class='b-info']/h1/text()").extract_first() # 名称
item['books_author'] = response.xpath("//dl[@class='bookso']/dd/text()").extract_first() # 作者
item['books_status'] = response.xpath("//dl/dd/text()").extract_first() # 状态
books_introduce = response.xpath("//div[@id='waa']/text()").extract_first().split("介绍:")[-1].split(",")[0] # 介绍
item['books_introduce'] = books_introduce
front_image_path = response.xpath("//a[@class='l mr11']/img/@src").extract_first() # 图片地址
# list = []
# list.append(front_image_path)
item['front_image_path'] = [front_image_path]
item['books_classify_id'] = response.xpath("//div[@class='main-index']/a[2]/@href").extract_first().split("/")[-1].split("_")[-2] #分类与小说关联id
item['books_chapter_id'] = response.xpath("//div[@class='b-oper']/a[1]/@href").extract_first().split("/")[-1] # 小说与章节关联id
yield item
section_list_url = response.xpath("//div[@class='b-oper']/a[1]/@href").extract() # 章节列表地址
for section_url in section_list_url:
url = section_url
yield scrapy.Request(url=url, callback=self.parse4, dont_filter=True, meta={'each_url': url})
第三部分,我们主要获取小说自己所需要的内容
def parse4(self, response):
# each_url = response.meta['each_url']
# 获取每章节名称
books_section_title = response.xpath("//div[@class='clearfix dirconone']/li/a/text()").extract()
for section_title in books_section_title:
chapter_title = section_title # 获取章节
books_content_list_url = response.xpath("//div[@class='clearfix dirconone']/li/a/@href").extract()
for book_content_url in books_content_list_url:
url = book_content_url
books_content_id = url.split("/")[-1].split(".")[-2] # 获取小说章节对应内容章节
yield scrapy.Request(url=url, callback=self.parse5, dont_filter=True, meta={'each_url': url})
第四部分: 主要是获取小说章节名称,以及内容地址
def parse5(self, response):
each_url = response.meta['each_url']
books_content_id = each_url.split('/')[-1].split(".")[-2] # 内容id对应小说章节
books_content_list = response.xpath("//div[@class='mainContenr']/text()").extract()
content = ""
for books_content in books_content_list:
content += books_content
content = content
最后一部分: 主要获取小说内容
以上就是所有spider中所有代码, 最后会来个完整的
接下我们看一下settings.py配置
# -*- coding: utf-8 -*-
# Scrapy settings for books project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
# https://doc.scrapy.org/en/latest/topics/settings.html
# https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
# https://doc.scrapy.org/en/latest/topics/spider-middleware.html
import os
import sys
# sys.path.append(os.path.dirname(os.path.abspath('.')))
# os.environ['DJANGO_SETTINGS_MODULE'] = 'BooksAdmin.settings'
# import django
#
# django.setup()
BOT_NAME = 'books'
SPIDER_MODULES = ['books.spiders']
NEWSPIDER_MODULE = 'books.spiders'
# Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = 'books (+http://www.yourdomain.com)'
# Obey robots.txt rules
ROBOTSTXT_OBEY = False
# Configure maximum concurrent requests performed by Scrapy (default: 16)
CONCURRENT_REQUESTS = 32
# Configure a delay for requests for the same website (default: 0)
# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
# DOWNLOAD_DELAY = 3
# The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16
# Disable cookies (enabled by default)
#COOKIES_ENABLED = False
# Disable Telnet Console (enabled by default)
#TELNETCONSOLE_ENABLED = False
# Override the default request headers:
# DEFAULT_REQUEST_HEADERS = {
# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
# 'Accept-Language': 'en',
# 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36'
# }
# Enable or disable spider middlewares
# See https://doc.scrapy.org/en/latest/topics/spider-middleware.html
#SPIDER_MIDDLEWARES = {
# 'books.middlewares.BooksSpiderMiddleware': 543,
#}
# Enable or disable downloader middlewares
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
# DOWNLOADER_MIDDLEWARES = {
# 'books.middlewares.BooksDownloaderMiddleware': 543,
# }
# Enable or disable extensions
# See https://doc.scrapy.org/en/latest/topics/extensions.html
#EXTENSIONS = {
# 'scrapy.extensions.telnet.TelnetConsole': None,
#}
# Configure item pipelines
# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
'books.pipelines.BooksPipeline': 300,
'scrapy.pipelines.images.ImagesPipeline': 2,
'books.pipelines.ArticleImagePipeline': 1,
}
IMAGES_URLS_FIELD = "front_image_path" # image_url是在items.py中配置的网络爬取得图片地址
#配置保存本地的地址
project_dir = os.path.abspath(os.path.dirname(__file__)) #获取当前爬虫项目的绝对路径
IMAGES_STORE = os.path.join(project_dir, 'images') #组装新的图片路径
IMAGES_MIN_HEIGHT = 100 #设定下载图片的最小高度
IMAGES_MIN_WIDTH = 100 #设定下载图片的最小宽度
IMAGES_EXPIRES = 90 #90天内抓取的都不会被重抓
# Enable and configure the AutoThrottle extension (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/autothrottle.html
#AUTOTHROTTLE_ENABLED = True
# The initial download delay
#AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
#AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
#AUTOTHROTTLE_DEBUG = False
# Enable and configure HTTP caching (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
HTTPCACHE_ENABLED = True
HTTPCACHE_EXPIRATION_SECS = 0
HTTPCACHE_DIR = 'httpcache'
HTTPCACHE_IGNORE_HTTP_CODES = []
HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
都有注释
接着item.py
# -*- coding: utf-8 -*-
# Define here the models for your scraped items
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/items.html
import scrapy
class BooksItem(scrapy.Item):
# define the fields for your item here like:
# name = scrapy.Field()
books_title = scrapy.Field()
books_author = scrapy.Field()
books_status = scrapy.Field()
books_introduce = scrapy.Field()
images_url = scrapy.Field()
front_image_path = scrapy.Field()
print("*****************************///////////////////", front_image_path)
books_classify_id = scrapy.Field()
books_chapter_id = scrapy.Field()
pass
这里我写了小说信息,其他的跟这个一样
我们主要来看一下图片下载以及获取图片本地地址怎么获取
front_image_path = response.xpath("//a[@class='l mr11']/img/@src").extract_first() # 图片地址
item['front_image_path'] = [front_image_path]
这是spider获取图片的代码,有两种写法,一种是上面这种,还有一种就是下面这种
front_image_path = response.xpath("//a[@class='l mr11']/img/@src").extract_first() # 图片地址
list = []
list.append(front_image_path)
item['front_image_path'] = [list]
以上两种方法都可以,第一种比较简单
接下载我们看看settings.py怎么配置的
ITEM_PIPELINES = {
'books.pipelines.BooksPipeline': 300,
'scrapy.pipelines.images.ImagesPipeline': 2,
}
IMAGES_URLS_FIELD = "front_image_path" # front_image_path是在items.py中配置的网络爬取得图片地址
#配置保存本地的地址
project_dir = os.path.abspath(os.path.dirname(__file__)) #获取当前爬虫项目的绝对路径
IMAGES_STORE = os.path.join(project_dir, 'images') #组装新的图片路径
IMAGES_MIN_HEIGHT = 100 #设定下载图片的最小高度
IMAGES_MIN_WIDTH = 100 #设定下载图片的最小宽度
IMAGES_EXPIRES = 90 #90天内抓取的都不会被重抓
重点: IMAGES_URLS_FIELD = "front_image_path" # front_image_path是在items.py中配置的网络爬取得图片地址一致
#配置保存本地的地址
project_dir = os.path.abspath(os.path.dirname(__file__)) #获取当前爬虫项目的绝对路径
IMAGES_STORE = os.path.join(project_dir, 'images') #组装新的图片路径
以上配置图片下载就没问题了,scrapy crawl book运行看一下
最后一个重点: 我们获取本地图片的地址,我们要重写pipeline
pipelines.py
#既然要重写，记得提前引入
from scrapy.pipelines.images import ImagesPipeline
# 重载ImagePipeline中的item_completed方法，获取下载地址
class ArticleImagePipeline(ImagesPipeline):
def item_completed(self, results, item, info):
for ok, value in results: #通过断点可以看到图片路径存在results内
images_url = value['path'] #将路径保存在item中返回
item['images_url'] = images_url
return item
最后在items.py中会有一个
images_url = scrapy.Field()
注: 一定要在settings.py中设置
'books.pipelines.ArticleImagePipeline': 1,
不然获取不到
完整代码
spider.py
# -*- coding: utf-8 -*-
import scrapy
from ..items import BooksItem
class BookSpider(scrapy.Spider):
name = 'book'
allowed_domains = ['quanshuwang.com']
start_urls = ['http://www.quanshuwang.com']
def __init__(self):
self.count = 0
def parse(self, response):
# 获取所有分类地址,目的所有分类下的所有小说
classify_list_title = response.xpath("//nav[@class='channel-nav']//li/a/text()").extract()[:-1]
for classify_title in classify_list_title:
classify_title = classify_title # 获取分类名称
classify_list_url = response.xpath("//nav[@class='channel-nav']//li/a/@href").extract()[:-1]
for classify_url in classify_list_url:
url = classify_url
books_id = url.split("/")[-1].split("_")[-2] # 获取分类id，目的关联每部小说对应的分类
yield scrapy.Request(url=url, callback=self.parse2, dont_filter=True, meta={'each_url': url})
def parse2(self, response):
# 获取小说地址,目的所有小说详情信息
books_list_url = response.xpath('//ul[@class="seeWell cf"]/li/a/@href').extract()
for books_url in books_list_url:
url = books_url
yield scrapy.Request(url=url, callback=self.parse3, dont_filter=True, meta={'each_url': url})
next_page = response.xpath('//a[@class="next"]/@href').extract_first()
if next_page is not None:
next_page = response.urljoin(next_page)
yield scrapy.Request(next_page, callback=self.parse2)
def parse3(self, response):
item = BooksItem()
# 获取自己所需要的数据
item['books_title'] = response.xpath("//div[@class='b-info']/h1/text()").extract_first() # 名称
item['books_author'] = response.xpath("//dl[@class='bookso']/dd/text()").extract_first() # 作者
item['books_status'] = response.xpath("//dl/dd/text()").extract_first() # 状态
books_introduce = response.xpath("//div[@id='waa']/text()").extract_first().split("介绍:")[-1].split(",")[0] # 介绍
item['books_introduce'] = books_introduce
front_image_path = response.xpath("//a[@class='l mr11']/img/@src").extract_first() # 图片地址
# list = []
# list.append(front_image_path)
item['front_image_path'] = [front_image_path]
item['books_classify_id'] = response.xpath("//div[@class='main-index']/a[2]/@href").extract_first().split("/")[-1].split("_")[-2] #分类与小说关联id
item['books_chapter_id'] = response.xpath("//div[@class='b-oper']/a[1]/@href").extract_first().split("/")[-1] # 小说与章节关联id
yield item
section_list_url = response.xpath("//div[@class='b-oper']/a[1]/@href").extract() # 章节列表地址
for section_url in section_list_url:
url = section_url
yield scrapy.Request(url=url, callback=self.parse4, dont_filter=True, meta={'each_url': url})
def parse4(self, response):
# each_url = response.meta['each_url']
# 获取每章节名称
books_section_title = response.xpath("//div[@class='clearfix dirconone']/li/a/text()").extract()
for section_title in books_section_title:
chapter_title = section_title # 获取章节
books_content_list_url = response.xpath("//div[@class='clearfix dirconone']/li/a/@href").extract()
for book_content_url in books_content_list_url:
url = book_content_url
books_content_id = url.split("/")[-1].split(".")[-2] # 获取小说章节对应内容章节
yield scrapy.Request(url=url, callback=self.parse5, dont_filter=True, meta={'each_url': url})
def parse5(self, response):
each_url = response.meta['each_url']
books_content_id = each_url.split('/')[-1].split(".")[-2] # 内容id对应小说章节
books_content_list = response.xpath("//div[@class='mainContenr']/text()").extract()
content = ""
for books_content in books_content_list:
content += books_content
content = content
settings.py
# -*- coding: utf-8 -*-
# Scrapy settings for books project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
# https://doc.scrapy.org/en/latest/topics/settings.html
# https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
# https://doc.scrapy.org/en/latest/topics/spider-middleware.html
import os
import sys
# sys.path.append(os.path.dirname(os.path.abspath('.')))
# os.environ['DJANGO_SETTINGS_MODULE'] = 'BooksAdmin.settings'
# import django
#
# django.setup()
BOT_NAME = 'books'
SPIDER_MODULES = ['books.spiders']
NEWSPIDER_MODULE = 'books.spiders'
# Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = 'books (+http://www.yourdomain.com)'
# Obey robots.txt rules
ROBOTSTXT_OBEY = False
# Configure maximum concurrent requests performed by Scrapy (default: 16)
CONCURRENT_REQUESTS = 32
# Configure a delay for requests for the same website (default: 0)
# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
# DOWNLOAD_DELAY = 3
# The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16
# Disable cookies (enabled by default)
#COOKIES_ENABLED = False
# Disable Telnet Console (enabled by default)
#TELNETCONSOLE_ENABLED = False
# Override the default request headers:
# DEFAULT_REQUEST_HEADERS = {
# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
# 'Accept-Language': 'en',
# 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36'
# }
# Enable or disable spider middlewares
# See https://doc.scrapy.org/en/latest/topics/spider-middleware.html
#SPIDER_MIDDLEWARES = {
# 'books.middlewares.BooksSpiderMiddleware': 543,
#}
# Enable or disable downloader middlewares
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
# DOWNLOADER_MIDDLEWARES = {
# 'books.middlewares.BooksDownloaderMiddleware': 543,
# }
# Enable or disable extensions
# See https://doc.scrapy.org/en/latest/topics/extensions.html
#EXTENSIONS = {
# 'scrapy.extensions.telnet.TelnetConsole': None,
#}
# Configure item pipelines
# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
'books.pipelines.BooksPipeline': 300,
'scrapy.pipelines.images.ImagesPipeline': 2,
'books.pipelines.ArticleImagePipeline': 1,
}
IMAGES_URLS_FIELD = "front_image_path" # front_image_path是在items.py中配置的网络爬取得图片地址
#配置保存本地的地址
project_dir = os.path.abspath(os.path.dirname(__file__)) #获取当前爬虫项目的绝对路径
IMAGES_STORE = os.path.join(project_dir, 'images') #组装新的图片路径
IMAGES_MIN_HEIGHT = 100 #设定下载图片的最小高度
IMAGES_MIN_WIDTH = 100 #设定下载图片的最小宽度
IMAGES_EXPIRES = 90 #90天内抓取的都不会被重抓
# Enable and configure the AutoThrottle extension (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/autothrottle.html
#AUTOTHROTTLE_ENABLED = True
# The initial download delay
#AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
#AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
#AUTOTHROTTLE_DEBUG = False
# Enable and configure HTTP caching (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
HTTPCACHE_ENABLED = True
HTTPCACHE_EXPIRATION_SECS = 0
HTTPCACHE_DIR = 'httpcache'
HTTPCACHE_IGNORE_HTTP_CODES = []
HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
items.py
# -*- coding: utf-8 -*-
# Define here the models for your scraped items
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/items.html
import scrapy
class BooksItem(scrapy.Item):
# define the fields for your item here like:
# name = scrapy.Field()
books_title = scrapy.Field()
books_author = scrapy.Field()
books_status = scrapy.Field()
books_introduce = scrapy.Field()
images_url = scrapy.Field()
front_image_path = scrapy.Field()
print("*****************************///////////////////", front_image_path)
books_classify_id = scrapy.Field()
books_chapter_id = scrapy.Field()
pass
pipelines.py
# -*- coding: utf-8 -*-
# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
#既然要重写，记得提前引入
from scrapy.pipelines.images import ImagesPipeline
class BooksPipeline(object):
def process_item(self, item, spider):
# print('打开数据库')
# item.save() # 数据将会自动添加到指定的表
# print('关闭数据库')
return item
class ArticleImagePipeline(ImagesPipeline):
def item_completed(self, results, item, info):
for ok, value in results:
images_url = value['path']
item['images_url'] = images_url
return item
以上就是所有代码
给大家看一下项目结构吧
运行结果
扫一扫
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
