python爬虫实战 爬取汽车之家上车型价格

相关库

import pymysql
import pymysql.cursors
from bs4 import BeautifulSoup
import requests
import random
import time
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait 
from selenium.webdriver.support import expected_conditions as EC
import codecs
from selenium.common.exceptions import TimeoutException

从数据库中读取车型（车型已经存放再数据库，这里读取车型的id，拼接到url上）

cars = []

conn = pymysql.connect(host='*******',charset='utf8',user=*******',passwd='*****',db='mysql',cursorclass=pymysql.cursors.DictCursor)

try:
    cur = conn.cursor()
    cur.execute("USE data_etl")
    cur.execute("select distinct(car_id),car_name from user_car_port")
    item = cur.fetchone()
    count = 0
    while item is not None:
        cars.append(item)
        count+=1
        item = cur.fetchone()
    print(count)
finally:
    conn.close()

由于汽车之家反爬比较复杂，我们直接调用浏览器接口

driver = webdriver.Chrome('chromedriver.exe'