Python爬取链家房产信息

最新推荐文章于 2024-08-13 21:38:27 发布

battlestar

最新推荐文章于 2024-08-13 21:38:27 发布

阅读量398

点赞数

CC 4.0 BY-SA版权

分类专栏： Python学习生活娱乐

本文链接：https://blog.csdn.net/battlestar/article/details/105341498

Python学习同时被 2 个专栏收录

22 篇文章

订阅专栏

生活娱乐

8 篇文章

订阅专栏

本文介绍了一款使用Python的requests、BeautifulSoup等库进行链家房产信息爬取的程序，展示了如何抓取和解析网页数据，以及如何将数据整理并保存为CSV文件。程序还包含了数据清洗和初步的数据分析，如平均房价和标准差的计算。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

Python爬取链家房产信息

需要的库

requests
BeautifulSoup
re
pandas

程序如下：

import requests
from bs4 import BeautifulSoup
import pandas as pd
import re
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from matplotlib.font_manager import FontProperties
myfont=FontProperties(fname=r'C:\Windows\Fonts\simhei.ttf',size=1)
sns.set(font=myfont.get_name())


def removenone(mylist):

    while '' in mylist:
        mylist.remove('')
    
    return mylist

def addnone(mylist,length,cha):
    while len(mylist) < length:
        mylist.append(cha)
    return mylist

def regnum(s):
    mylist = re.findall(r'[\d+\.\d]*', s)
    mylist = removenone(mylist)
    return mylist

def lianjia(url,page_range,district):
    #Initialization
    colum_name = ['Title','Position','Tag','followInfo','VR','Info','Total price','RMB/m^2','Attention','Update day','Bed room','Living room','Area','Floors','Year','WebPage']
    data_list = []
    for page in range(page_range):
        pgurl = url+'/pg'+str(page+1)
        print (pgurl)
        header = {
            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.109 Safari/537.36'}
        page = requests.get(pgurl, headers=header)  #访问网址 获取该 html内容
        a = page.text
        
        soup = BeautifulSoup(a,"lxml")  #解析该网页内容
        for b in soup.find_all('div',class_='info clear'): #find_all 找到 div  class='info clear' 的标签
            temp = []
            for wz in b.find_all('div',class_ = ['title','positionInfo','tag','houseInfo','priceInfo','followInfo']):
                temp.append(wz.get_text())
            # price = temp[3].split('万')
            # price = temp[3].split('元')
            tag = regnum(temp[2])
            date = regnum(temp[3])
            date = addnone(date,2,'>365')
            price = regnum(temp[5])
            temp.extend(price)
            temp.extend(date)

            temp.extend(tag)
            # print (temp)
            temp = addnone(temp,15,'0') 
            for title in b.find_all('div',class_ = 'title'):
                for link in title.find_all('a'):  
                    temp.append(link.get('href'))        
            # print (temp)
            # for pos in b.find_all('div',class_ = 'positionInfo'): 
            #     temp.append(pos.get_text())
            data_list.append(temp)	    
    data = pd.DataFrame(data_list,columns=colum_name)
    data[['Total price','RMB/m^2','Area']]=data[['Total price','RMB/m^2','Area']].astype('float')
    data.to_csv(district+'.csv',encoding='utf_8_sig')
    return data

if __name__ == '__main__':
    #User defined
    district_list = ['putuo']
    for district in district_list:
        url = "https://sh.lianjia.com/ershoufang/"+district
        # url = "https://xinxiang.lianjia.com/ershoufang/"+district
        # url = "https://sh.lianjia.com/weifang"
        page_range=100
        my = lianjia(url,page_range,district)
        DG = my[my['Update day']!='>365'][['Update day','Position','Total price','RMB/m^2','Area']].groupby("Position")
        res = DG.agg([np.mean,np.std])
        name = [i.split()[0] for i in res.index] 
        plt.scatter(name,res['RMB/m^2']['mean'])
        plt.xticks(rotation=45,size =4)
        plt.savefig("xuhui.png")