Python爬虫网安-request+示例

最新推荐文章于 2025-06-30 14:33:24 发布

Whoisshutiao

最新推荐文章于 2025-06-30 14:33:24 发布

阅读量284

点赞数 5

CC 4.0 BY-SA版权

分类专栏： python爬虫网安文章标签： python 爬虫开发语言网络安全

本文链接：https://blog.csdn.net/lemo_qi/article/details/148993620

python爬虫网安专栏收录该内容

7 篇文章

订阅专栏

ssl证书校验超时身份认证（httpbasicAuth）

代理配置

get&post

#！/usr/bin/env python

import requests

#get
# r = requests.get('http://httpbin.org/get')
# print(r.text)

# 添加参数的get请求
data = {
    'name':'kenan',
    'age':'16'
}
r = requests.get('http://www.baidu.com/get',params=data)
print(r.text)

#！/usr/bin/env python

import requests

# post
data = {
     'name':'kenan',
     'age':'16'
}
r = requests.post('http://httpbin.org/post',data=data)
print(r.text)

自定义请求头

#！/usr/bin/env python

import requests

headers = {
    'Usrt-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36'
}
r = requests.get('https://www.zhihu.com',headers=headers)
print(r.text)

文件上传

import requests

url = 'http://httpbin.org/post'
files = {'file':open('a.txt','rb')}
r = requests.post(url,files=files)
print(r.text)

添加cookie获取网页

import requests

url = 'https://www.zhihu.com'
# r = requests.get(url=url)
# print(r.cookies)
# for key,value in r.cookies.items():
#     print(key + '=' + value)

headers = {
    'Cookie':'_zap=65985310-a068-4c12-ae0a-56251c2c5c12; d_c0=AbCTugmbSBePTrEOSOMV93HuhQPipyoMkyM=|1692789471; YD00517437729195%3AWM_TID=emVDLGILJsBERAVQVAeAkpebh4G4Aa0t; __snaker__id=zptBm6fqrykPGJWm; YD00517437729195%3AWM_NI=SQ6wtpN50b7JklREIGLJWfA7KXvU0ElY78UhdWVg5RLCcXFSmIutvYWSTxuU4CpLkUPWI2Y4mwYdcmBA086%2BVej4kBzqutG6WzywEoor7JT9in6Xt7l1sOIXt%2ByjYphdTVg%3D; YD00517437729195%3AWM_NIKE=9ca17ae2e6ffcda170e2e6ee83aa66a1ecfd96eb4b8ea88aa7c44e838b8a87c56297abf990f15cf68e9887d02af0fea7c3b92a918ffa98f77c8eeb0083b642a9b49ea6cc4d90969987c43d8988bab6b34fb087f9d9d864a8899a88b85a909dadb9c559af97bb95f553958d9c99f85ff89dc0adea5f948987d3dc40afa99785e267829f00d9b52196bba6d5ea4abaafbb95c63e95a6b798c865adaef9abd733abb2fab9f73db0efbdb0cc54b8f5b78bdc7cacbc9cb7e637e2a3; _xsrf=jWERx0Ik6tjUozZKGWIpPBRGFOJ6QYhq; l_cap_id="NWI2N2QwMDM0MjUzNDU4OWExMmE3NmI0Y2I5ZTc3MmQ=|1693290667|022d1fb6013e570378e4fa421ac1e25e984e3165"; r_cap_id="YjY0YWI2YjcyM2QxNGI1Mjg5MTM5ZmRjMGEwYzEwYTg=|1693290667|1fcd27c525bc660ecf654c711ac76b67d452689c"; cap_id="YjU5YjE5MzFjN2ZlNDhlZDhlYWQ0NTE5OTRkZTRjZjU=|1693290667|a118f84fa5d20957be92227edacf0e66d30eeec0"; captcha_session_v2=2|1:0|10:1693292926|18:captcha_session_v2|88:TWFFU1NCVHlldDVJaTR5cTlwM2h4SmhGeTJTZ010WFV5Rm8xVzE3WWpOUjFyL0xkcVI3TjZGeHNxY3FGVmxCVQ==|2ce870f0a811b7f7d08e168a66921c06fe61a12bda683b178a4b602e4752f7a0; gdxidpyhxdE=QGggrpC%2BxZPIYJYBt8R4ujh6KD%5CTpC9W20pZ6YAcDsSM81tdUWJuNon2TBMMS3NC%2BjjbAMgsmIoioR%2BnaJbPnN3TJma2%2B%5C%5CVBm7gS4%5CsviBdQK6%2BolIZbAtHwcEfBz0VRYv6eQ%2FjmU4SrTrwXqesKfBaqvIRyDvr69M1gSgHinN%2FUV1s%3A1693293827600; q_c1=c4dd7565996f498d8689e31708ec9ef1|1693292955000|1693292955000; z_c0=2|1:0|10:1693292957|4:z_c0|92:Mi4xVWdrZUJnQUFBQUFCc0pPNkNadElGeVlBQUFCZ0FsVk5tdWZhWlFEa0pEVnZ5MGRVbmNuQzhDM3ljT0VyVnFUUUtB|924bb27a52a11d16167abbc8c01824e09e82e2df7f3724ceea2b675a569137ad; Hm_lvt_98beee57fd2ef70ccdd5ca52b9740c49=1693288624,1693290816,1693292927,1693315711; SESSIONID=7NnIui3CdpkQ2CVVCuillDLNphd1UQs3BxB7AVZiwUy; JOID=VlwQC0nBLspKFqRRHM0YnbfzUXEK9mGpCWXNY1KjXqAWLuEHS0DQICIQpV0bHMfWHsi4i_WxPEPuNtHxDESP0qY=; osd=UF0dC0LHL8dKHaJQEc0Tm7b-UXoM92ypAmPMblKoWKEbLuoBSk3QKyQRqF0QGsbbHsO-ivixN0XvO9H6CkWC0q0=; Hm_lpvt_98beee57fd2ef70ccdd5ca52b9740c49=1693316313; tst=r; KLBRSID=f48cb29c5180c5b0d91ded2e70103232|1693316989|1693315625',
    'Usrt-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36'
}
r = requests.get(url=url,headers=headers)
with open('1.html','wb') as f:
    f.write(r.text.encode('utf-8'))

使用cookiejar

import requests.cookies 


Cookie='_zap=65985310-a068-4c12-ae0a-56251c2c5c12; d_c0=AbCTugmbSBePTrEOSOMV93HuhQPipyoMkyM=|1692789471; YD00517437729195%3AWM_TID=emVDLGILJsBERAVQVAeAkpebh4G4Aa0t; __snaker__id=zptBm6fqrykPGJWm; YD00517437729195%3AWM_NI=SQ6wtpN50b7JklREIGLJWfA7KXvU0ElY78UhdWVg5RLCcXFSmIutvYWSTxuU4CpLkUPWI2Y4mwYdcmBA086%2BVej4kBzqutG6WzywEoor7JT9in6Xt7l1sOIXt%2ByjYphdTVg%3D; YD00517437729195%3AWM_NIKE=9ca17ae2e6ffcda170e2e6ee83aa66a1ecfd96eb4b8ea88aa7c44e838b8a87c56297abf990f15cf68e9887d02af0fea7c3b92a918ffa98f77c8eeb0083b642a9b49ea6cc4d90969987c43d8988bab6b34fb087f9d9d864a8899a88b85a909dadb9c559af97bb95f553958d9c99f85ff89dc0adea5f948987d3dc40afa99785e267829f00d9b52196bba6d5ea4abaafbb95c63e95a6b798c865adaef9abd733abb2fab9f73db0efbdb0cc54b8f5b78bdc7cacbc9cb7e637e2a3; _xsrf=jWERx0Ik6tjUozZKGWIpPBRGFOJ6QYhq; l_cap_id="NWI2N2QwMDM0MjUzNDU4OWExMmE3NmI0Y2I5ZTc3MmQ=|1693290667|022d1fb6013e570378e4fa421ac1e25e984e3165"; r_cap_id="YjY0YWI2YjcyM2QxNGI1Mjg5MTM5ZmRjMGEwYzEwYTg=|1693290667|1fcd27c525bc660ecf654c711ac76b67d452689c"; cap_id="YjU5YjE5MzFjN2ZlNDhlZDhlYWQ0NTE5OTRkZTRjZjU=|1693290667|a118f84fa5d20957be92227edacf0e66d30eeec0"; captcha_session_v2=2|1:0|10:1693292926|18:captcha_session_v2|88:TWFFU1NCVHlldDVJaTR5cTlwM2h4SmhGeTJTZ010WFV5Rm8xVzE3WWpOUjFyL0xkcVI3TjZGeHNxY3FGVmxCVQ==|2ce870f0a811b7f7d08e168a66921c06fe61a12bda683b178a4b602e4752f7a0; gdxidpyhxdE=QGggrpC%2BxZPIYJYBt8R4ujh6KD%5CTpC9W20pZ6YAcDsSM81tdUWJuNon2TBMMS3NC%2BjjbAMgsmIoioR%2BnaJbPnN3TJma2%2B%5C%5CVBm7gS4%5CsviBdQK6%2BolIZbAtHwcEfBz0VRYv6eQ%2FjmU4SrTrwXqesKfBaqvIRyDvr69M1gSgHinN%2FUV1s%3A1693293827600; q_c1=c4dd7565996f498d8689e31708ec9ef1|1693292955000|1693292955000; z_c0=2|1:0|10:1693292957|4:z_c0|92:Mi4xVWdrZUJnQUFBQUFCc0pPNkNadElGeVlBQUFCZ0FsVk5tdWZhWlFEa0pEVnZ5MGRVbmNuQzhDM3ljT0VyVnFUUUtB|924bb27a52a11d16167abbc8c01824e09e82e2df7f3724ceea2b675a569137ad; Hm_lvt_98beee57fd2ef70ccdd5ca52b9740c49=1693288624,1693290816,1693292927,1693315711; SESSIONID=7NnIui3CdpkQ2CVVCuillDLNphd1UQs3BxB7AVZiwUy; JOID=VlwQC0nBLspKFqRRHM0YnbfzUXEK9mGpCWXNY1KjXqAWLuEHS0DQICIQpV0bHMfWHsi4i_WxPEPuNtHxDESP0qY=; osd=UF0dC0LHL8dKHaJQEc0Tm7b-UXoM92ypAmPMblKoWKEbLuoBSk3QKyQRqF0QGsbbHsO-ivixN0XvO9H6CkWC0q0=; Hm_lpvt_98beee57fd2ef70ccdd5ca52b9740c49=1693316313; tst=r; KLBRSID=f48cb29c5180c5b0d91ded2e70103232|1693316989|1693315625'
#创建一个空的RequestsCookieJar来保存cookies
jar = requests.cookies.RequestsCookieJar()
headers = {
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36'
}
#遍历分割cookies字符串，将每一个cookie的建值对添加到 cookiejar中
for cookie in Cookie.split(';'):
    key,value = cookie.split('=',1)
    # print(key)
    jar.set(key,value)
  
#发送请求并且附带cookies和header    
r = requests.get('http://www.zhihu.com',cookies=jar,headers=headers)
with open('2.html','wb') as f:
    f.write(r.text.encode('utf-8'))

session

import requests

# r1 = requests.get('http://httpbin.org/cookies/set/number/123456')
# print(r1.text)
# r2 = requests.get('http://httpbin.org/cookies')
# print(r2.text)

#1.创建一个session
s = requests.Session()

#2.向网站发送get并且设置cookie
s.get('http://httpbin.org/cookies/set/number/123456')
#3.再次发送请求 获取当前会话里的cookie
r = s.get('http://httpbin.org/cookies')
print(r.text)

ssl证书校验
超时
身份认证（httpbasicAuth）

import requests
from requests.auth import HTTPBasicAuth

# r = requests.get('https://ssr2.scrape.center/')
# print(r.status_code)

# r = requests.get('https://ssr2.scrape.center/',verify=False,timeout=0.1)
# print(r.status_code)

r = requests.get('https://ssr3.scrape.center/',verify=False,auth=HTTPBasicAuth('admin','admin'))
print(r.text)

代理配置

import requests

# HTTP/HTTPS 代理配置
proxy_list = {
    'http': 'http://代理地址:端口',
    'https': 'http://代理地址:端口'
}

# SOCKS 代理配置示例（需安装依赖：pip install requests[socks]）
socks_proxy = {
    'http': 'socks5://username:password@host:port',
    'https': 'socks5://username:password@host:port'
}

try:
    # 使用 HTTP 代理发送请求
    response = requests.get('https://www.baidu.com', proxies=proxy_list, timeout=10)
    print(response.status_code)

    # 使用 SOCKS 代理发送请求
    response_socks = requests.get('https://www.baidu.com', proxies=socks_proxy, timeout=10)
    print(response_socks.status_code)

except requests.exceptions.RequestException as e:
    print(f"请求失败: {e}")