尚硅谷爬虫note006

一、ajax的get请求

1. ajax的get请求—豆瓣电影第一页

# _*_ coding : utf-8 _*_
# @Time : 2025/2/13 15:14
# @Author : 20250206-里奥
# @File : demo23_ajax的get请求
# @Project : PythonProject10-14import urllib.requestfrom demo17_qingqiuduixaingdedingzhi import headers, request, response# get请求
# 获取第一页数据，并保存# url
url = "https://movie.douban.com/j/chart/top_list?type=4&interval_id=100%3A90"# headers
headers ={"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:135.0) Gecko/20100101 Firefox/135.0"
}# 1.请求对象的定制
request = urllib.request.Request(url = url,  headers= headers)# 2.获取响应数据
response = urllib.request.urlopen(request)content = response.read().decode('utf-8')# 打印数据
# print(content)#3.下载数据到本地
# 需要encoding进行编码，否则会报错
# open()方法默认使用的是gbk编码，需要进行解码保存：encoding = "utf-8"
fp = open('豆瓣.jsom','w',encoding = "utf-8")
fp.write(content)# with open('豆瓣1.json','w',encoding = 'utf-8') as fp:
# fp.write(content)

2. ajax的get请求—豆瓣电影前十页

# _*_ coding : utf-8 _*_
# @Time : 2025/2/13 15:46
# @Author : 20250206-里奥
# @File : demo24_ajax的get请求_豆瓣电影所有页.py
# @Project : PythonProject10-14
from Tools.scripts.generate_opcode_h import headerfrom demo17_qingqiuduixaingdedingzhi import request, response, content# url = ("https://movie.douban.com/j/chart/top_list?type=4&interval_id=100%3A90&action=&start=4&limit=20")import urllib.parse
import urllib.request# 下载豆瓣电影前十页数据
# 1. 请求对象定制
#2. 获取响应数据
# 3. 下载数据# 定义create_requests
def create_request(page):basic_url = "https://movie.douban.com/j/chart/top_list?type=4&interval_id=100%3A90&action=&start=4&limit=20"# 参数data = {'start':(page - 1) * 20,'limit':20}data = urllib.parse.urlencode(data)url = basic_url + data# print(url)headers = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:135.0) Gecko/20100101 Firefox/135.0"}# 请求对象定制request = urllib.request.Request(url=url, headers=headers)return request# 定义get_content
def get_content(request):response = urllib.request.urlopen(request)content = response.read().decode('utf-8')return content# 定义down_load
def down_load(page,content):# '+'号两端必须都是字符串，才能拼接.   类型转换：str(page)with open('豆瓣_' + str(page) + '.json','w',encoding='utf-8') as fp:fp.write(content)# 封装函数
# 这是1个程序的入口
if __name__ == '__main__':start_page = int(input("请输入起始页码:"))end_page = int(input("请输入结束页码:"))
# 遍历for page in range(start_page, end_page + 1):# print(page)#每一页都有自己请求对象的定制request = create_request(page)# 获取响应数据content = get_content(request)#下载down_load(page,content)#

二、Ajax的post请求

# _*_ coding : utf-8 _*_
# @Time : 2025/2/14 09:05
# @Author : 20250206-里奥
# @File : demo25_ajax的post请求_获取杭州KFC餐厅信息
# @Project : PythonProject10-14# post请求
#   第1页
# https://www.kfc.com.cn/kfccda/ashx/GetStoreList.ashx?op=cname
# cname: 杭州
# pid:
# pageIndex: 1
# pageSize: 10#   第2页
# https://www.kfc.com.cn/kfccda/ashx/GetStoreList.ashx?op=cname
# cname: 杭州
# pid:
# pageIndex: 2
# pageSize: 10# 1.导入
import urllib.requestfrom demo20_url的urlencode方法 import basic_url, request
from demo24_ajax的get请求_豆瓣电影所有页 import create_request, down_load#2. 定义basic_url
# basic_url = "https://www.kfc.com.cn/kfccda/ashx/GetStoreList.ashx?op=cname"# 3. 参数# 定义create_request
def create_request(page):basic_url = "https://www.kfc.com.cn/kfccda/ashx/GetStoreList.ashx?op=cname"#数据data = {"cname": "杭州","pid": '','pageIndex': page ,'pageSize': '10'}
# 编码#导入import urllib.parse# 编码data = urllib.parse.urlencode(data ).encode('utf-8')#headersheaders = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:135.0) Gecko/20100101 Firefox/135.0"}# 请求对象定制request = urllib.request.Request(url = basic_url,headers=headers,data=data)
#        1)返回requestreturn request# 定义
#        4）接收request
def get_content(request):#    5)使用requestresponse = urllib.request.urlopen(request)content = response.read().decode('utf-8')return content# 定义
def down_load(page,content):with open('KFC_' + str(page) + '.json','w',encoding = 'utf-8') as fp:fp.write(content)# 程序入口
if __name__ == '__main__':# 起始页start_page = int(input('请输入起始页码：'))end_page = int(input('请输入结束页码：'))# for循环遍历for page in range(start_page,end_page + 1):print(page)#请求对象定制#    2)接收requestrequest  = create_request(page)# 获取网页源码#     3)传递requestcontent = get_content(request)# 下载down_load(page,content)# 查看.json文件快捷键  ：CTRL + alt + L