前言

使用requests库爬取宝可梦列表，使用BeautifulSoup库解析并查找对应宝可梦的属性和分类

代码

1.引入库

import requests
from bs4 import BeautifulSoup
import re
import time
import random
import sys

2.定义Pokemon类

class Pokemon(object):def __init__(self,num=0,name='',atr='',cla=''):self.num=num#编号self.name=name#名称self.atr=atr#属性self.cla=cla#分类

3.打印进度条

def Print(i):global start#起始时间global count#总数p=round(50*i/count)#进度条数目p1='█'*pp2='-'*(50-p)dur=time.perf_counter()-start#用时print("\r{}{} {}/{}".format(p1,p2,i,count),end='  ')print("{:.1%}".format(i/count),end='  ')print("用时{:.2f}s".format(dur),end='  ')print("状态:",end=' ')

4.伪造浏览器

def GetAgent():agent=['Mozilla/5.0 (Windows NT 5.1; U; en; rv:1.8.1) Gecko/20061208 Firefox/2.0.0 Opera 9.50',\'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 9.50',\'Opera/8.0 (Windows NT 5.1; U; en)',\'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0',\'Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko',\'UCWEB7.0.2.37/28/999',\'NOKIA5700/ UCWEB7.0.2.37/28/999',\'Openwave/ UCWEB7.0.2.37/28/999',\'Mozilla/4.0 (compatible; MSIE 6.0; ) Opera/UCWEB7.0.2.37/28/999']return agent

爬到一半时ip被封，代理和cookies都没有弄明白，伪造浏览器比较简单。

5.爬取网页

def GetHtml(url):try:r=requests.get(url,headers={'User-Agent':random.choice(GetAgent())},timeout=30)r.raise_for_status()return r.text#返回网页内容except:return ''

6.获取宝可梦列表

def GetList(ls,name,demo):soup=BeautifulSoup(demo,'html.parser')flag=0for tag in soup('a'):try:s=re.findall(r'.{2,5}',tag.attrs['title'])[0]t=re.findall(r'/wiki/%E[^3]%',tag.attrs['href'])[0]if s=='妙蛙种子':#起始宝可梦flag=1if re.match(r'第.世代',s):#排除非名称项continueif flag and len(t):ls.append(s)#写入列表if s==name:#末尾宝可梦breakexcept:continue

7.查找属性和分类

def Find(demo):soup=BeautifulSoup(demo,'html.parser')for tag in soup('td'):try:t=re.findall(r'bgwhite',tag.attrs['class'][2])if len(t[0]):lt=tag.string#分类breakexcept:continuetry:s=re.findall(r'LPLE .+? icon.png',demo)lr=[s[0].split(' ')[1],s[1].split(' ')[1]]if lr[0]==lr[1]:return lr[0],lt[:-1]else:return lr[0]+','+lr[1],lt[:-1]#可能存在双属性except:return '',''#当ip被封或查找失败时，产生索引异常，返回空字符串

8.爬取与写入

def main (ls,f,j=0,flag=0):global suc#成功次数pa='{0:{4:}>3}{1:{5:}^10}{2:{5:}^10}{3:{5:}^10}'#写入样式try:for i in range(j,len(ls)):pet=Pokemon(i+1,ls[i])#生成序号url="https://wiki.52poke.com/wiki/"+ls[i]pet.atr,pet.cla=Find(GetHtml(url))#获取属性和分类if pet.atr=='' and pet.cla=='':raise IndexError#为空时抛出异常f.write('\n#'+pa.format(pet.num,pet.name,pet.atr,pet.cla,'0',chr(12288)))time.sleep(random.random()*3)flag=0#重置尝试次数Print(i+1)suc+=1print('写入成功,已写入{}项'.format(suc),end='          ')except:  if flag<3:#最大尝试次数              flag+=1Print(i+1)print('查找失败,第{}次重连'.format(flag),end='          ')time.sleep(5+random.random()*5)#暂停5~10smain(ls,f,i,flag)else:Print(i+1)flag=0lost.append(i+1)#记录未找到的序号print('重连失败,查找下一项',end='          ')main(ls,f,i+1,flag)

9.主程序

start=time.perf_counter()
key=eval(input('输入世代数(1-8)'))
ls=[]
lost=[]
suc=0
url="https://wiki.52poke.com/wiki/宝可梦列表%EF%BC%88按全国图鉴编号%EF%BC%89/简单版"
d1={1:'梦幻',2:'时拉比',3:'代欧奇希斯',4:'阿尔宙斯',\5:'盖诺赛克特',6:'波尔凯尼恩',7:'美录梅塔',8:'蕾冠王'}#末尾名称
d2={1:151 , 2:251 , 3:386 , 4:493 , 5:649 , 6:721 , 7:809 , 8:898}#总数
count=d2[key]
pa='{0:{4:}>3}{1:{5:}^10}{2:{5:}^10}{3:{5:}^10}'
path='C:/Users/lenovo/Desktop/py/其他/pokemon-r.txt'
with open(path,'w') as f:for flag in range(3):#提取列表，尝试三次GetList(ls,d1[key],GetHtml(url))if len(ls):f.write(pa.format('序号','名称','属性','分类',' ',chr(12288)))print('列表提取成功')flag=0breakprint('列表提取失败,第{}次重连'.format(flag+1))time.sleep(5+random.random()*5)else:     print('列表提取失败,程序退出')time.sleep(10)sys.exit()main(ls,f)
f.close()
print('程序结束,共写入{}项'.format(suc))
print('遗漏的序号')
print(lost)
time.sleep(10)