没得事就爬一下我喜欢的海贼王上的图片
需要在d盘下建立一个imgcache文件夹
# -*- coding: utf-8 -*-
import urllib
import urllib2
import json
from bs4 import BeautifulSoup
import threadpool
import thread
class htmlpaser:
def __init__(self):
self.url='http://1.hzfans.sinaapp.com/process.php'
#POST数据到接口
def Post(self,postdata):
# headers = {
# 'User-Agent':'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6'
# }
# data = urllib.urlencode(postdata)
# req = urllib2.Request(self.url,data,headers)
# resp = urllib2.urlopen(req,None,20)
# html = resp.read()
# return html
data = urllib.urlencode(postdata)
req = urllib2.Request(url, data)
html= urllib2.urlopen(req).read()
print html
#获取html内容
def GetHtml(self,url):
headers = {
'User-Agent':'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6'
}
req = urllib2.Request(url,None,headers)
resp = urllib2.urlopen(req,None,5)
html = resp.read()
#return html.decode('utf8')
return html
def GetHtml2(self,url):
page = urllib.urlopen(url)
html = page.read()
page.close()
return html
def GetHtml3(self,url):
req_header = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
'Accept':'text/html;q=0.9,*/*;q=0.8',
'Accept-Charset':'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
'Accept-Encoding':'gzip',
'Connection':'close',
'Referer':None #注意如果依然不能抓取的话，这里可以设置抓取网站的host
}
req_timeout = 5
req = urllib2.Request(url,None,req_header)
resp = urllib2.urlopen(req,None,req_timeout)
html = resp.read()
return html
def GetList(self,html):
soup = BeautifulSoup(''.join(html))
baseitem=soup.find('ul',{'class':'list'})
slist=baseitem.select('li a')
return slist
def DownImg(self,imgurl):
path= r"d:/imgcache/"+self.gGetFileName(imgurl)
data = urllib.urlretrieve(imgurl,path)
return data
def gGetFileName(self,url):
if url==None: return None
if url=="" : return ""
arr=url.split("/")
return arr[len(arr)-1]
def mkdir(path):
import os
path=path.strip()
path=path.rstrip("\")
# 判断路径是否存在
# 存在 True
# 不存在 False
isExists=os.path.exists(path)
# 判断结果
if not isExists:
# 如果不存在则创建目录
# 创建目录操作函数
os.makedirs(path)
return True
else:
# 如果目录存在则不创建，并提示目录已存在
return False
#返回两个值
def ParseContent(self,html):
soup = BeautifulSoup(''.join(html))
baseitem=soup.find('div',{'class':'showbox'})
title=soup.find('div',{'class':'msg'}).find('div',{'class':'m_left'}).get_text()
imglist=baseitem.find_all('img')
for img in imglist:
imgurl=img.get('src')
self.DownImg(imgurl)
content=baseitem.get_text().encode('utf8')
position=content.find('热点推荐')
return title,content[0:position]
def ParseItem(self,item):
url=item.get('href')
if url==None:
return
#print url+'
'
html=obj.GetHtml2(url)
title,content=obj.ParseContent(html)
#print title+'
'
return title
def print_result(request, result):
print str(request.requestID)+":"+result
obj=htmlpaser()
pool = threadpool.ThreadPool(10)
for i in range(1,40):
url="http://op.52pk.com/shtml/op_wz/list_2594_%d.shtml"%(i)
html=obj.GetHtml2(url)
items=obj.GetList(html)
print 'add job %d
' % (i)
requests = threadpool.makeRequests(obj.ParseItem, items, print_result)
[pool.putRequest(req) for req in requests]
pool.wait()
