# CY3761 | 2021-11-04 18:23
import json
import os
import queue
import time
import urllibimport requestsfrom urllib import parse
import threading
from queue import Queue# 王者荣耀-官网 https://pvp.qq.com
# 王者荣耀-高清壁纸 https://pvp.qq.com/web201605/wallpaper.shtml | 游戏资料-游戏壁纸# 多线程下载王者荣耀-高清壁纸 使用 queue
# 生产者线程 page_queue.get img_url_queue.put
# 请求json数据获得图片链接, 并对图片链接进行处理成可下载的图片链接
# 消费者线程 img_url_queue.get
# 进行读取图片链接,并进行下载图片# -------------------------------------------------------------
# 创建文件夹
def makeDirPlus(_dirPath):if not os.path.exists(_dirPath):print('创建目录: %s' % _dirPath)os.makedirs(_dirPath)# -------------------------------------------------------------
# 请求数据
def request(_url, _encoding=None):print(f'请求数据: {_url}')resp = requests.get(_url, headers=headers)resp.encoding = _encoding if _encoding else encodingif _url.endswith(imgRepNew):  # 属于图片return resp.content  # 图片返回二进制else:return resp.text  # 返回字符串文本# -------------------------------------------------------------# 保存文件
def saveData(_data, saveFilePath):# 不存在才进行保存if not os.path.exists(saveFilePath):# print(type(_data))if isinstance(_data, bytes):  # 保存图片w = open(saveFilePath, 'wb')else:w = open(saveFilePath, 'w', encoding=encoding)w.write(_data)w.close()print(f'写入数据: {saveFilePath}')# -------------------------------------------------------------
# 读取数据
def getData(saveFilePath):# 存在才进行读取if os.path.exists(saveFilePath):r = open(saveFilePath, 'r', encoding=encoding)# 读取 json 数据if saveFilePath.endswith('json'):_data = json.load(r)else:_data = r.read()r.close()print(f'读取数据: {saveFilePath}')return _data# -------------------------------------------------------------
# 获取请求数据
def get_request(requestUrl, saveFilePath='', requestEncoding=None):if not os.path.exists(saveFilePath):saveData(request(requestUrl, requestEncoding), saveFilePath)# 图片不需要读取if saveFilePath.endswith('jpg'):returnreturn getData(saveFilePath)# -------------------------------------------------------------
# 获取json数据
def get_json(_page):global cQscQs['page'] = _pagedEncode = urllib.parse.urlencode(cQs)dUrl = '?'.join([bUrl, dEncode])# print(dUrl)  # 这才是请求的地址jsonFilePath = f'{jsonDirPath}%d.json'return get_request(dUrl,jsonFilePath % _page)# -------------------------------------------------------------
# 处理图片链接
def get_image_urls(_datas):# 图片数据dItems = _datas.get('List')# 1 ~ 8dImgItems = []for k, _ in enumerate(dItems):dImgItems.append([])  # 初始化图片列表# 图片名字 sProdNamesProdName = _.get('sProdName')sProdName = urllib.parse.unquote(sProdName)sProdName = sProdName.strip()  # 清除空格# 图片id iProdId_iProdId = _.get('iProdId')for i in range(1, imgSize + 1):dImgItem = _.get(f'sProdImgNo_{i}')dImgItem = urllib.parse.unquote(dImgItem)dImgItem = dImgItem.replace(imgRepOld, imgRepNew)if dImgItem:dImgItems[k].append((sProdName, _iProdId, i, dImgItem))return dImgItems# -------------------------------------------------------------# 生产者线程
class P(threading.Thread):def __init__(self, j_queue, i_queue):super(P, self).__init__()self.j_queue = j_queue  # 页面数self.i_queue = i_queue  # 图片数# 进行数据请求获取JSON数据, 从JSON数据获取图片链接并处理成可下载的链接def run(self) -> None:global imgItems# 队列不为空 执行以下代码段while not self.j_queue.empty():# 获取取URL并发送请求 / 获取JSON数据# 获取JSON数据后, 从JSON数据获取图片链接并处理成可下载的链接# sProdName, _iProdId, i, dImgItemdatas = get_image_urls(get_json(self.j_queue.get()))for k1, v1 in enumerate(datas):k1 = str(k1)for k2, v2 in enumerate(v1):fileName = v2[0]iProdId = v2[1].zfill(5)i = v2[2]fileSrc = v2[3]# 图片路径imgItemDirPath = f'{imgDirPath}%s/' % iProdIdmakeDirPlus(imgItemDirPath)# 创建图片名字文件saveData('', f'{imgItemDirPath}{fileName}.txt')self.i_queue.put((fileSrc, f'{imgItemDirPath}%s.jpg' % i))imgItems.setdefault(iProdId, fileName)pass# for in# 创建文件夹与对应各图片名字文件 | 一页的数据 20个# 生成图片路径与本地路径数据 并且进行# 最后需要 这里进行图片名字组装并写入一个文件pass# -------------------------------------------------------------
# 消费者线程
class C(threading.Thread):def __init__(self, j_queue, i_queue):super(C, self).__init__()self.j_queue = j_queue  # 页面数self.i_queue = i_queue  # 图片数# 获取图片链接并下载图片def run(self) -> None:while True:try:get = self.i_queue.get(timeout=32)  # 获取图片并下载 | 超时结束get_request(get[0], get[1])except queue.Empty as e:  # 没有数据就停止循环breakpass# -------------------------------------------------------------
# 全局数据
dirPath = '10.王者荣耀/'
jsonDirPath = dirPath + 'json/'
imgDirPath = dirPath + 'img/'imgRepOld = '/200'
imgRepNew = '/0'
imgSize = 8
imgItems = {}makeDirItems = [jsonDirPath,imgDirPath
]headers = {'referer': 'https://pvp.qq.com/','user-agent': ('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) ''Chrome/95.0.4638.69 Safari/537.36')
}encoding = 'utf-8'# -------------------------------------------------------------
# 数据链接分析与解析
aUrl = ('https://apps.game.qq.com/cgi-bin/ams/module/ishow/V1.0/query/workList_inc.cgi?activityId=2735&sVerifyCode=ABCD''&sDataType=JSON&iListNum=20&totalpage=0&page=1&iOrder=0&iSortNumClose=1&jsoncallback''=jQuery1710035807958553548014_1635991723290&iAMSActivityId=51991&_everyRead=true&iTypeId=2&iFlowId=267733''&iActId=2735&iModuleId=2735&_=1635992383307')
aSplit = aUrl.split('?')
bUrl, bQuery = aSplit[0], aSplit[1]  # 将链接与请求数据进行拆分# print(bUrl, bQuery, '?'.join([bUrl, bQuery]), sep='\n')cQs = urllib.parse.parse_qsl(bQuery)
cQs = {_[0]: _[1] for _ in cQs}
del cQs['jsoncallback']  # 删除请求响应前的字符串# 改变请求页码 | 默认 0 | 第一页# -------------------------------------------------------------
def main():s = time.time()for _ in makeDirItems:makeDirPlus(_)# 获取总页面 总图片数data = get_json(0)totalPages = int(data.get('iTotalPages'))totalLines = int(data.get('iTotalLines'))# 设置2个队列j_queue = Queue(totalPages)  # 页面数i_queue = Queue(totalLines * imgSize)  # 图片数for page in range(0, totalPages):# 传入链接页面 pagej_queue.put(page)pass# 创建生产者线程对象 同时进行ts = []for _ in range(5):t = P(j_queue, i_queue)  # 存入2个队列对象ts.append(t)t.start()# 创建消费者线程对象 同时进行for _ in range(10):t = C(j_queue, i_queue)ts.append(t)t.start()for _ in ts:_.join()with open(f'{imgDirPath}目录对应名称.txt', 'w', encoding=encoding) as w:for _ in imgItems:k = _v = imgItems[_]w.write('|'.join([k, v]) + '\n')w.write(f'共{len(imgItems)}个目录\n')e = time.time()print('耗时 %.8f' % (e - s))# -------------------------------------------------------------
if __name__ == '__main__':main()