301 lines
13 KiB
Python
301 lines
13 KiB
Python
# -*- coding: utf-8 -*-
|
|
# by @嗷呜
|
|
import json
|
|
import sys
|
|
import time
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
from urllib.parse import quote
|
|
from Crypto.Hash import MD5
|
|
import requests
|
|
sys.path.append('..')
|
|
from base.spider import Spider
|
|
|
|
|
|
class Spider(Spider):
|
|
|
|
def init(self, extend=""):
|
|
self.session = requests.Session()
|
|
self.session.headers.update(self.headers)
|
|
self.session.cookies.update(self.cookie)
|
|
self.get_ctoken()
|
|
pass
|
|
|
|
def getName(self):
|
|
pass
|
|
|
|
def isVideoFormat(self, url):
|
|
pass
|
|
|
|
def manualVideoCheck(self):
|
|
pass
|
|
|
|
def destroy(self):
|
|
pass
|
|
|
|
host='https://www.youku.com'
|
|
|
|
shost='https://search.youku.com'
|
|
|
|
h5host='https://acs.youku.com'
|
|
|
|
ihost='https://v.youku.com'
|
|
|
|
headers = {
|
|
'User-Agent': 'Mozilla/5.0 (; Windows 10.0.26100.3194_64 ) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.82 Electron/14.2.0 Safari/537.36 Node/14.17.0 YoukuDesktop/9.2.60 UOSYouku (2.0.1)-Electron(UTDID ZYmGMAAAACkDAMU8hbiMmYdd;CHANNEL official;ZREAL 0;BTYPE TM2013;BRAND TIMI;BUILDVER 9.2.60.1001)',
|
|
'Referer': f'{host}/'
|
|
}
|
|
|
|
cookie={
|
|
"__ysuid": "17416134165380iB",
|
|
"__aysid": "1741613416541WbD",
|
|
"xlly_s": "1",
|
|
"isI18n": "false",
|
|
"cna": "bNdVIKmmsHgCAXW9W6yrQ1/s",
|
|
"__ayft": "1741672162330",
|
|
"__arpvid": "1741672162331FBKgrn-1741672162342",
|
|
"__ayscnt": "1",
|
|
"__aypstp": "1",
|
|
"__ayspstp": "3",
|
|
"tfstk": "gZbiib4JpG-6DqW-B98_2rwPuFrd1fTXQt3vHEp4YpJIBA3OgrWcwOi90RTOo9XVQ5tAM5NcK_CP6Ep97K2ce1XDc59v3KXAgGFLyzC11ET2n8U8yoyib67M3xL25e8gS8pbyzC1_ET4e8URWTsSnHv2uh8VTeJBgEuN3d-ELQAWuKWV36PHGpJ2uEWVTxvicLX1ewyUXYSekxMf-CxMEqpnoqVvshvP_pABOwvXjL5wKqeulm52np_zpkfCDGW9Ot4uKFIRwZtP7vP9_gfAr3KEpDWXSIfWRay-DHIc_Z-hAzkD1i5Ooi5LZ0O5YO_1mUc476YMI3R6xzucUnRlNe_zemKdm172xMwr2L7CTgIkbvndhFAVh3_YFV9Ng__52U4SQKIdZZjc4diE4EUxlFrfKmiXbBOHeP72v7sAahuTtWm78hRB1yV3tmg9bBOEhWVnq5KwOBL5."
|
|
}
|
|
|
|
def homeContent(self, filter):
|
|
result = {}
|
|
categories = ["电视剧", "电影", "综艺", "动漫", "少儿", "纪录片", "文化", "亲子", "教育", "搞笑", "生活",
|
|
"体育", "音乐", "游戏"]
|
|
classes = [{'type_name': category, 'type_id': category} for category in categories]
|
|
filters = {}
|
|
self.typeid = {}
|
|
with ThreadPoolExecutor(max_workers=len(categories)) as executor:
|
|
tasks = {
|
|
executor.submit(self.cf, {'type': category}, True): category
|
|
for category in categories
|
|
}
|
|
|
|
for future in as_completed(tasks):
|
|
try:
|
|
category = tasks[future]
|
|
session, ft = future.result()
|
|
filters[category] = ft
|
|
self.typeid[category] = session
|
|
except Exception as e:
|
|
print(f"处理分类 {tasks[future]} 时出错: {str(e)}")
|
|
|
|
result['class'] = classes
|
|
result['filters'] = filters
|
|
return result
|
|
|
|
def homeVideoContent(self):
|
|
try:
|
|
vlist = []
|
|
params={"ms_codes":"2019061000","params":"{\"debug\":0,\"gray\":0,\"pageNo\":1,\"utdid\":\"ZYmGMAAAACkDAMU8hbiMmYdd\",\"userId\":\"\",\"bizKey\":\"YOUKU_WEB\",\"appPackageKey\":\"com.youku.YouKu\",\"showNodeList\":0,\"reqSubNode\":0,\"nodeKey\":\"WEBHOME\",\"bizContext\":\"{\\\"spmA\\\":\\\"a2hja\\\"}\"}","system_info":"{\"device\":\"pcweb\",\"os\":\"pcweb\",\"ver\":\"1.0.0.0\",\"userAgent\":\"Mozilla/5.0 (; Windows 10.0.26100.3194_64 ) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.82 Electron/14.2.0 Safari/537.36 Node/14.17.0 YoukuDesktop/9.2.60 UOSYouku (2.0.1)-Electron(UTDID ZYmGMAAAACkDAMU8hbiMmYdd;CHANNEL official;ZREAL 0;BTYPE TM2013;BRAND TIMI;BUILDVER 9.2.60.1001)\",\"guid\":\"1590141704165YXe\",\"appPackageKey\":\"com.youku.pcweb\",\"young\":0,\"brand\":\"\",\"network\":\"\",\"ouid\":\"\",\"idfa\":\"\",\"scale\":\"\",\"operator\":\"\",\"resolution\":\"\",\"pid\":\"\",\"childGender\":0,\"zx\":0}"}
|
|
data=self.getdata(f'{self.h5host}/h5/mtop.youku.columbus.home.query/1.0/',params)
|
|
okey=list(data['data'].keys())[0]
|
|
for i in data['data'][okey]['data']['nodes'][0]['nodes'][-1]['nodes'][0]['nodes']:
|
|
if i.get('nodes') and i['nodes'][0].get('data'):
|
|
i=i['nodes'][0]['data']
|
|
if i.get('assignId'):
|
|
vlist.append({
|
|
'vod_id': i['assignId'],
|
|
'vod_name': i.get('title'),
|
|
'vod_pic': i.get('vImg') or i.get('img'),
|
|
'vod_year': i.get('mark',{}).get('data',{}).get('text'),
|
|
'vod_remarks': i.get('summary')
|
|
})
|
|
return {'list': vlist}
|
|
except Exception as e:
|
|
print(f"处理主页视频数据时出错: {str(e)}")
|
|
return {'list': []}
|
|
|
|
def categoryContent(self, tid, pg, filter, extend):
|
|
result = {}
|
|
vlist = []
|
|
result['page'] = pg
|
|
result['limit'] = 90
|
|
result['total'] = 999999
|
|
pagecount = 9999
|
|
params = {'type': tid}
|
|
id = self.typeid[tid]
|
|
params.update(extend)
|
|
if pg == '1':
|
|
id=self.cf(params)
|
|
data=self.session.get(f'{self.host}/category/data?session={id}¶ms={quote(json.dumps(params))}&pageNo={pg}').json()
|
|
try:
|
|
data=data['data']['filterData']
|
|
for i in data['listData']:
|
|
if i.get('videoLink') and 's=' in i['videoLink']:
|
|
vlist.append({
|
|
'vod_id': i.get('videoLink').split('s=')[-1],
|
|
'vod_name': i.get('title'),
|
|
'vod_pic': i.get('img'),
|
|
'vod_year': i.get('rightTagText'),
|
|
'vod_remarks': i.get('summary')
|
|
})
|
|
self.typeid[tid]=quote(json.dumps(data['session']))
|
|
except:
|
|
pagecount=pg
|
|
result['list'] = vlist
|
|
result['pagecount'] = pagecount
|
|
return result
|
|
|
|
def detailContent(self, ids):
|
|
try:
|
|
data=self.session.get(f'{self.ihost}/v_getvideo_info/?showId={ids[0]}').json()
|
|
v=data['data']
|
|
vod = {
|
|
'type_name': v.get('showVideotype'),
|
|
'vod_year': v.get('lastUpdate'),
|
|
'vod_remarks': v.get('rc_title'),
|
|
'vod_actor': v.get('_personNameStr'),
|
|
'vod_content': v.get('showdesc'),
|
|
'vod_play_from': '优酷',
|
|
'vod_play_url': ''
|
|
}
|
|
params={"biz":"new_detail_web2","videoId":v.get('vid'),"scene":"web_page","componentVersion":"3","ip":data.get('ip'),"debug":0,"utdid":"ZYmGMAAAACkDAMU8hbiMmYdd","userId":0,"platform":"pc","nextSession":"","gray":0,"source":"pcNoPrev","showId":ids[0]}
|
|
sdata,index=self.getinfo(params)
|
|
pdata=sdata['nodes']
|
|
if index > len(pdata):
|
|
batch_size = len(pdata)
|
|
total_batches = ((index + batch_size - 1) // batch_size) - 1
|
|
ssj = json.loads(sdata['data']['session'])
|
|
with ThreadPoolExecutor(max_workers=total_batches) as executor:
|
|
futures = []
|
|
for batch in range(total_batches):
|
|
start = batch_size + 1 + (batch * batch_size)
|
|
end = start + batch_size - 1
|
|
next_session = ssj.copy()
|
|
next_session.update({
|
|
"itemStartStage": start,
|
|
"itemEndStage": min(end, index)
|
|
})
|
|
current_params = params.copy()
|
|
current_params['nextSession'] = json.dumps(next_session)
|
|
futures.append((start, executor.submit(self.getvinfo, current_params)))
|
|
futures.sort(key=lambda x: x[0])
|
|
|
|
for _, future in futures:
|
|
try:
|
|
result = future.result()
|
|
pdata.extend(result['nodes'])
|
|
except Exception as e:
|
|
print(f"Error fetching data: {str(e)}")
|
|
vod['vod_play_url'] = '#'.join([f"{i['data'].get('title')}${i['data']['action'].get('value')}" for i in pdata])
|
|
return {'list': [vod]}
|
|
except Exception as e:
|
|
print(e)
|
|
return {'list': [{'vod_play_from': '哎呀翻车啦', 'vod_play_url': f'呜呜呜${self.host}'}]}
|
|
|
|
def searchContent(self, key, quick, pg="1"):
|
|
data=self.session.get(f'{self.shost}/api/search?pg={pg}&keyword={key}').json()
|
|
vlist = []
|
|
for i in data['pageComponentList']:
|
|
if i.get('commonData') and (i['commonData'].get('showId') or i['commonData'].get('realShowId')):
|
|
i=i['commonData']
|
|
vlist.append({
|
|
'vod_id': i.get('showId') or i.get('realShowId'),
|
|
'vod_name': i['titleDTO'].get('displayName'),
|
|
'vod_pic': i['posterDTO'].get('vThumbUrl'),
|
|
'vod_year': i.get('feature'),
|
|
'vod_remarks': i.get('updateNotice')
|
|
})
|
|
return {'list': vlist, 'page': pg}
|
|
|
|
def playerContent(self, flag, id, vipFlags):
|
|
return {'jx':1,'parse': 1, 'url': f"{self.ihost}/video?vid={id}", 'header': ''}
|
|
|
|
def localProxy(self, param):
|
|
pass
|
|
|
|
def cf(self,params,b=False):
|
|
response = self.session.get(f'{self.host}/category/data?params={quote(json.dumps(params))}&optionRefresh=1&pageNo=1').json()
|
|
data=response['data']['filterData']
|
|
session=quote(json.dumps(data['session']))
|
|
if b:
|
|
return session,self.get_filter_data(data['filter']['filterData'][1:])
|
|
return session
|
|
|
|
def process_key(self, key):
|
|
if '_' not in key:
|
|
return key
|
|
parts = key.split('_')
|
|
result = parts[0]
|
|
for part in parts[1:]:
|
|
if part:
|
|
result += part[0].upper() + part[1:]
|
|
return result
|
|
|
|
def get_filter_data(self, data):
|
|
result = []
|
|
try:
|
|
for item in data:
|
|
if not item.get('subFilter'):
|
|
continue
|
|
first_sub = item['subFilter'][0]
|
|
if not first_sub.get('filterType'):
|
|
continue
|
|
filter_item = {
|
|
'key': self.process_key(first_sub['filterType']),
|
|
'name': first_sub['title'],
|
|
'value': []
|
|
}
|
|
for sub in item['subFilter']:
|
|
if 'value' in sub:
|
|
filter_item['value'].append({
|
|
'n': sub['title'],
|
|
'v': sub['value']
|
|
})
|
|
if filter_item['value']:
|
|
result.append(filter_item)
|
|
|
|
except Exception as e:
|
|
print(f"处理筛选数据时出错: {str(e)}")
|
|
|
|
return result
|
|
|
|
def get_ctoken(self):
|
|
data=self.session.get(f'{self.h5host}/h5/mtop.ykrec.recommendservice.recommend/1.0/?jsv=2.6.1&appKey=24679788')
|
|
|
|
def md5(self,t,text):
|
|
h = MD5.new()
|
|
token=self.session.cookies.get('_m_h5_tk').split('_')[0]
|
|
data=f"{token}&{t}&24679788&{text}"
|
|
h.update(data.encode('utf-8'))
|
|
return h.hexdigest()
|
|
|
|
def getdata(self, url, params, recursion_count=0, max_recursion=3):
|
|
data = json.dumps(params)
|
|
t = int(time.time() * 1000)
|
|
jsdata = {
|
|
'appKey': '24679788',
|
|
't': t,
|
|
'sign': self.md5(t, data),
|
|
'data': data
|
|
}
|
|
response = self.session.get(url, params=jsdata)
|
|
if '令牌过期' in response.text:
|
|
if recursion_count >= max_recursion:
|
|
raise Exception("达到最大递归次数，无法继续请求")
|
|
self.get_ctoken()
|
|
return self.getdata(url, params, recursion_count + 1, max_recursion)
|
|
else:
|
|
return response.json()
|
|
|
|
def getvinfo(self,params):
|
|
body = {
|
|
"ms_codes": "2019030100",
|
|
"params": json.dumps(params),
|
|
"system_info": "{\"os\":\"iku\",\"device\":\"iku\",\"ver\":\"9.2.9\",\"appPackageKey\":\"com.youku.iku\",\"appPackageId\":\"pcweb\"}"
|
|
}
|
|
data = self.getdata(f'{self.h5host}/h5/mtop.youku.columbus.gateway.new.execute/1.0/', body)
|
|
okey = list(data['data'].keys())[0]
|
|
i = data['data'][okey]['data']
|
|
return i
|
|
|
|
def getinfo(self,params):
|
|
i = self.getvinfo(params)
|
|
jdata=i['nodes'][0]['nodes'][3]
|
|
info=i['data']['extra']['episodeTotal']
|
|
if i['data']['extra']['showCategory'] in ['电影','游戏']:
|
|
jdata = i['nodes'][0]['nodes'][4]
|
|
return jdata,info
|