感觉代码很冗余 尤其是分页下载的怎么优化 协程里面不能用递归吗
刚刚看了下载会有编程错误的问题 是因为网页本身有些特殊字符识别不出来 忽略就可以了 改下这里就可以了 content = await response.text(encoding='utf-8',errors='ignore')
成品就不放了 源码运行就可以了 仅做学习研究使用 请勿滥用
[Python] 纯文本查看 复制代码import asyncio
import os.path
import time
import aiohttp
import aiofiles
import requests
from lxml import etree
from urllib.parse import urljoin
# 小说下载器，基于异步IO和aiohttp库 用于合成网址
main_url = 'https://www.23wxx.cc/'
async def download(down_url, file_name):
"""
异步下载小说章节并保存到文件。
:param down_url: 章节内容的URL
:param file_name: 保存章节内容的文件名（包含路径）
"""
headers = {
# 翻页的cookie需要从随便一个翻页获取
'Cookie': 'waf_sc=5889647726; novel_3341=1254203%7C1715791397; novel_5992=894457%7C1715795498; Hm_lvt_214c2a461550c6e33904c2abd04c890a=1715787170,1715819564; novel_5177=966743%7C1715821321; Hm_lpvt_214c2a461550c6e33904c2abd04c890a=1715821317',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36 Edg/123.0.0.0',
'Referer': down_url
}
async with aiohttp.ClientSession() as session:
try:
async with session.get(down_url, headers=headers) as response:
content = await response.text()
# 获取响应的Cookie
response_cookies = response.cookies
# 更新Cookie 否则有下一页会下载失败
cookie_str = "; ".join([f"{key}={value.value}" for key, value in response_cookies.items()])
headers['Cookie'] = cookie_str
tree = etree.HTML(content)
texts = tree.xpath('//div[@id="content"]/p//text()')[:-1]
next_characters = tree.xpath('//a[@id="A3"]//text()')[0]
if next_characters:
next_url = urljoin(main_url, tree.xpath('//a[@id="A3"]//@href')[0])
texts = ''.join(texts)
async with aiofiles.open(file_name, 'w', encoding='utf-8') as f:
await f.write(texts)
while next_characters == '下一页':
"""
如果存在下一页，则继续下载。
"""
down_url = next_url
async with aiohttp.ClientSession() as session:
async with session.get(down_url, headers=headers) as response:
content = await response.text()
# print(response.url)
tree = etree.HTML(content)
texts = tree.xpath('//div[@id="content"]/p//text()')[:-1]
next_characters = tree.xpath('//a[@id="A3"]//text()')[0]
if next_characters:
next_url = urljoin(main_url, tree.xpath('//a[@id="A3"]//@href')[0])
texts = ''.join(texts)
async with aiofiles.open(file_name, 'a', encoding='utf-8') as f:
await f.write(texts)
print(f'{file_name}下载完成')
except Exception as e:
print(f'{file_name}下载失败：{e}')
async def main(lst_info, name):
"""
主函数，用于处理小说的下载任务。
:param lst_info: 包含小说章节信息的列表
:param name: 小说名称，用于保存文件
"""
if not os.path.exists(name):
os.makedirs(name)
num = 1
tasks = []
for info in lst_info[:]:
down_url = info['chapter_url']
chapter_name = info['chapter_name']
file_name = f'{name}/{num}-{chapter_name}.txt'
file_name = file_name.replace(' ', '').strip().replace('*', '') # * 号不能用于文件名 避免报错
num += 1
tasks.append(asyncio.create_task(download(down_url, file_name)))
await asyncio.gather(*tasks)
def main_search():
while True:
url = 'https://www.23wxx.cc/search.html'
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36 Edg/123.0.0.0',
'Cookie': 'waf_sc=5889647726; Hm_lvt_214c2a461550c6e33904c2abd04c890a=1715787170; Hm_lpvt_214c2a461550c6e33904c2abd04c890a=1715787479',
'Referer': 'https://www.23wxx.cc/search/804/1.html'
}
# 用户输入小说名称，输入q退出
name = input('请输入小说名,输入q退出：')
if name == 'q':
break
else:
# 发送POST请求搜索小说
data = {
'searchtype': 'all',
'searchkey': name
}
res = requests.post(url, data=data, headers=headers)
# 更新Referer为搜索结果页面的URL
# headers['Referer'] = str(res.url)
# 解析搜索结果页面
tree = etree.HTML(res.text)
dls = tree.xpath('//div[//div[@id="sitembox"]]/dl')
if len(dls) == 0:
print('没有找到小说')
continue
else:
# 打印搜索到的小说信息
lst = []
num = 1
for dl in dls:
title = dl.xpath('./dd[1]//a//text()')[0]
info = dl.xpath('./dd[2]//text()')
info = ' '.join(info).strip()
url = dl.xpath('./dd[1]//a/@href')[0]
url = urljoin(main_url, url)
print(num, title, info)
lst_info = [title, url]
lst.append(lst_info[:])
num += 1
# 用户输入要下载的小说序号
num = input('请输入你想下载的小说序号：')
if num.isdigit() and 0
