import scrapy
from xxx.items import WorkItem
class XXXSpider(scrapy.Spider):
name = "xxx"
allowed_domains = ["example.com"]
start_urls = ["https://example.com/xx/xx"]
def parse(self, response):
year_list = response.xpath('//ul[@class="p-accordion"]/li')
for year in year_list[:2]:
release_dates_url_of_year = year.xpath('.//div[@class="genre -s"]/a/@href').extract()
for date_url in release_dates_url_of_year:
yield scrapy.Request (
url = date_url,
callback =self.date_detail_parse
)
def date_detail_parse(self, response):
work_list = response.xpath('.//div[@class="swiper-slide c-low--6"]/div')
for work in work_list:
actress_name = work.xpath('.//a[@class="name c-main-font-hover"]/text()').extract_first()
if actress_name is not None:
item = WorkItem()
item['actress_name'] = actress_name
item['image_hover'] = work.xpath('.//img[@class="c-main-bg lazyload"]/@data-src').extract_first()
work_detail_url = work.xpath('.//a[@class="img hover"]/@href').extract_first()
if work_detail_url is not None:
yield scrapy.Request (
url = work_detail_url,
callback = self.work_detail_pares,
meta = {'workItem' : item}
)
def work_detail_pares(self, response):
item = response.meta['workItem']
pics_list = response.xpath('.//div[@class="swiper-wrapper"]/div')
pre_images = []
for pic in pics_list:
img_url = pic.xpath('./img/@data-src').extract_first()
pre_images.append(img_url)
item['pre_images'] = pre_images
item['name'] = response.xpath('.//div[@class="p-workPage l-wrap"]/h2/text()').extract_first().strip()
item['id'] = response.xpath('.//span[@class="c-tag02 c-main-bg-hover c-main-bg"]/../text()').extract_first()
item['company'] = 'xxx'
item['release_date'] = response.xpath('.//div[@class="p-workPage__table"]/div[2]//div[@class="item"]/a/text()').extract_first()
actress_detail_url = response.xpath('.//div[@class="p-workPage__table"]/div[1]//div[@class="item"]/a/@href').extract_first()
yield scrapy.Request(
url = actress_detail_url,
callback = self.actress_detail_pase,
meta = {'workItem' : item}
)
def actress_detail_pase(self, response):
item = response.meta['workItem']
item['actress_avatar'] = response.xpath('.//div[@class="swiper-slide"]/img/@data-src').extract_first()
yield item
[ol]
[/ol]
有没有大佬知道我这问题到底出在哪里？问 GPT 的各种解答方案也试了，还是不行。
