from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import pandas as pd
import time
# 设置 Chrome 选项以启用 headless 模式和自定义 user-agent
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument("--headless")
chrome_options.add_argument(
f'--user-agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36"'
)
# 初始化 WebDriver
driver = webdriver.Chrome(
executable_path="D:/lab/chromedriver-win64/chromedriver.exe", options=chrome_options
)
url = "https://webs.bjidex.com/sys-bsc-home/#/bscConsole/tradingMarket"
# 访问网页
driver.get(url)
# 初始化一个列表来保存数据
data_list = []
# 爬取数据
for page in range(184):
# 构建翻页按钮的 XPath
if page
输出
PS D:\lab\bigdata24.9.9> & C:/tools/miniconda3/python.exe d:/lab/bigdata24.9.9/bjidex.com.py
DevTools listening on ws://127.0.0.1:60057/devtools/browser/66a61aa5-3598-4069-94bd-d4f10be20d96
[42892:8184:0915/232611.717:ERROR:ssl_client_socket_impl.cc(882)] handshake failed; returned -1, SSL error code 1, net_error -101
[42892:8184:0915/232611.834:ERROR:ssl_client_socket_impl.cc(882)] handshake failed; returned -1, SSL error code 1, net_error -101
[42892:8184:0915/232630.213:ERROR:ssl_client_socket_impl.cc(882)] handshake failed; returned -1, SSL error code 1, net_error -101
翻页出错或已经是最后一页: Message:
页数 ... 价格
0 1 ... 0.5 元/次
1 1 ... 0 元/次
2 1 ... 0 元/次
3 1 ... 2.5 元/次
...
37 4 ... 0.2 元/次
38 4 ... 0.1 元/次
39 4 ... 0.15 元/次
[40 rows x 7 columns]
