爬虫整理

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

headers = {
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
    "Accept-Encoding": "gzip, deflate",
    "Accept-Language": "zh-CN,zh;q=0.9",
    "Cache-Control": "max-age=0",
    "Connection": "keep-alive",
    "Host": "example.com",
    "Referer": "https://example.com",
    "Upgrade-Insecure-Requests": "1",
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
                  "AppleWebKit/537.36 (KHTML, like Gecko) "
                  "Chrome/108.0.0.0 Safari/537.36"
}

1
2
3
4
5
6
7
8

import re, requests

resp = requests.get(url, headers=headers, timeout=10)
print("HTTP 状态码:", resp.status_code)

# 正则提取示例（根据实际页面结构调整）
title = re.search(r'<h1[^>]*>(.*?)</h1>', resp.text).group(1)
playinfo = re.search(r'window.__playinfo__\s*=\s*(\{.*?\});', resp.text).group(1)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

import json


# loads 方法可以理解为
# 字符串类型 转为 字典
# 注意里面的键值用的是【双引号】
json_data = '{"name": "Mike", "age": "30"}'
dict_data = json.loads(json_data)


# dumps 方法可以理解为
# 字典 转为 字符串类型
dict_data = {"name": "Mike", "age": "30"}
json_data = json.dumps(dict_data, ensure_ascii=False, indent=2)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

# 模块导入
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver import ActionChains

import pyautogui
from lxml import html


# driver 的路径
DRIVER_PATH = "D:/python/msedgedriver.exe"

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25

driver = webdriver.Edge(DRIVER_PATH)

# 最大化窗口
driver.maximize_window()
# 自定义窗口大小
driver.set_window_size(1280, 720)

# 设置等待时间
# 设置最长等待时间为 20 秒，且每 1 秒就检测一下网页是否加载完全
wait = WebDriverWait(driver, 20, 1)

# 向指定的 url 发送 get 请求
driver.get(url)
# 也可以利用 BeautifulSoup 对获取到的 html 内容进行解析
# html = driver.page_source
# soup = BeautifulSoup(html, 'html.parser')

# 查找元素（直接查找）
article_loc = driver.find_element(by=By.XPATH, value="//*[@id='tbody']/tr[1]/td[1]/a")
# 查找元素（等到元素都加载完后再查找）
article_loc = wait.until(
    EC.presence_of_element_located((By.XPATH, "//*[@id='tbody']/tr[1]/td[1]/a"))
)
# 获取文章的链接
article_link = article_loc.get_attribute(name='href')

1
2
3
4
5
6
7
8
9

# 模拟鼠标的移动和点击
ActionChains(driver).move_to_element(article_loc).perform()
article_loc.click()
# 模拟键盘的输入
ActionChains(driver).send_keys(Keys.SPACE).perform() #传入空格键

# 控制鼠标点击相应坐标
pyautogui.moveTo(1380, 650)
pyautogui.click() #控制鼠标点击

1
2
3
4
5
6

# 获取当前窗口
currentwindow = driver.current_window_handle
# 获取所有窗口，返回一个列表，里面放着一些窗口对象
all_window = driver.window_handles
# 切换窗口
driver.switch_to.window('某个窗口对象')

1
2
3
4
5

import json

json_str = json.dumps(dict_data, ensure_ascii=False, indent=2)
with open("data.json", "w", encoding="utf-8") as f:
    f.write(json_str)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18

import random, requests

USER_AGENTS = [
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/108.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) Chrome/108.0.0.0 Safari/537.36",
    # 更多...
]

PROXIES = [
    {"http": "http://user:pass@proxy1:8080", "https": "https://user:pass@proxy1:8080"},
    {"http": "http://user:pass@proxy2:8080", "https": "https://user:pass@proxy2:8080"},
]

def fetch(url):
    hdr = {"User-Agent": random.choice(USER_AGENTS)}
    prx = random.choice(PROXIES)
    resp = requests.get(url, headers=hdr, proxies=prx, timeout=10)
    return resp.text