selenium 优化

类别杂记

1 selenium 优化

google analytics
平均会话时长 1.3 ->20
每次会话浏览页数 1.2 ->3.3
用户数 5->10
城市1-》3

2 禁止加载图片 css

chrome\_options = webdriver.ChromeOptions()  
prefs = {"profile.managed\_default\_content\_settings.images": 2}  
chrome\_options.add\_experimental\_option("prefs", prefs)  
browser = webdriver.Chrome(chrome\_options=chrome\_options)

chrome\_options.set\_preference('permissions.default.image', 2)

或者

prefs = {  
'profile.default\_content\_setting\_values' : {  
'images' : 2  
}  
}

或者

from selenium import webdriver  
from selenium.webdriver.chrome.options import Options  
import time  
import random

options = Options()  
num=str(float(random.randint(500,600)))  
options.add\_argument("user-agent=Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/{}"  
" (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/{}".format(num,num))

## 禁止图片和css加载

prefs = {"profile.managed\_default\_content\_settings.images": 2,'permissions.default.stylesheet':2}  
options.add\_experimental\_option("prefs", prefs)

driver = webdriver.Chrome(executable\_path='D:\\chromedriver\\chromedriver.exe',chrome\_options=options)  
driver.get('https://www.ly.com/')  
time.sleep(5)  
html=driver.find\_element\_by\_xpath("//body").get\_attribute("innerHTML")  
print(html)  
可以看到，只需要在头部参数加上一个prefs就可以了。未加参数之前的效果如图1所示：  
————————————————

#加载浏览器
    def loadDriver(self):

        option = webdriver.FirefoxOptions()
        #无头模式
        option.add_argument("-headless")
        #禁止加载图片
        option.set_preference('permissions.default.image', 2)
        #禁止加载css样式表
        option.set_preference('permissions.default.stylesheet',2)
        self.firefox = webdriver.Firefox(options=option)
        #设置页面加载超时，超过这个时间就会抛出异常，执行之后的代码，不然会卡在>一直加载
        self.firefox.set_page_load_timeout(5)
        self.firefox.set_script_timeout(5)
        try:
            #超时会抛出异常，这里try一下
            self.firefox.get("https://hotels.ctrip.com/hotel/479628.html")
        except Exception as e:
            pass

oss api 28 4095

3 随机head

# -*-coding:utf-8 -*-

import random

# 返回一个随机的请求头 headers
def getheaders():
    # 各种PC端
    user_agent_list_2 = [
        # Opera
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36 OPR/26.0.1656.60",
        "Opera/8.0 (Windows NT 5.1; U; en)",
        "Mozilla/5.0 (Windows NT 5.1; U; en; rv:1.8.1) Gecko/20061208 Firefox/2.0.0 Opera 9.50",
        "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 9.50",
        # Firefox
        "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0",
        "Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10) Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10",
        # Safari
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.57.2 (KHTML, like Gecko) Version/5.1.7 Safari/534.57.2",
        # chrome
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71 Safari/537.36",
        "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11",
        "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.133 Safari/534.16",
        # 360
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36",
        "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko",
        # 淘宝浏览器
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.11 TaoBrowser/2.0 Safari/536.11",
        # 猎豹浏览器
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71 Safari/537.1 LBBROWSER",
        "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; LBBROWSER)",
        "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E; LBBROWSER)",
        # QQ浏览器
        "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; QQBrowser/7.0.3698.400)",
        "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)",
        # sogou浏览器
        "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 SE 2.X MetaSr 1.0",
        "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SV1; QQDownload 732; .NET4.0C; .NET4.0E; SE 2.X MetaSr 1.0)",
        # maxthon浏览器
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Maxthon/4.4.3.4000 Chrome/30.0.1599.101 Safari/537.36",
        # UC浏览器
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 UBrowser/4.0.3214.0 Safari/537.36",
    ]
    #各种移动端
    user_agent_list_3 = [
        # IPhone
        "Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5",
        # IPod
        "Mozilla/5.0 (iPod; U; CPU iPhone OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5",
        # IPAD
        "Mozilla/5.0 (iPad; U; CPU OS 4_2_1 like Mac OS X; zh-cn) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8C148 Safari/6533.18.5",
        "Mozilla/5.0 (iPad; U; CPU OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5",
        # Android
        "Mozilla/5.0 (Linux; U; Android 2.2.1; zh-cn; HTC_Wildfire_A3333 Build/FRG83D) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
        "Mozilla/5.0 (Linux; U; Android 2.3.7; en-us; Nexus One Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
        # QQ浏览器 Android版本
        "MQQBrowser/26 Mozilla/5.0 (Linux; U; Android 2.3.7; zh-cn; MB200 Build/GRJ22; CyanogenMod-7) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
        # Android Opera Mobile
        "Opera/9.80 (Android 2.3.4; Linux; Opera Mobi/build-1107180945; U; en-GB) Presto/2.8.149 Version/11.10",
        # Android Pad Moto Xoom
        "Mozilla/5.0 (Linux; U; Android 3.0; en-us; Xoom Build/HRI39) AppleWebKit/534.13 (KHTML, like Gecko) Version/4.0 Safari/534.13",
        # BlackBerry
        "Mozilla/5.0 (BlackBerry; U; BlackBerry 9800; en) AppleWebKit/534.1+ (KHTML, like Gecko) Version/6.0.0.337 Mobile Safari/534.1+",
        # WebOS HP Touchpad
        "Mozilla/5.0 (hp-tablet; Linux; hpwOS/3.0.0; U; en-US) AppleWebKit/534.6 (KHTML, like Gecko) wOSBrowser/233.70 Safari/534.6 TouchPad/1.0",
        # Nokia N97
        "Mozilla/5.0 (SymbianOS/9.4; Series60/5.0 NokiaN97-1/20.0.019; Profile/MIDP-2.1 Configuration/CLDC-1.1) AppleWebKit/525 (KHTML, like Gecko) BrowserNG/7.1.18124",
        # Windows Phone Mango
        "Mozilla/5.0 (compatible; MSIE 9.0; Windows Phone OS 7.5; Trident/5.0; IEMobile/9.0; HTC; Titan)",
        # UC浏览器
        "UCWEB7.0.2.37/28/999",
        "NOKIA5700/ UCWEB7.0.2.37/28/999",
        # UCOpenwave
        "Openwave/ UCWEB7.0.2.37/28/999",
        # UC Opera
        "Mozilla/4.0 (compatible; MSIE 6.0; ) Opera/UCWEB7.0.2.37/28/999"
    ]
    # 一部分 PC端的
    user_agent_list_1 = [
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1",
        "Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11",
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6",
        "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6",
        "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1",
        "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5",
        "Mozilla/5.0 (Windows NT 6.0) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.36 Safari/536.5",
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
        "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
        "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",
        "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
        "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
        "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.0 Safari/536.3",
        "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24",
        "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24"
    ]
    user_agent_list = user_agent_list_1+user_agent_list_2+user_agent_list_3;
    UserAgent = random.choice(user_agent_list)
    headers = {'User-Agent': UserAgent}
    return headers

https://www.fffmo.com/question/raid0-raid1-raid5-%e4%b8%89%e7%a7%8d%e5%b7%a5%e4%bd%9c%e6%a8%a1%e5%bc%8f%e7%9a%84%e5%b7%a5%e4%bd%9c%e5%8e%9f%e7%90%86%e5%8f%8a%e7%89%b9%e7%82%b9%ef%bc%9f', '0', 'dwqa-question', '', '0');

4 Chrome 常用参数

常用的行为一般有以下几种：

禁止图片和视频的加载：提升网页加载速度。添加代理：用于FQ访问某些页面，或者应对IP访问频率限制的反爬技术。使用移动头：访问移动端的站点，一般这种站点的反爬技术比较薄弱。添加扩展：像正常使用浏览器一样的功能。设置编码：应对中文站，防止乱码。阻止JavaScript执行 …

Chrome Options是一个配置chrome启动时属性的类，通过这个参数我们可以为Chrome添加如下参数：

设置 chrome 二进制文件位置 (binary_location) 添加启动参数 (add_argument) 添加扩展应用 (add_extension, add_encoded_extension) 添加实验性质的设置参数 (add_experimental_option) 设置调试器地址 (debugger_address)

针对编码格式的操作

# 设置默认编码为 utf-8 from selenium import webdriver options = webdriver.ChromeOptions() options.add_argument(’lang=zh_CN.UTF-8’) driver = webdriver.Chrome(chrome_options = options)

针对UA请求头的操作

# 设置请求头为huaweiMeta10 Pro from selenium import webdriver options = webdriver.ChromeOptions() options.add_argument(‘User-Agent=Mozilla/5.0 (Linux; U; Android 8.1.0; zh-cn; BLA-AL00 Build/HUAWEIBLA-AL00) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/57.0.2987.132 MQQBrowser/8.9 Mobile Safari/537.36’) options.add_argument(’–headless’) # 浏览器不提供可视化页面 driver = webdriver.Chrome(chrome_options = options)

http://www.fynas.com/ua

手机UA网址链接

针对禁止加载图片的操作

# 设置浏览器禁止加载图片 from selenium import webdriver options = webdriver.ChromeOptions() prefs = {“profile.managed_default_content_settings.images”: 2} options.add_experimental_option(“prefs”, prefs) driver = webdriver.Chrome(chrome_options = options)

针对IP代理的操作

特别需要注意，在选择代理时，尽量选择静态IP，才能提升爬取的稳定性。如果使用动态匿名IP，每个IP的存活时间是很短的。

代理说明

# 设置无账号密码的代理 chromeOptions = webdriver.ChromeOptions() chromeOptions.add_argument(’–proxy-server=http://ip:port’)
driver = webdriver.Chrome(chrome_options=chromeOptions)

# 设置有账号密码的代理 proxyauth_plugin_path = create_proxyauth_extension( proxy_host=‘host’, proxy_port=‘port’, proxy_username=“username”, proxy_password=“password” ) options.add_extension(proxyauth_plugin_path)

查看IP地址的链接：

[http://httpbin.org/ip](http://httpbin.org/ip)

针对添加插件的操作

\# 添加xpath helper应用

from selenium import webdriver
chrome\_options = webdriver.ChromeOptions()

# 设置好应用扩展
extension\_path = 'your file\_path'
chrome\_options.add\_extension(extension\_path)

针对登录时关闭弹出的密码保存提示框

from selenium import webdriver 
from selenium.webdriver.common.by import By
options = webdriver.ChromeOptions() 
prefs = {} 
# 设置这两个参数就可以避免密码提示框的弹出

prefs\[“credentials\_enable\_service”\] = False 
prefs\[“profile.password\_manager\_enabled”\] = False 
options.add\_experimental\_option(“prefs”, prefs) 
browser = webdriver.Chrome(chrome\_options=options) 
browser.get('https://www.baidu.com/')
browser.quit()

options.add\_argument('--disable-infobars')  # 禁止策略化
options.add\_argument('--no-sandbox')  # 解决DevToolsActivePort文件不存在的报错
options.add\_argument('window-size=1920x3000')  # 指定浏览器分辨率
options.add\_argument('--disable-gpu')  # 谷歌文档提到需要加上这个属性来规避bug
options.add\_argument('--incognito')  # 隐身模式（无痕模式）
options.add\_argument('--disable-javascript')  # 禁用javascript
options.add\_argument('--start-maximized')  # 最大化运行（全屏窗口）,不设置，取元素会报错
options.add\_argument('--disable-infobars')  # 禁用浏览器正在被自动化程序控制的提示
options.add\_argument('--hide-scrollbars')  # 隐藏滚动条, 应对一些特殊页面
options.add\_argument('blink-settings=imagesEnabled=false')  # 不加载图片, 提升速度
options.add\_argument('--headless')  # 浏览器不提供可视化页面. linux下如果系统不支持可视化不加这条会启动失败
options.binary\_location = r"C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe"  # 手动指定使用的浏览器位置

其它配置

selenium 优化

1 selenium 优化

2 禁止加载图片 css

3 随机head

4 Chrome 常用参数

相关推荐

随机文章