Shariar
/
raena-crawler-engine
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

added Hasaki crawler

This commit is contained in:
Shariar Imtiaz 2024-04-03 13:42:04 +04:00
parent 3d87935aa2
commit f46771b287
1 changed files with 9 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
hasaki_crawler_engine/test_selenium.py
View File

@ -2,6 +2,7 @@ import logging
from fake_useragent import UserAgent from fake_useragent import UserAgent
import brotli import brotli
import seleniumwire.undetected_chromedriver as uc import seleniumwire.undetected_chromedriver as uc
from selenium_stealth import stealth
import json import json
import time import time
@ -12,6 +13,7 @@ def get_raw_product_data_selenium(url):
op = uc.ChromeOptions() op = uc.ChromeOptions()
op.add_argument(f"user-agent={random_mobile_ua}") op.add_argument(f"user-agent={random_mobile_ua}")
op.add_argument('--blink-settings=imagesEnabled=false')
#op.add_experimental_option("useAutomationExtension", False) #op.add_experimental_option("useAutomationExtension", False)
#op.add_argument('--no-sandbox') #op.add_argument('--no-sandbox')
#op.add_argument('--disable-notifications') #op.add_argument('--disable-notifications')
@ -19,6 +21,13 @@ def get_raw_product_data_selenium(url):
op.headless = False op.headless = False
driver = uc.Chrome(version_main=122, options=op) driver = uc.Chrome(version_main=122, options=op)
stealth(driver,
languages=["en-US", "en"],
vendor="Google Inc.",
webgl_vendor="Intel Inc.",
renderer="Intel Iris OpenGL Engine",
fix_hairline=True,
)
driver.get(url) driver.get(url)