Shariar
/
raena-crawler-engine
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

added Hasaki crawler

This commit is contained in:
Shariar Imtiaz 2024-03-15 16:17:42 +04:00
parent 30732aa065
commit ed49c2e464
2 changed files with 15 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
hasaki_crawler_engine/hasaki_product_info.py
View File

@ -92,12 +92,20 @@ class HasakiProductInfo:
for _ in range(retries): for _ in range(retries):
try: try:
with sync_playwright() as p: with sync_playwright() as p:
browser = p.chromium.launch(headless=False) browser = p.chromium.launch(
headless=False,
args=[
"--disable-dev-shm-usage",
"--disable-blink-features=AutomationControlled",
"--disable-component-extensions-with-background-pages"
]
)
ua = UserAgent(platforms='mobile') ua = UserAgent(platforms='mobile')
random_mobile_ua = ua.random random_mobile_ua = ua.random
logging.info("using user agent: {}".format(random_mobile_ua)) logging.info("using user agent: {}".format(random_mobile_ua))
context = browser.new_context(user_agent=random_mobile_ua) context = browser.new_context(user_agent=random_mobile_ua)
context.add_init_script("Object.defineProperty(navigator, 'webdriver', {get: () => undefined})")
page = context.new_page() page = context.new_page()
api_requests = {} api_requests = {}

7
hasaki_crawler_engine/test.py
View File

@ -6,12 +6,17 @@ import logging
with sync_playwright() as p: with sync_playwright() as p:
browser = p.chromium.launch(headless=False) browser = p.chromium.launch(headless=False, args=[
"--disable-dev-shm-usage",
"--disable-blink-features=AutomationControlled",
"--disable-component-extensions-with-background-pages"
])
ua = UserAgent(platforms='mobile') ua = UserAgent(platforms='mobile')
random_mobile_ua = ua.random random_mobile_ua = ua.random
logging.info("using user agent: {}".format(random_mobile_ua)) logging.info("using user agent: {}".format(random_mobile_ua))
context = browser.new_context(user_agent=random_mobile_ua) context = browser.new_context(user_agent=random_mobile_ua)
context.add_init_script("Object.defineProperty(navigator, 'webdriver', {get: () => undefined})")
page = context.new_page() page = context.new_page()
try: try: