Shariar
/
raena-crawler-engine
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

FB Marketplace crawler

This commit is contained in:
Shariar Imtiaz 2024-02-01 13:42:27 +04:00
parent 2b0800dd46
commit 5c341453ac
1 changed files with 7 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
fb_marketplace/fb_marketplace_products.py
View File

@ -18,14 +18,10 @@ class FbMarketplaceProducts:
self.config = config self.config = config
self.url = [ self.url = [
"https://www.facebook.com/marketplace/hochiminhcity/beauty-products", "https://www.facebook.com/marketplace/hochiminhcity/beauty-products",
"https://www.facebook.com/marketplace/106388046062960/beauty-products/?exact=false", "https://www.facebook.com/marketplace/113269895355265/beauty-products/?exact=false",
"https://www.facebook.com/marketplace/111711568847056/beauty-products/?exact=false", "https://www.facebook.com/marketplace/111711568847056/beauty-products/?exact=false",
"https://www.facebook.com/marketplace/107751605926185/beauty-products/?exact=false",
"https://www.facebook.com/marketplace/108121405876426/beauty-products/?exact=false",
"https://www.facebook.com/marketplace/109205905763791/beauty-products/?exact=false",
"https://www.facebook.com/marketplace/109477335744202/beauty-products/?exact=false",
"https://www.facebook.com/marketplace/114668461883395/beauty-products/?exact=false", "https://www.facebook.com/marketplace/114668461883395/beauty-products/?exact=false",
"https://www.facebook.com/marketplace/112933135384816/beauty-products/?exact=false" "https://www.facebook.com/marketplace/106388046062960/beauty-products/?exact=false"
] ]
self.conn = psycopg2.connect(database=self.config.get('database'), user=self.config.get('db_user'), self.conn = psycopg2.connect(database=self.config.get('database'), user=self.config.get('db_user'),
@ -57,11 +53,11 @@ class FbMarketplaceProducts:
##### Scrolling ##### Scrolling
#smartScroll(self.driver, stopAtBorder=True, distancePerSecond=2000, humanBreaks=True, timeout=10, stopWhenDocHeightNotChangedSince=60) #smartScroll(self.driver, stopAtBorder=True, distancePerSecond=2000, humanBreaks=True, timeout=10, stopWhenDocHeightNotChangedSince=60)
######## Scroll down using Page Down key ####### Scroll down using Page Down key
# html_element = self.driver.find_element(By.TAG_NAME, 'html') html_element = self.driver.find_element(By.TAG_NAME, 'html')
# for _ in range(1000): for _ in range(1000):
# html_element.send_keys(Keys.PAGE_DOWN) html_element.send_keys(Keys.PAGE_DOWN)
# time.sleep(random.randint(1,3)) time.sleep(random.randint(1,3))
product_elements = self.driver.find_elements(By.CSS_SELECTOR,".x9f619.x78zum5.x1r8uery.xdt5ytf.x1iyjqo2.xs83m0k.x1e558r4.x150jy0e.x1iorvi4.xjkvuk6.xnpuxes.x291uyu.x1uepa24") product_elements = self.driver.find_elements(By.CSS_SELECTOR,".x9f619.x78zum5.x1r8uery.xdt5ytf.x1iyjqo2.xs83m0k.x1e558r4.x150jy0e.x1iorvi4.xjkvuk6.xnpuxes.x291uyu.x1uepa24")