Shariar
/
raena-crawler-engine
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

FB Marketplace crawler

This commit is contained in:
Shariar Imtiaz 2024-02-01 10:57:02 +04:00
parent 309bc78606
commit 5d1e79452b
2 changed files with 4 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
fb_marketplace/fb_marketplace_product_info.py
View File

@ -22,9 +22,9 @@ class FbMarketplaceProductInfo:
self.cur = self.conn.cursor() self.cur = self.conn.cursor()
op = ChromeOptions() op = ChromeOptions()
op.add_argument('--user-data-dir=/Users/simtiaz/chrome_cache/fb_marketplace') op.add_argument('--user-data-dir=/home/ec2-user/fb-crawler-binary/chrome/fb_marketplace')
self.version_main = 120 self.version_main = 120
self.driver = Chrome(options=op, version_main=120) self.driver = Chrome(options=op, version_main=self.version_main)
#time.sleep(3000) #time.sleep(3000)
self.cur.execute("select id from " + self.config.get('crawler_schema') + "." + self.config.get( self.cur.execute("select id from " + self.config.get('crawler_schema') + "." + self.config.get(

4
fb_marketplace/fb_marketplace_products.py
View File

@ -25,14 +25,14 @@ class FbMarketplaceProducts:
self.cur = self.conn.cursor() self.cur = self.conn.cursor()
op = ChromeOptions() op = ChromeOptions()
op.add_argument('--user-data-dir=/Users/simtiaz/chrome_cache/fb_marketplace') op.add_argument('--user-data-dir=/home/ec2-user/fb-crawler-binary/chrome/fb_marketplace')
###### S3 bucket information ###### S3 bucket information
self.S3_KEY = "AKIAR2YL57QCWO34OE4J" self.S3_KEY = "AKIAR2YL57QCWO34OE4J"
self.S3_SECRET = "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" self.S3_SECRET = "kCDHvIPzM1sdN8JG37vL1Zujt1EmiRBTQhuJnXVJ"
self.S3_BUCKET = "s3://raena-prod-data-engineering/mongo_order/" self.S3_BUCKET = "s3://raena-prod-data-engineering/mongo_order/"
self.version_main = 120 self.version_main = 120
self.driver = Chrome(options=op, version_main=120) self.driver = Chrome(options=op, version_main=self.version_main)
def __del__(self): def __del__(self):
self.driver.quit() self.driver.quit()