Shariar
/
raena-crawler-engine
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

FB Marketplace crawler

This commit is contained in:
Shariar Imtiaz 2024-02-05 14:33:41 +04:00
parent aaf84022e6
commit e4b1f8c269
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
fb_marketplace/fb_marketplace_product_info.py
View File

@ -25,7 +25,7 @@ class FbMarketplaceProductInfo:
op.add_argument('--user-data-dir=/home/ec2-user/fb-crawler-binary/chrome/fb_marketplace') op.add_argument('--user-data-dir=/home/ec2-user/fb-crawler-binary/chrome/fb_marketplace')
self.version_main = 120 self.version_main = 120
self.driver = Chrome(options=op, version_main=self.version_main) self.driver = Chrome(options=op, version_main=self.version_main)
time.sleep(3000) #time.sleep(3000)
self.cur.execute("select id from " + self.config.get('crawler_schema') + "." + self.config.get( self.cur.execute("select id from " + self.config.get('crawler_schema') + "." + self.config.get(
'source_tab') + " where source_name='fb_marketplace'") 'source_tab') + " where source_name='fb_marketplace'")
@ -40,7 +40,7 @@ class FbMarketplaceProductInfo:
def run_product_pipeline(self): def run_product_pipeline(self):
table = f"""{self.config.get('crawler_schema')}.{self.config.get('tracker_tab')}""" table = f"""{self.config.get('crawler_schema')}.{self.config.get('tracker_tab')} limit 1"""
sql = f""" sql = f"""
SELECT * FROM {table} where flag=0 limit 1 SELECT * FROM {table} where flag=0 limit 1