Shariar
/
raena-crawler-engine
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

FB Marketplace crawler

This commit is contained in:
Shariar Imtiaz 2024-02-05 14:57:57 +04:00
parent 7e46c28355
commit 245a01d64a
1 changed files with 4 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
fb_marketplace/fb_marketplace_product_info.py
View File

@ -50,9 +50,12 @@ class FbMarketplaceProductInfo:
results = self.cur.fetchall() results = self.cur.fetchall()
logging.info("Total rows: {}".format(len(results)))
cnt = 1
for result in results: for result in results:
try: try:
logging.info("============== Collecting data for {} ==============".format(result[0])) logging.info("============== {}/{}: Collecting data for {} ==============".format(str(cnt),str(len(results)),result[0]))
self.getProductInfo(result) self.getProductInfo(result)
sql = f""" sql = f"""
update {self.config.get('crawler_schema')}.{self.config.get('tracker_tab')} set flag=1 where name='{result[0]}' and price={result[1]} and url='{result[2]}' update {self.config.get('crawler_schema')}.{self.config.get('tracker_tab')} set flag=1 where name='{result[0]}' and price={result[1]} and url='{result[2]}'