27 lines
49 KiB
Python
27 lines
49 KiB
Python
|
import hashlib
|
||
|
|
|
||
|
|
|
||
|
|
def insert_tracker_tab(objs):
|
||
|
|
|
||
|
|
for obj in objs:
|
||
|
|
category = obj['catagory']
|
||
|
|
key = obj['key']
|
||
|
|
items = obj['value']
|
||
|
|
for item in items:
|
||
|
|
product_page_url = item
|
||
|
|
product_page_url_hash = hashlib.md5(product_page_url.encode('utf-8')).hexdigest()
|
||
|
|
flag = 0
|
||
|
|
|
||
|
|
sql = "select * from "+self.config.get('crawler_schema')+"."+self.config.get('tracker_tab')+" where product_page_url = '"+product_page_url+"'"
|
||
|
|
self.cur.execute(sql)
|
||
|
|
res = self.cur.fetchall()
|
||
|
|
|
||
|
|
if not res:
|
||
|
|
sql = "insert into "+self.config.get('crawler_schema')+"."+self.config.get('tracker_tab')+"(crawler_name,category,keyword,product_page_url,product_page_url_hash,flag) values('"+str(self.crawler_name)+"','"+category+"','"+key+"','"+product_page_url+"','"+product_page_url_hash+"',"+str(flag)+")"
|
||
|
|
self.cur.execute(sql)
|
||
|
|
else:
|
||
|
|
print("Already Collected. Skipping......")
|
||
|
|
|
||
|
|
|
||
|
|
section_products = [{'catagory': 3, 'key': 'Hot new releases', 'value': ['https://www.amazon.ae/Ordinary-Glycolic-Acid-Toning-Solution/dp/B0CG149P34?ref_=Oct_d_onr_d_12149483031_0&pd_rd_w=Ex782&content-id=amzn1.sym.f3b61736-36a6-4d09-982b-b2bcfe2a5d41&pf_rd_p=f3b61736-36a6-4d09-982b-b2bcfe2a5d41&pf_rd_r=65Q4BM6NAVRAFDTAT0C4&pd_rd_wg=YA5Iw&pd_rd_r=fc776482-09be-42af-853d-b8ac4e5ca90d&pd_rd_i=B0CG149P34', 'https://www.amazon.ae/Ordinary-Glycolic-Acid-Toning-Solution/dp/B0CG149P34?ref_=Oct_d_onr_d_12149483031_0&pd_rd_w=Ex782&content-id=amzn1.sym.f3b61736-36a6-4d09-982b-b2bcfe2a5d41&pf_rd_p=f3b61736-36a6-4d09-982b-b2bcfe2a5d41&pf_rd_r=65Q4BM6NAVRAFDTAT0C4&pd_rd_wg=YA5Iw&pd_rd_r=fc776482-09be-42af-853d-b8ac4e5ca90d&pd_rd_i=B0CG149P34', 'https://www.amazon.ae/Roche-s%C3%A9rum-r%C3%A9novateur-%C3%A9clat-VITAMINE/dp/B0C9WW6F2M?ref_=Oct_d_onr_d_12149483031_1&pd_rd_w=Ex782&content-id=amzn1.sym.f3b61736-36a6-4d09-982b-b2bcfe2a5d41&pf_rd_p=f3b61736-36a6-4d09-982b-b2bcfe2a5d41&pf_rd_r=65Q4BM6NAVRAFDTAT0C4&pd_rd_wg=YA5Iw&pd_rd_r=fc776482-09be-42af-853d-b8ac4e5ca90d&pd_rd_i=B0C9WW6F2M', 'https://www.amazon.ae/Roche-s%C3%A9rum-r%C3%A9novateur-%C3%A9clat-VITAMINE/dp/B0C9WW6F2M?ref_=Oct_d_onr_d_12149483031_1&pd_rd_w=Ex782&content-id=amzn1.sym.f3b61736-36a6-4d09-982b-b2bcfe2a5d41&pf_rd_p=f3b61736-36a6-4d09-982b-b2bcfe2a5d41&pf_rd_r=65Q4BM6NAVRAFDTAT0C4&pd_rd_wg=YA5Iw&pd_rd_r=fc776482-09be-42af-853d-b8ac4e5ca90d&pd_rd_i=B0C9WW6F2M', 'https://www.amazon.ae/Eucerin-protection-touch-Gel-Cream-SPF50/dp/B0CBVNNSLS?ref_=Oct_d_onr_d_12149483031_2&pd_rd_w=Ex782&content-id=amzn1.sym.f3b61736-36a6-4d09-982b-b2bcfe2a5d41&pf_rd_p=f3b61736-36a6-4d09-982b-b2bcfe2a5d41&pf_rd_r=65Q4BM6NAVRAFDTAT0C4&pd_rd_wg=YA5Iw&pd_rd_r=fc776482-09be-42af-853d-b8ac4e5ca90d&pd_rd_i=B0CBVNNSLS', 'https://www.amazon.ae/Eucerin-protection-touch-Gel-Cream-SPF50/dp/B0CBVNNSLS?ref_=Oct_d_onr_d_12149483031_2&pd_rd_w=Ex782&content-id=amzn1.sym.f3b61736-36a6-4d09-982b-b2bcfe2a5d41&pf_rd_p=f3b61736-36a6-4d09-982b-b2bcfe2a5d41&pf_rd_r=65Q4BM6NAVRAFDTAT0C4&pd_rd_wg=YA5Iw&pd_rd_r=fc776482-09be-42af-853d-b8ac4e5ca90d&pd_rd_i=B0CBVNNSLS', 'https://www.amazon.ae/Cool-Divine-Essence-Exotic-Bloom/dp/B0CDCHQX1L?ref_=Oct_d_onr_d_12149483031_3&pd_rd_w=Ex782&content-id=amzn1.sym.f3b61736-36a6-4d09-982b-b2bcfe2a5d41&pf_rd_p=f3b61736-36a6-4d09-982b-b2bcfe2a5d41&pf_rd_r=65Q4BM6NAVRAFDTAT0C4&pd_rd_wg=YA5Iw&pd_rd_r=fc776482-09be-42af-853d-b8ac4e5ca90d&pd_rd_i=B0CDCHQX1L', 'https://www.amazon.ae/Cool-Divine-Essence-Exotic-Bloom/dp/B0CDCHQX1L?ref_=Oct_d_onr_d_12149483031_3&pd_rd_w=Ex782&content-id=amzn1.sym.f3b61736-36a6-4d09-982b-b2bcfe2a5d41&pf_rd_p=f3b61736-36a6-4d09-982b-b2bcfe2a5d41&pf_rd_r=65Q4BM6NAVRAFDTAT0C4&pd_rd_wg=YA5Iw&pd_rd_r=fc776482-09be-42af-853d-b8ac4e5ca90d&pd_rd_i=B0CDCHQX1L', 'https://www.amazon.ae/Paulas-Choice-Blackheads-Lines-SKIN-PERFECTING/dp/B0CDV575YT?ref_=Oct_d_onr_d_12149483031_4&pd_rd_w=Ex782&content-id=amzn1.sym.f3b61736-36a6-4d09-982b-b2bcfe2a5d41&pf_rd_p=f3b61736-36a6-4d09-982b-b2bcfe2a5d41&pf_rd_r=65Q4BM6NAVRAFDTAT0C4&pd_rd_wg=YA5Iw&pd_rd_r=fc776482-09be-42af-853d-b8ac4e5ca90d&pd_rd_i=B0CDV575YT', 'https://www.amazon.ae/Paulas-Choice-Blackheads-Lines-SKIN-PERFECTING/dp/B0CDV575YT?ref_=Oct_d_onr_d_12149483031_4&pd_rd_w=Ex782&content-id=amzn1.sym.f3b61736-36a6-4d09-982b-b2bcfe2a5d41&pf_rd_p=f3b61736-36a6-4d09-982b-b2bcfe2a5d41&pf_rd_r=65Q4BM6NAVRAFDTAT0C4&pd_rd_wg=YA5Iw&pd_rd_r=fc776482-09be-42af-853d-b8ac4e5ca90d&pd_rd_i=B0CDV575YT', 'https://www.amazon.ae/Beauty-Joseeon-Relief-Probiotics-1-69fl-oz/dp/B0CGXQXK5B?ref_=Oct_d_onr_d_12149483031_5&pd_rd_w=Ex782&content-id=amzn1.sym.f3b61736-36a6-4d09-982b-b2bcfe2a5d41&pf_rd_p=f3b61736-36a6-4d09-982b-b2bcfe2a5d41&pf_rd_r=65Q4BM6NAVRAFDTAT0C4&pd_rd_wg=YA5Iw&pd_rd_r=fc776482-09be-42af-853d-b8ac4e5ca90d&pd_rd_i=B0CGXQXK5B', 'https://www.amazon.ae/Beauty-Joseeon-Relief-Probiotics-1-69fl-oz/dp/B0CGXQXK5B?ref_=Oct_d_onr_d_12149483031_5&pd_rd_w=Ex782&content-id=amzn1.sym.f3b61736-36a6-4d09-982b-b2bcfe2a5d41&pf_rd_p=f3b61736-36a6-4d09-982b-b2bcfe2a5d41&pf_rd_r=65Q4BM6NAVRAFDTAT0C4&pd_rd_wg=YA5Iw&pd_rd_r=fc776482-09be-42af-853d-b8ac
|
||
|
|
insert_tracker_tab(section_products)
|