Shariar
/
raena-crawler-engine
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

added Hasaki crawler

This commit is contained in:
Shariar Imtiaz 2024-04-01 11:59:47 +04:00
parent 82a3d9d9b8
commit 1e7dcaa894
1 changed files with 21 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
hasaki_crawler_engine/hasaki_categories.py
View File

@ -20,27 +20,30 @@ logging.basicConfig(filename="/home/ubuntu/logs/hasaki_crawler.log",
class HasakiCategories:
def __init__(self, config):
logging.info("Initializing HasakiSubCategories")
self.master_category = []
self.config = config
self.crawler_name = self.config.get("crawler_name")
self.product_limit = int(self.config.get("product_per_category"))
self.conn = psycopg2.connect(database=self.config.get('database'), user=self.config.get('db_user'),
password=self.config.get('db_pass'), host=self.config.get('db_host'),
port=self.config.get('db_port'))
self.conn.autocommit = True
self.cur = self.conn.cursor()
self.cur.execute(f"""select id from {self.config.get('crawler_schema')}.{self.config.get('source_tab')} where source_name='Hasaki'""")
try:
self.rce_source_id = self.cur.fetchone()[0]
except:
logging.info("Source tab is empty. Please check. Exiting.....")
exit(1)
logging.info("Initializing HasakiSubCategories")
self.master_category = []
self.config = config
self.crawler_name = self.config.get("crawler_name")
self.product_limit = int(self.config.get("product_per_category"))
self.conn = psycopg2.connect(database=self.config.get('database'), user=self.config.get('db_user'),
password=self.config.get('db_pass'), host=self.config.get('db_host'),
port=self.config.get('db_port'))
self.conn.autocommit = True
self.cur = self.conn.cursor()
self.cur.execute(f"""select id from {self.config.get('crawler_schema')}.{self.config.get('source_tab')} where source_name='Hasaki'""")
try:
self.rce_source_id = self.cur.fetchone()[0]
except:
logging.info("Source tab is empty. Please check. Exiting.....")
exit(1)
self.db_writer = hasaki_db_writer(config)
self.db_writer = hasaki_db_writer(config)
self.display = Display(visible=0, size=(800, 600))
self.display.start()
self.display = Display(visible=0, size=(800, 600))
self.display.start()
except Exception as e:
logging.info(e)
def __del__(self):
print("Closing connection.....")