import logging
import json
import time
import smtplib
import sqlite3
from hasaki_categories import HasakiCategories
from hasaki_category_products import HasakiCategoryProducts
from hasaki_product_info import HasakiProductInfo
from email.message import EmailMessage
##### Looger ######
format = "%(asctime)s: %(message)s"
logging.basicConfig(format=format, level=logging.INFO, datefmt="%Y-%m-%d %H:%M:%S")
config = {}
def main(cur):
cur.execute(f"""select flag from process_tracker where process = 'category'""")
cat_flags = cur.fetchone()
if cat_flags[0]==0:
hasaki_categories = HasakiCategories(config)
hasaki_categories.start_processing()
cur.execute(f"""update process_tracker set flag = 1 where process = 'category'""")
#time.sleep(60)
cur.execute(f"""select flag from process_tracker where process = 'category_product'""")
cat_pro_flags = cur.fetchone()
if cat_pro_flags[0] == 0:
hasaki_category_products = HasakiCategoryProducts(config)
hasaki_category_products.start_processing()
cur.execute(f"""update process_tracker set flag = 1 where process = 'category_product'""")
#time.sleep(60)
cur.execute(f"""select flag from process_tracker where process = 'product_info'""")
prod_flag = cur.fetchone()
if prod_flag[0] == 0:
hasaki_products = HasakiProductInfo(config)
hasaki_products.start_processing()
cur.execute(f"""update process_tracker set flag = 1 where process = 'product_info'""")
else:
cur.execute(f"""update process_tracker set flag = 0 where process = 'category'""")
cur.execute(f"""update process_tracker set flag = 0 where process = 'category_product'""")
cur.execute(f"""update process_tracker set flag = 0 where process = 'product_info'""")
def send_mail(msg):
try:
EMAIL_ADDRESS = "AKIAR2YL57QC6NITTJN5"
EMAIL_PASSWORD = "BAs9W772KNxLL1xnMzYhdIkpflQ8H+KP0Zbl8dphQZWh"
From = 'data_reporting@raenabeauty.com'
To = 'shariar@raenabeauty.com, data_reporting@raenabeauty.com'
# To = 'shariar@raenabeauty.com'
html = f'''
Hasaki Crawler Status
{msg}
This is system generated mail. Please do not reply
'''
msg = EmailMessage()
msg['Subject'] = 'Hasaki Crawler Status'
msg['From'] = From
msg['To'] = To
msg.set_content(html, subtype='html')
with smtplib.SMTP('email-smtp.ap-southeast-1.amazonaws.com', 587) as smtp:
smtp.ehlo()
smtp.starttls()
smtp.login(EMAIL_ADDRESS, EMAIL_PASSWORD)
smtp.send_message(msg)
except Exception as e:
logging.info("Error while sending mail: {}".format(e))
def init_tracker_tab(cur):
cur.execute(f"""CREATE TABLE IF NOT EXISTS process_tracker (
process TEXT,
flag int
)""")
# logging.info("++++++++++++++++++++++++++++++++++++++")
# cur.execute(f"""select * from process_tracker""")
# logging.info(cur.fetchall())
cur.execute(f"""select * from process_tracker where process = 'category'""")
if cur.fetchone() is None:
cur.execute(f"""insert into process_tracker (process, flag) values('category', 0)""")
cur.execute(f"""select * from process_tracker where process = 'category_product'""")
if cur.fetchone() is None:
cur.execute(f"""insert into process_tracker (process, flag) values('category_product', 0)""")
cur.execute(f"""select * from process_tracker where process = 'product_info'""")
if cur.fetchone() is None:
cur.execute(f"""insert into process_tracker (process, flag) values('product_info', 0)""")
logging.info("++++++++++++++++ process tracker tab status ++++++++++++++++++++++")
cur.execute(f"""select * from process_tracker""")
logging.info(cur.fetchall())
if __name__ == "__main__":
logging.info("Starting Hasaki Crawler.......")
try:
logging.info("Loading config file.......")
with open("conf.json", "r") as jsonfile:
config = json.load(jsonfile)
logging.info("Config file loaded.......")
logging.info(config)
conn = sqlite3.connect('process_tracker.db')
conn.isolation_level = None
cur = conn.cursor()
# cur.execute(f"""update process_tracker set flag = 1 where process = 'category'""")
# cur.execute(f"""update process_tracker set flag = 1 where process = 'category_product'""")
# cur.execute(f"""update process_tracker set flag = 0 where process = 'product_info'""")
init_tracker_tab(cur)
main(cur)
cur.close()
conn.close()
send_mail("Hasaki crawler run complete.")
except Exception as e:
logging.info("Error: ".format(e))
logging.info("Error occurred. Please check config file or the internal SQLLITE DB. Exiting......")
send_mail("Error occurred. Please check config file or the internal SQLLITE DB.")
exit(1)