diff --git a/hasaki_crawler_engine/conf.json b/hasaki_crawler_engine/conf.json index 88f2bfe..e822066 100755 --- a/hasaki_crawler_engine/conf.json +++ b/hasaki_crawler_engine/conf.json @@ -21,6 +21,5 @@ "database": "analytics", "db_host": "redshift-cluster-1.cdqj58hfx4p7.ap-southeast-1.redshift.amazonaws.com", "db_port": "5439", - "crawler_main": "1", - "crawler_slave_no": "" + "log_loc": "/home/ubuntu/logs/hasaki_crawler.log" } \ No newline at end of file diff --git a/hasaki_crawler_engine/hasaki_categories.py b/hasaki_crawler_engine/hasaki_categories.py index 6dcbadf..1172906 100644 --- a/hasaki_crawler_engine/hasaki_categories.py +++ b/hasaki_crawler_engine/hasaki_categories.py @@ -10,21 +10,17 @@ from hasaki_db_writer import hasaki_db_writer from Util import translate_text_to_english -###### Looger ###### -logname = '/home/ubuntu/logs/hasaki_crawler.log' -#logname = 'hasaki_crawler.log' -logging.basicConfig(filename=logname, - filemode='a', - format='%(asctime)s,%(msecs)d %(name)s %(levelname)s: %(message)s', - datefmt="%Y-%m-%d %H:%M:%S", - level=logging.INFO) - - class HasakiCategories: def __init__(self, config): + ###### Looger ###### + logging.basicConfig(filename=config.get("log_loc"), + filemode='a', + format='%(asctime)s,%(msecs)d %(name)s %(levelname)s: %(message)s', + datefmt="%Y-%m-%d %H:%M:%S", + level=logging.INFO) logging.info("Initializing HasakiSubCategories") self.master_category = [] self.config = config diff --git a/hasaki_crawler_engine/hasaki_category_products.py b/hasaki_crawler_engine/hasaki_category_products.py index cadf1f2..b7883aa 100644 --- a/hasaki_crawler_engine/hasaki_category_products.py +++ b/hasaki_crawler_engine/hasaki_category_products.py @@ -7,17 +7,15 @@ from playwright.sync_api import sync_playwright from hasaki_db_writer import hasaki_db_writer from Util import translate_text_to_english -###### Looger ###### -logname = '/home/ubuntu/logs/hasaki_crawler.log' -#logname = 'hasaki_crawler.log' -logging.basicConfig(filename=logname, - filemode='a', - format='%(asctime)s,%(msecs)d %(name)s %(levelname)s: %(message)s', - datefmt="%Y-%m-%d %H:%M:%S", - level=logging.INFO) class HasakiCategoryProducts: def __init__(self, config): + ###### Looger ###### + logging.basicConfig(filename=config.get("log_loc"), + filemode='a', + format='%(asctime)s,%(msecs)d %(name)s %(levelname)s: %(message)s', + datefmt="%Y-%m-%d %H:%M:%S", + level=logging.INFO) logging.info("Initializing HasakiCategoryProducts........") self.config = config self.crawler_name = self.config.get("crawler_name") diff --git a/hasaki_crawler_engine/hasaki_crawler.py b/hasaki_crawler_engine/hasaki_crawler.py index b728b41..ecaf2fd 100644 --- a/hasaki_crawler_engine/hasaki_crawler.py +++ b/hasaki_crawler_engine/hasaki_crawler.py @@ -13,16 +13,6 @@ from email.message import EmailMessage config = {} -###### Looger ###### -logname = '/home/ubuntu/logs/hasaki_crawler.log' -#logname = 'hasaki_crawler.log' -logging.basicConfig(filename=logname, - filemode='a', - format='%(asctime)s,%(msecs)d %(name)s %(levelname)s: %(message)s', - datefmt="%Y-%m-%d %H:%M:%S", - level=logging.INFO) - - def main(cur): @@ -181,6 +171,13 @@ if __name__ == "__main__": logging.info("Config file loaded.......") logging.info(config) + ###### Looger ###### + logging.basicConfig(filename=config.get("log_loc"), + filemode='a', + format='%(asctime)s,%(msecs)d %(name)s %(levelname)s: %(message)s', + datefmt="%Y-%m-%d %H:%M:%S", + level=logging.INFO) + conn = sqlite3.connect('process_tracker.db') conn.isolation_level = None diff --git a/hasaki_crawler_engine/hasaki_db_writer.py b/hasaki_crawler_engine/hasaki_db_writer.py index f9b3d32..0d37d8e 100755 --- a/hasaki_crawler_engine/hasaki_db_writer.py +++ b/hasaki_crawler_engine/hasaki_db_writer.py @@ -1,17 +1,15 @@ import logging import psycopg2 -###### Looger ###### -logname = '/home/ubuntu/logs/hasaki_crawler.log' -#logname = 'hasaki_crawler.log' -logging.basicConfig(filename=logname, - filemode='a', - format='%(asctime)s,%(msecs)d %(name)s %(levelname)s: %(message)s', - datefmt="%Y-%m-%d %H:%M:%S", - level=logging.INFO) class hasaki_db_writer: def __init__(self, config): + ###### Looger ###### + logging.basicConfig(filename=config.get("log_loc"), + filemode='a', + format='%(asctime)s,%(msecs)d %(name)s %(levelname)s: %(message)s', + datefmt="%Y-%m-%d %H:%M:%S", + level=logging.INFO) self.config = config self.conn = psycopg2.connect(database=self.config.get('database'), user=self.config.get('db_user'), password=self.config.get('db_pass'), host=self.config.get('db_host'), port=self.config.get('db_port')) self.conn.autocommit = True diff --git a/hasaki_crawler_engine/hasaki_product_info.py b/hasaki_crawler_engine/hasaki_product_info.py index a418e6d..5abd1ef 100644 --- a/hasaki_crawler_engine/hasaki_product_info.py +++ b/hasaki_crawler_engine/hasaki_product_info.py @@ -19,17 +19,15 @@ from webdriver_manager.chrome import ChromeDriverManager import brotli import json -###### Looger ###### -logname = '/home/ubuntu/logs/hasaki_crawler.log' -#logname = 'hasaki_crawler.log' -logging.basicConfig(filename=logname, - filemode='a', - format='%(asctime)s,%(msecs)d %(name)s %(levelname)s: %(message)s', - datefmt="%Y-%m-%d %H:%M:%S", - level=logging.INFO) class HasakiProductInfo: def __init__(self, config): + ###### Looger ###### + logging.basicConfig(filename=config.get("log_loc"), + filemode='a', + format='%(asctime)s,%(msecs)d %(name)s %(levelname)s: %(message)s', + datefmt="%Y-%m-%d %H:%M:%S", + level=logging.INFO) logging.info("Initializing HasakiProductInfo") self.pattern = r'[' + string.punctuation + ']' self.config = config