110 lines
3.6 KiB
Python
110 lines
3.6 KiB
Python
from tokopedia_logger import logger
|
|
from tokopedia_db_writer import DB
|
|
from tokopedia_config import Config
|
|
|
|
class db_migrations():
|
|
config = Config().get()
|
|
|
|
def __init__(self):
|
|
logger.info('Running database migrations')
|
|
self.updateSource()
|
|
self.updateCategoryColumn()
|
|
self.alterCrawlerTracker()
|
|
self.alterProductTab()
|
|
self.alterResellerStoreTab()
|
|
logger.info('Database migrations completed')
|
|
|
|
def updateSource(self):
|
|
table = f'{self.config.get("crawler_schema")}.{self.config.get("source_tab")}'
|
|
target = self.config.get("crawler_target")
|
|
target_url = self.config.get("crawler_target_url")
|
|
query = f'''INSERT INTO {table} (source_name, source_main_url)
|
|
SELECT '{target}', '{target_url}'
|
|
WHERE
|
|
NOT EXISTS (
|
|
SELECT id FROM {table} WHERE source_name = '{target}'
|
|
);'''
|
|
try:
|
|
DB().execute_query(query)
|
|
except:
|
|
logger.error(f'Problem while creating source in {table}')
|
|
exit(1)
|
|
|
|
def updateCategoryColumn(self):
|
|
table = f'{self.config.get("crawler_schema")}.{self.config.get("category_tab")}'
|
|
aud_table = f'{self.config.get("crawler_schema")}.aud_{self.config.get("category_tab")}'
|
|
query = f'Alter table {table} ADD COLUMN IF NOT EXISTS category_slug character varying UNIQUE'
|
|
aud_query = f'Alter table {aud_table} ADD COLUMN IF NOT EXISTS category_slug character varying UNIQUE'
|
|
try:
|
|
DB().execute_query(query)
|
|
DB().execute_query(aud_query)
|
|
except:
|
|
logger.error(f'Problem while updating column in {table}')
|
|
exit(1)
|
|
|
|
def alterCrawlerTracker(self):
|
|
table = f'{self.config.get("crawler_schema")}.{self.config.get("tracker_tab")}'
|
|
query = f'''
|
|
ALTER TABLE {table}
|
|
ADD CONSTRAINT unique_product_page_url UNIQUE (product_page_url);
|
|
'''
|
|
try:
|
|
DB().execute_query(query)
|
|
except:
|
|
# This might be the reason of a silent error
|
|
pass
|
|
|
|
def alterProductTab(self):
|
|
table = f'{self.config.get("crawler_schema")}.{self.config.get("product_tab")}'
|
|
aud_table = f'{self.config.get("crawler_schema")}.aud_{self.config.get("product_tab")}'
|
|
query = f'Alter table {table} ADD COLUMN IF NOT EXISTS rce_source_id bigint;'
|
|
aud_query = f'Alter table {aud_table} ADD COLUMN IF NOT EXISTS rce_source_id bigint;'
|
|
|
|
constraint_query = f'''
|
|
ALTER TABLE {table}
|
|
ADD CONSTRAINT product_source_id_ukey UNIQUE (rce_source_product_id, rce_source_id);
|
|
'''
|
|
|
|
try:
|
|
DB().execute_query(query + aud_query)
|
|
except:
|
|
logger.error(f'Problem while updating column in {table}')
|
|
exit(1)
|
|
|
|
try:
|
|
DB().execute_query(constraint_query)
|
|
except:
|
|
pass
|
|
|
|
def alterResellerStoreTab(self):
|
|
table = f'{self.config.get("crawler_schema")}.{self.config.get("reseller_store_tab")}'
|
|
aud_table = f'{self.config.get("crawler_schema")}.aud_{self.config.get("reseller_store_tab")}'
|
|
query = f'Alter table {table} ADD COLUMN IF NOT EXISTS rce_source_id bigint;'
|
|
aud_query = f'Alter table {aud_table} ADD COLUMN IF NOT EXISTS rce_source_id bigint;'
|
|
|
|
constraint_query = f'''
|
|
ALTER TABLE {table}
|
|
ADD CONSTRAINT store_source_id_ukey UNIQUE (rce_source_store_id, rce_source_id);
|
|
'''
|
|
|
|
aud_constraint_query = f'''
|
|
ALTER TABLE {aud_table}
|
|
ADD CONSTRAINT aud_store_source_id_ukey UNIQUE (rce_source_store_id, rce_source_id);
|
|
'''
|
|
|
|
try:
|
|
DB().execute_query(query + aud_query)
|
|
except:
|
|
logger.error(f'Problem while updating column in {table}')
|
|
exit(1)
|
|
|
|
try:
|
|
DB().execute_query(constraint_query)
|
|
except:
|
|
pass
|
|
|
|
try:
|
|
DB().execute_query(aud_constraint_query)
|
|
except:
|
|
pass
|