44 lines
1.1 KiB
Python
44 lines
1.1 KiB
Python
|
from tokopedia_logger import logger
|
||
|
|
from tokopedia_db_writer import DB
|
||
|
|
from tokopedia_config import Config
|
||
|
|
from tokopedia_sub_categories import TokopediaSubCategories
|
||
|
|
from tokopedia_db_migrations import db_migrations
|
||
|
|
from tokopedia_product_list import ProductList
|
||
|
|
from tokopedia_products import Products
|
||
|
|
|
||
|
|
def checkSource():
|
||
|
|
config = Config().get()
|
||
|
|
table = config.get("crawler_schema") + "." + config.get("source_tab")
|
||
|
|
query = "select id from " + table + " where source_name='Tokopedia'"
|
||
|
|
data = DB().fetchone(query)
|
||
|
|
if not data:
|
||
|
|
logger.error("Please create source in " + table)
|
||
|
|
exit(1)
|
||
|
|
|
||
|
|
def runMainCrawler():
|
||
|
|
db_migrations()
|
||
|
|
checkSource()
|
||
|
|
TokopediaSubCategories()
|
||
|
|
ProductList()
|
||
|
|
|
||
|
|
def runSlaveCrawler():
|
||
|
|
config = Config().get()
|
||
|
|
try:
|
||
|
|
int(config.get('crawler_slave_no'))
|
||
|
|
except:
|
||
|
|
logger.error("Please set slave number")
|
||
|
|
exit(1)
|
||
|
|
Products()
|
||
|
|
|
||
|
|
def main():
|
||
|
|
config = Config().get()
|
||
|
|
isMainCrawler = bool(int(config.get('crawler_main')))
|
||
|
|
|
||
|
|
if isMainCrawler:
|
||
|
|
runMainCrawler()
|
||
|
|
else:
|
||
|
|
runSlaveCrawler()
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
main()
|