38 lines
1.7 KiB
Python
38 lines
1.7 KiB
Python
|
from fb_marketplace_products import FbMarketplaceProducts
|
||
|
|
from fb_marketplace_product_info import FbMarketplaceProductInfo
|
||
|
|
import logging
|
||
|
|
|
||
|
|
###### Looger ######
|
||
|
|
format = "%(asctime)s: %(message)s"
|
||
|
|
logging.basicConfig(format=format, level=logging.INFO, datefmt="%Y-%m-%d %H:%M:%S")
|
||
|
|
|
||
|
|
|
||
|
|
config = {
|
||
|
|
"crawler_name": "raena_crawler_engine_fb_marketplace",
|
||
|
|
"crawler_schema": "test_spider_management",
|
||
|
|
"tracker_tab": "facebook_crawler_tracker",
|
||
|
|
"category_tab": "rce_category",
|
||
|
|
"product_tab": "rce_product",
|
||
|
|
"reseller_tab": "rce_reseller",
|
||
|
|
"reseller_store_tab": "rce_reseller_store",
|
||
|
|
"source_tab": "rce_source",
|
||
|
|
"db_user": "dbadmin",
|
||
|
|
"db_pass": "5qCif6eyY3Kmg4z",
|
||
|
|
"database": "analytics",
|
||
|
|
"db_host": "redshift-cluster-1.cdqj58hfx4p7.ap-southeast-1.redshift.amazonaws.com",
|
||
|
|
"db_port": "5439"
|
||
|
|
}
|
||
|
|
|
||
|
|
def run_pipeline ():
|
||
|
|
#marketplace_products = FbMarketplaceProducts(config=config)
|
||
|
|
#marketplace_products.getProducts()
|
||
|
|
|
||
|
|
#url = "https://www.facebook.com/marketplace/item/266638329606593?ref=category_feed&referral_code=null&referral_story_type=listing&tracking=%7B%22qid%22%3A%22-2656048044644247513%22%2C%22mf_story_key%22%3A%2224843562218625057%22%2C%22commerce_rank_obj%22%3A%22%7B%5C%22target_id%5C%22%3A24843562218625057%2C%5C%22target_type%5C%22%3A0%2C%5C%22primary_position%5C%22%3A43%2C%5C%22ranking_signature%5C%22%3A6122896018789849685%2C%5C%22commerce_channel%5C%22%3A504%2C%5C%22value%5C%22%3A0.00044445870408613%2C%5C%22candidate_retrieval_source_map%5C%22%3A%7B%5C%2224843562218625057%5C%22%3A3001%7D%7D%22%2C%22ftmd_400706%22%3A%22111112l%22%7D"
|
||
|
|
product_info = FbMarketplaceProductInfo(config)
|
||
|
|
product_info.run_product_pipeline()
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
run_pipeline()
|