Shariar
/
raena-crawler-engine
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
raena-crawler-engine/shopee_crawler_engine/Readme.md

2.2 KiB
Raw Blame History

Run:

  1. Change config accourding to the crawler type.
  2. run "python shopee_crawler.py"

Config for Master:

config = { "crawler_name": "raena_crawler_enginer_shopee", "crawler_schema": "raena_spider_management", "category_tab": "rce_category", "tracker_tab": "crawler_tracker", "product_tab": "rce_product", "variant_tab": "rce_product_variant", "brand_tab": "rce_brand", "reseller_tab": "rce_reseller", "reseller_store_tab": "rce_reseller_store", "review_tab": "rce_ratings_reviews", "product_per_category": "136", "source_category": "11043145", "db_user": "crawler", "db_pass": "4Z063Zp9Aczv", "database": "raena_db", "db_host": "raen-prd-sg-aurora-pg-rds-cluster-instance-1.cd7qipz3esdx.ap-southeast-1.rds.amazonaws.com", "db_port": "5432", "crawler_main": "1", "crawler_slave_no": "" }

Config for Slave01:

config = { "crawler_name": "raena_crawler_enginer_shopee", "crawler_schema": "raena_spider_management", "category_tab": "rce_category", "tracker_tab": "crawler_tracker", "product_tab": "rce_product", "variant_tab": "rce_product_variant", "brand_tab": "rce_brand", "reseller_tab": "rce_reseller", "reseller_store_tab": "rce_reseller_store", "review_tab": "rce_ratings_reviews", "product_per_category": "136", "source_category": "11043145", "db_user": "crawler", "db_pass": "4Z063Zp9Aczv", "database": "raena_db", "db_host": "raen-prd-sg-aurora-pg-rds-cluster-instance-1.cd7qipz3esdx.ap-southeast-1.rds.amazonaws.com", "db_port": "5432", "crawler_main": "0", "crawler_slave_no": "1" }

Config for Slave02:

config = { "crawler_name": "raena_crawler_enginer_shopee", "crawler_schema": "raena_spider_management", "category_tab": "rce_category", "tracker_tab": "crawler_tracker", "product_tab": "rce_product", "variant_tab": "rce_product_variant", "brand_tab": "rce_brand", "reseller_tab": "rce_reseller", "reseller_store_tab": "rce_reseller_store", "review_tab": "rce_ratings_reviews", "product_per_category": "136", "source_category": "11043145", "db_user": "crawler", "db_pass": "4Z063Zp9Aczv", "database": "raena_db", "db_host": "raen-prd-sg-aurora-pg-rds-cluster-instance-1.cd7qipz3esdx.ap-southeast-1.rds.amazonaws.com", "db_port": "5432", "crawler_main": "0", "crawler_slave_no": "2" }