Shariar
/
raena-crawler-engine
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
raena-crawler-engine/shopee_crawler_engine
History
Shariar Imtiaz 28f584f829 added Hasaki crawler 2024-03-18 12:12:50 +04:00
..
Readme.md first commit 2024-01-24 17:05:07 +04:00
conf.json first commit 2024-01-24 17:05:07 +04:00
requirments.txt first commit 2024-01-24 17:05:07 +04:00
shopee_category_products.py first commit 2024-01-24 17:05:07 +04:00
shopee_crawler.py first commit 2024-01-24 17:05:07 +04:00
shopee_db_writer.py first commit 2024-01-24 17:05:07 +04:00
shopee_products.py added Hasaki crawler 2024-03-18 12:12:50 +04:00
shopee_sub_categories.py first commit 2024-01-24 17:05:07 +04:00

Readme.md

Run:

  1. Change config accourding to the crawler type.
  2. run "python shopee_crawler.py"

Config for Master:

config = { "crawler_name": "raena_crawler_enginer_shopee", "crawler_schema": "raena_spider_management", "category_tab": "rce_category", "tracker_tab": "crawler_tracker", "product_tab": "rce_product", "variant_tab": "rce_product_variant", "brand_tab": "rce_brand", "reseller_tab": "rce_reseller", "reseller_store_tab": "rce_reseller_store", "review_tab": "rce_ratings_reviews", "product_per_category": "136", "source_category": "11043145", "db_user": "crawler", "db_pass": "4Z063Zp9Aczv", "database": "raena_db", "db_host": "raen-prd-sg-aurora-pg-rds-cluster-instance-1.cd7qipz3esdx.ap-southeast-1.rds.amazonaws.com", "db_port": "5432", "crawler_main": "1", "crawler_slave_no": "" }

Config for Slave01:

config = { "crawler_name": "raena_crawler_enginer_shopee", "crawler_schema": "raena_spider_management", "category_tab": "rce_category", "tracker_tab": "crawler_tracker", "product_tab": "rce_product", "variant_tab": "rce_product_variant", "brand_tab": "rce_brand", "reseller_tab": "rce_reseller", "reseller_store_tab": "rce_reseller_store", "review_tab": "rce_ratings_reviews", "product_per_category": "136", "source_category": "11043145", "db_user": "crawler", "db_pass": "4Z063Zp9Aczv", "database": "raena_db", "db_host": "raen-prd-sg-aurora-pg-rds-cluster-instance-1.cd7qipz3esdx.ap-southeast-1.rds.amazonaws.com", "db_port": "5432", "crawler_main": "0", "crawler_slave_no": "1" }

Config for Slave02:

config = { "crawler_name": "raena_crawler_enginer_shopee", "crawler_schema": "raena_spider_management", "category_tab": "rce_category", "tracker_tab": "crawler_tracker", "product_tab": "rce_product", "variant_tab": "rce_product_variant", "brand_tab": "rce_brand", "reseller_tab": "rce_reseller", "reseller_store_tab": "rce_reseller_store", "review_tab": "rce_ratings_reviews", "product_per_category": "136", "source_category": "11043145", "db_user": "crawler", "db_pass": "4Z063Zp9Aczv", "database": "raena_db", "db_host": "raen-prd-sg-aurora-pg-rds-cluster-instance-1.cd7qipz3esdx.ap-southeast-1.rds.amazonaws.com", "db_port": "5432", "crawler_main": "0", "crawler_slave_no": "2" }