76 lines
2.2 KiB
Markdown
76 lines
2.2 KiB
Markdown
|
***Run:***
|
||
|
|
1. Change config accourding to the crawler type.
|
||
|
|
2. run "python shopee_crawler.py"
|
||
|
|
|
||
|
|
***Config for Master:***
|
||
|
|
|
||
|
|
config = {
|
||
|
|
"crawler_name": "raena_crawler_enginer_shopee",
|
||
|
|
"crawler_schema": "raena_spider_management",
|
||
|
|
"category_tab": "rce_category",
|
||
|
|
"tracker_tab": "crawler_tracker",
|
||
|
|
"product_tab": "rce_product",
|
||
|
|
"variant_tab": "rce_product_variant",
|
||
|
|
"brand_tab": "rce_brand",
|
||
|
|
"reseller_tab": "rce_reseller",
|
||
|
|
"reseller_store_tab": "rce_reseller_store",
|
||
|
|
"review_tab": "rce_ratings_reviews",
|
||
|
|
"product_per_category": "136",
|
||
|
|
"source_category": "11043145",
|
||
|
|
"db_user": "crawler",
|
||
|
|
"db_pass": "4Z063Zp9Aczv",
|
||
|
|
"database": "raena_db",
|
||
|
|
"db_host": "raen-prd-sg-aurora-pg-rds-cluster-instance-1.cd7qipz3esdx.ap-southeast-1.rds.amazonaws.com",
|
||
|
|
"db_port": "5432",
|
||
|
|
"crawler_main": "1",
|
||
|
|
"crawler_slave_no": ""
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
***Config for Slave01:***
|
||
|
|
|
||
|
|
config = {
|
||
|
|
"crawler_name": "raena_crawler_enginer_shopee",
|
||
|
|
"crawler_schema": "raena_spider_management",
|
||
|
|
"category_tab": "rce_category",
|
||
|
|
"tracker_tab": "crawler_tracker",
|
||
|
|
"product_tab": "rce_product",
|
||
|
|
"variant_tab": "rce_product_variant",
|
||
|
|
"brand_tab": "rce_brand",
|
||
|
|
"reseller_tab": "rce_reseller",
|
||
|
|
"reseller_store_tab": "rce_reseller_store",
|
||
|
|
"review_tab": "rce_ratings_reviews",
|
||
|
|
"product_per_category": "136",
|
||
|
|
"source_category": "11043145",
|
||
|
|
"db_user": "crawler",
|
||
|
|
"db_pass": "4Z063Zp9Aczv",
|
||
|
|
"database": "raena_db",
|
||
|
|
"db_host": "raen-prd-sg-aurora-pg-rds-cluster-instance-1.cd7qipz3esdx.ap-southeast-1.rds.amazonaws.com",
|
||
|
|
"db_port": "5432",
|
||
|
|
"crawler_main": "0",
|
||
|
|
"crawler_slave_no": "1"
|
||
|
|
}
|
||
|
|
|
||
|
|
***Config for Slave02:***
|
||
|
|
|
||
|
|
config = {
|
||
|
|
"crawler_name": "raena_crawler_enginer_shopee",
|
||
|
|
"crawler_schema": "raena_spider_management",
|
||
|
|
"category_tab": "rce_category",
|
||
|
|
"tracker_tab": "crawler_tracker",
|
||
|
|
"product_tab": "rce_product",
|
||
|
|
"variant_tab": "rce_product_variant",
|
||
|
|
"brand_tab": "rce_brand",
|
||
|
|
"reseller_tab": "rce_reseller",
|
||
|
|
"reseller_store_tab": "rce_reseller_store",
|
||
|
|
"review_tab": "rce_ratings_reviews",
|
||
|
|
"product_per_category": "136",
|
||
|
|
"source_category": "11043145",
|
||
|
|
"db_user": "crawler",
|
||
|
|
"db_pass": "4Z063Zp9Aczv",
|
||
|
|
"database": "raena_db",
|
||
|
|
"db_host": "raen-prd-sg-aurora-pg-rds-cluster-instance-1.cd7qipz3esdx.ap-southeast-1.rds.amazonaws.com",
|
||
|
|
"db_port": "5432",
|
||
|
|
"crawler_main": "0",
|
||
|
|
"crawler_slave_no": "2"
|
||
|
|
}
|