raena_analytics_scripts/redshift/retention_dashboard.sh

460 lines
26 KiB
Bash
Raw Normal View History

2024-06-24 12:26:08 +00:00
!/bin/bash
echo -e " \n----------- ACCEPTING NUMBER OF DAYS BEFORE THE RUN DATE FOR WHICH THE REPORT IS TO BE RUN --------------\n"
backDay=$1
echo $backDay
echo -e " \n------------- DATE IN THE REQUIRED FORMAT --------------\n"
reportDate=$(date -d"$backDay day ago" "+%Y-%m-%d")
echo 'reportDate'=$reportDate
echo "
DROP TABLE IF EXISTS raena_analytics.OM_Reseller_Wise_latest_tier;
CREATE TABLE raena_analytics.OM_Reseller_Wise_latest_tier AS
SELECT mobile,
tier from
( SELECT mobile,
tier,
row_number() over (partition BY mobile ORDER BY created_date DESC) AS R
FROM
( SELECT DISTINCT replace(json_extract_path_text(o.reseller_info,'mobile',true),'+','') AS mobile,
json_extract_path_text(o.reseller_info,'tierName',true) AS tier,
cast(created_at AS date) AS created_date
FROM raena_order_management.order o) A ) B
WHERE R=1
AND mobile is NOT NULL;
DROP TABLE IF EXISTS raena_analytics.Retention_new_filters_install;
CREATE TABLE raena_analytics.Retention_new_filters_install AS
SELECT DISTINCT TO_CHAR(o.created_at, 'Month') AS MONTH,
cast(date_part('year',o.created_at) AS varchar) AS YEAR,
CASE
WHEN shipping_to='Reseller' THEN 'Reseller'
ELSE 'Dropshipper'
END AS customer_type,
order_placed_by,
CASE
WHEN order_placed_by='admin' THEN 'admin'
ELSE marketplace
END AS Sales_channel,
provider,
c.channel_name AS channel_name,
replace(json_extract_path_text(o.reseller_info,'mobile',true),'+','') mobile,
verification_status,
tier
FROM raena_user_management.user u
LEFT JOIN raena_order_management.order o ON replace(json_extract_path_text(o.reseller_info,'mobile',true),'+','')=replace(u.mobile,'+','')
LEFT JOIN raena_analytics.OM_channel c ON cast(c.id AS text)=cast(o.channel_id AS text)
LEFT JOIN
(SELECT mobile,
tier
FROM
(SELECT mobile,
tier,
row_number() over (partition BY mobile
ORDER BY created_date DESC) AS R
FROM
(SELECT DISTINCT replace(json_extract_path_text(o.reseller_info,'mobile',true),'+','') AS mobile,
json_extract_path_text(o.reseller_info,'tierName',true) AS tier,
cast(created_at AS date) AS created_date
FROM raena_order_management.order o)A) B
WHERE R=1
AND mobile IS NOT NULL) t ON u.mobile=t.mobile;
DROP TABLE IF EXISTS raena_analytics.DAU_RETENTION;
CREATE TABLE raena_analytics.DAU_RETENTION AS
SELECT *
FROM
(SELECT DISTINCT base_1.mobile,
cast(date_part('YEAR',base_1.created_date) AS varchar) AS conversion_year,
TO_CHAR(base_1.created_date, 'Month') AS conversion_month,
date_part('MONTH',base_1.created_date) AS month_sort,
base_1.created_date,
base_1.bucket,
case when datediff(month,base_1.created_date,base_2.created_date)=0 then base_1.mobile end as M0,
case when datediff(month,base_1.created_date,base_3.created_date)=1 then base_1.mobile end as M1,
case when datediff(month,base_1.created_date,base_3.created_date)=2 then base_1.mobile end as M2,
case when datediff(month,base_1.created_date,base_3.created_date)=3 then base_1.mobile end as M3,
case when datediff(month,base_1.created_date,base_3.created_date)=4 then base_1.mobile end as M4,
case when datediff(month,base_1.created_date,base_3.created_date)=5 then base_1.mobile end as M5,
case when datediff(month,base_1.created_date,base_3.created_date)=6 then base_1.mobile end as M6,
case when datediff(month,base_1.created_date,base_3.created_date)>6 then base_1.mobile end as GT_M6
FROM
(SELECT *,
CASE WHEN payment_amount < 2000000 THEN '<2M' WHEN payment_amount BETWEEN 2000001 AND 10000000 THEN '2-10M' WHEN payment_amount>10000000 THEN '+10M' END AS Bucket
FROM
(SELECT DISTINCT replace(json_extract_path_text(o.reseller_info,'mobile',true),'+','') AS mobile,
cast(created_at AS Date) created_date,
payment_amount,
row_number() over(partition BY replace(json_extract_path_text(o.reseller_info,'mobile',true),'+','')
ORDER BY cast(created_at AS Date)) created_rnk
FROM raena_order_management.order o
WHERE payment_status = 'Paid' or id like 'PL%') a
WHERE a.created_rnk=1) base_1
INNER JOIN
(SELECT mobile,
date_trunc('month',created_date) AS created_date,
sum(payment_amount) AS payment_amount
FROM
(SELECT DISTINCT replace(json_extract_path_text(o.reseller_info,'mobile',true),'+','') AS mobile,
cast(created_at AS Date) created_date,
payment_amount,
row_number() over(partition BY replace(json_extract_path_text(o.reseller_info,'mobile',true),'+',''), date_part('YEAR',cast(created_at AS Date)),date_part('MONTH',cast(created_at AS Date))
ORDER BY date_part('YEAR',cast(created_at AS Date)),date_part('MONTH',cast(created_at AS Date))) created_rnk
FROM raena_order_management.order o
WHERE payment_status = 'Paid' or id like 'PL%') b
GROUP BY 1,
2) base_2 ON base_1.mobile=base_2.mobile
LEFT JOIN
(SELECT DISTINCT date_trunc('month',created_date) AS created_date,
active_user
FROM
(SELECT DISTINCT cast(to_date(substring(ts,1,8),'yyyymmdd') AS date) created_date,
CASE WHEN profile_phone <> '' THEN profile_phone ELSE profile_objectid END Active_user,
dense_rank() over(partition BY (CASE WHEN profile_phone <> '' THEN profile_phone ELSE profile_objectid END)
ORDER BY date_part('year',cast(to_date(substring(ts,1,8),'yyyymmdd') AS date)),date_part('month',cast(to_date(substring(ts,1,8),'yyyymmdd') AS date))) created_rnk_2
FROM analytics.clevertap.app_launched al
WHERE eventname= 'App Launched'
AND profile_phone <> '' ) c ) base_3 ON base_1.mobile=base_3.active_user) bucket_final;
DROP TABLE IF EXISTS raena_analytics.INSTALLBASE_REVENUE_RETENTION;
CREATE TABLE raena_analytics.INSTALLBASE_REVENUE_RETENTION AS
SELECT *
FROM
(SELECT cast(acquisition_year AS varchar) AS acquisition_year,
acquisition_month,
month_sort,
bucket_check.bucket,
install_base.Active_user,
case when datediff(month,install_base.created_date,conversion_base.created_date)=0 then conversion_base.payment_amount end as M0,
case when datediff(month,install_base.created_date,conversion_base.created_date)=1 then conversion_base.payment_amount end as M1,
case when datediff(month,install_base.created_date,conversion_base.created_date)=2 then conversion_base.payment_amount end as M2,
case when datediff(month,install_base.created_date,conversion_base.created_date)=3 then conversion_base.payment_amount end as M3,
case when datediff(month,install_base.created_date,conversion_base.created_date)=4 then conversion_base.payment_amount end as M4,
case when datediff(month,install_base.created_date,conversion_base.created_date)=5 then conversion_base.payment_amount end as M5,
case when datediff(month,install_base.created_date,conversion_base.created_date)=6 then conversion_base.payment_amount end as M6,
case when datediff(month,install_base.created_date,conversion_base.created_date)>6 then conversion_base.payment_amount end as GT_M6
FROM
(SELECT DISTINCT base.Active_user,
date_part('YEAR',base.created_date) AS acquisition_year,
TO_CHAR(base.created_date, 'Month') AS acquisition_month,
date_part('MONTH',base.created_date) AS month_sort,
base.created_date
FROM
(SELECT DISTINCT cast(to_date(substring(ts,1,8),'yyyymmdd') AS date) AS created_date,
date_part('month',cast(to_date(substring(ts,1,8),'yyyymmdd') AS date)) month_sort,
CASE WHEN profile_phone <> '' THEN profile_phone ELSE profile_objectid END Active_user,
dense_rank() over(partition BY (CASE WHEN profile_phone <> '' THEN profile_phone ELSE profile_objectid END)
ORDER BY cast(to_date(substring(ts,1,8),'yyyymmdd') AS date)) created_rnk
FROM analytics.clevertap.app_launched al
WHERE eventname= 'App Launched'
) base
WHERE created_rnk=1) install_base
INNER JOIN
(SELECT mobile,
created_date,
payment_amount
FROM
(SELECT mobile,
created_date,
CASE WHEN payment_amount < 2000000 THEN '<2M'
WHEN payment_amount BETWEEN 2000001 AND 10000000 THEN '2-10M'
WHEN payment_amount>10000000 THEN '+10M' END AS Bucket,
payment_amount,
row_number() over (partition BY mobile ORDER BY created_date) AS rnk
FROM
(SELECT DISTINCT mobile,
date_trunc('month',created_date) AS created_date,
sum(payment_amount) AS payment_amount
FROM
(SELECT DISTINCT replace(json_extract_path_text(o.reseller_info,'mobile',true),'+','') AS mobile,
cast(created_at AS Date) created_date,
payment_amount
FROM raena_order_management.order o
WHERE payment_status = 'Paid' or id like 'PL%') A
GROUP BY 1,
2)B) C) conversion_base ON install_base.active_user=conversion_base.mobile
INNER JOIN
(SELECT mobile,
CASE WHEN payment_amount < 2000000 THEN '<2M'
WHEN payment_amount BETWEEN 2000001 AND 10000000 THEN '2-10M'
WHEN payment_amount>10000000 THEN '+10M' END AS Bucket
FROM
(SELECT replace(json_extract_path_text(o.reseller_info,'mobile',true),'+','') AS mobile,
sum(payment_amount) AS payment_amount
FROM raena_order_management.order o
WHERE payment_status = 'Paid' or id like 'PL%'
GROUP BY 1)A) bucket_check ON install_base.active_user=bucket_check.mobile) a;
DROP TABLE IF EXISTS raena_analytics.INSTALLBASE_RESELLER_RETENTION;
CREATE TABLE raena_analytics.INSTALLBASE_RESELLER_RETENTION AS
SELECT *
FROM
(SELECT cast(acquisition_year AS varchar) AS acquisition_year,
acquisition_month,
month_sort,
bucket_check.bucket,
install_base.Active_user,
case when datediff(month,install_base.created_date,conversion_base.created_date)=0 then install_base.Active_user end as M0,
case when datediff(month,install_base.created_date,conversion_base.created_date)=1 then install_base.Active_user end as M1,
case when datediff(month,install_base.created_date,conversion_base.created_date)=2 then install_base.Active_user end as M2,
case when datediff(month,install_base.created_date,conversion_base.created_date)=3 then install_base.Active_user end as M3,
case when datediff(month,install_base.created_date,conversion_base.created_date)=4 then install_base.Active_user end as M4,
case when datediff(month,install_base.created_date,conversion_base.created_date)=5 then install_base.Active_user end as M5,
case when datediff(month,install_base.created_date,conversion_base.created_date)=6 then install_base.Active_user end as M6,
case when datediff(month,install_base.created_date,conversion_base.created_date)>6 then install_base.Active_user end as GT_M6
FROM
(SELECT DISTINCT base.Active_user,
date_part('YEAR',base.created_date) AS acquisition_year,
TO_CHAR(base.created_date, 'Month') AS acquisition_month,
date_part('MONTH',base.created_date) AS month_sort,
base.created_date
FROM
(SELECT DISTINCT cast(to_date(substring(ts,1,8),'yyyymmdd') AS date) AS created_date,
date_part('month',cast(to_date(substring(ts,1,8),'yyyymmdd') AS date)) month_sort,
CASE WHEN profile_phone <> '' THEN profile_phone ELSE profile_objectid END Active_user,
dense_rank() over(partition BY (CASE WHEN profile_phone <> '' THEN profile_phone ELSE profile_objectid END)
ORDER BY cast(to_date(substring(ts,1,8),'yyyymmdd') AS date)) created_rnk
FROM analytics.clevertap.app_launched al
WHERE eventname= 'App Launched'
) base
WHERE created_rnk=1) install_base
INNER JOIN
(SELECT mobile,
created_date
FROM
(SELECT mobile,
created_date,
CASE WHEN payment_amount < 2000000 THEN '<2M'
WHEN payment_amount BETWEEN 2000001 AND 10000000 THEN '2-10M'
WHEN payment_amount>10000000 THEN '+10M' END AS Bucket,
row_number() over (partition BY mobile ORDER BY created_date) AS rnk
FROM
(SELECT DISTINCT mobile,
date_trunc('month',created_date) AS created_date,
sum(payment_amount) AS payment_amount
FROM
(SELECT DISTINCT replace(json_extract_path_text(o.reseller_info,'mobile',true),'+','') AS mobile,
cast(created_at AS Date) created_date,
payment_amount
FROM raena_order_management.order o
WHERE payment_status = 'Paid' or id like 'PL%')A
GROUP BY 1,
2)B)C) conversion_base ON install_base.active_user=conversion_base.mobile
INNER JOIN
(SELECT mobile,
CASE WHEN payment_amount < 2000000 THEN '<2M'
WHEN payment_amount BETWEEN 2000001 AND 10000000 THEN '2-10M'
WHEN payment_amount>10000000 THEN '+10M' END AS Bucket
FROM
(SELECT replace(json_extract_path_text(o.reseller_info,'mobile',true),'+','') AS mobile,
sum(payment_amount) AS payment_amount
FROM raena_order_management.order o
WHERE payment_status = 'Paid' or id like 'PL%'
GROUP BY 1)A) bucket_check ON install_base.active_user=bucket_check.mobile) a;
DROP TABLE IF EXISTS raena_analytics.DAU_RETENTION_BASIS_INSTALL_COHORT;
CREATE TABLE raena_analytics.DAU_RETENTION_BASIS_INSTALL_COHORT AS
SELECT *
FROM
(SELECT install_base.*,
bucket.bucket
FROM
(SELECT DISTINCT a.Active_user,
date_part('YEAR',a.created_date) AS acquisition_year,
TO_CHAR(a.created_date, 'Month') AS acquisition_month,
date_part('MONTH',a.created_date) AS month_sort,
a.created_date,
case when datediff(month,a.created_date,b.created_date)=0 then a.Active_user end as M0,
case when datediff(month,a.created_date,b.created_date)=1 then a.Active_user end as M1,
case when datediff(month,a.created_date,b.created_date)=2 then a.Active_user end as M2,
case when datediff(month,a.created_date,b.created_date)=3 then a.Active_user end as M3,
case when datediff(month,a.created_date,b.created_date)=4 then a.Active_user end as M4,
case when datediff(month,a.created_date,b.created_date)=5 then a.Active_user end as M5,
case when datediff(month,a.created_date,b.created_date)=6 then a.Active_user end as M6,
case when datediff(month,a.created_date,b.created_date)>6 then a.Active_user end as GT_M6
FROM
(SELECT *
FROM
(SELECT DISTINCT cast(to_date(substring(ts,1,8),'yyyymmdd') AS date) AS created_date,
date_part('month',cast(to_date(substring(ts,1,8),'yyyymmdd') AS date)) month_sort,
CASE WHEN profile_phone <> '' THEN profile_phone ELSE profile_objectid END Active_user,
dense_rank() over(partition BY (CASE WHEN profile_phone <> '' THEN profile_phone ELSE profile_objectid END)
ORDER BY cast(to_date(substring(ts,1,8),'yyyymmdd') AS date)) created_rnk
FROM analytics.clevertap.app_launched al
WHERE eventname= 'App Launched') base
WHERE created_rnk=1) a
INNER JOIN
(SELECT DISTINCT cast(to_date(substring(ts,1,8),'yyyymmdd') AS date) created_date,
CASE WHEN profile_phone <> '' THEN profile_phone ELSE profile_objectid END Active_user,
dense_rank() over(partition BY (CASE WHEN profile_phone <> '' THEN profile_phone ELSE profile_objectid END)
ORDER BY date_part('year',cast(to_date(substring(ts,1,8),'yyyymmdd') AS date)),date_part('month',cast(to_date(substring(ts,1,8),'yyyymmdd') AS date))) created_rnk_2
FROM analytics.clevertap.app_launched al
WHERE eventname= 'App Launched') b ON a.Active_user=b.Active_user) install_base
LEFT JOIN
(SELECT conversion_base.mobile,
date_part('YEAR',created_date) AS acquistion_year,
TO_CHAR(created_date,'month') AS acquistion_month,
bucket_check.bucket
FROM
(SELECT mobile,
created_date
FROM
(SELECT mobile,
created_date,
CASE WHEN payment_amount < 2000000 THEN '<2M'
WHEN payment_amount BETWEEN 2000001 AND 10000000 THEN '2-10M'
WHEN payment_amount>10000000 THEN '+10M' END AS Bucket,
row_number() over (partition BY mobile ORDER BY created_date) AS rnk
FROM
(SELECT DISTINCT mobile,
date_trunc('month',created_date) AS created_date,
sum(payment_amount) AS payment_amount
FROM
(SELECT DISTINCT replace(json_extract_path_text(o.reseller_info,'mobile',true),'+','') AS mobile,
cast(created_at AS Date) created_date,
payment_amount
FROM raena_order_management.order o
WHERE payment_status = 'Paid' or id like 'PL%') b
GROUP BY 1,
2) c) a
WHERE a.rnk=1)conversion_base
INNER JOIN
(SELECT mobile,
CASE WHEN payment_amount < 2000000 THEN '<2M'
WHEN payment_amount BETWEEN 2000001 AND 10000000 THEN '2-10M'
WHEN payment_amount>10000000 THEN '+10M' END AS Bucket
FROM
(SELECT replace(json_extract_path_text(o.reseller_info,'mobile',true),'+','') AS mobile,
sum(payment_amount) AS payment_amount
FROM raena_order_management.order o
WHERE payment_status = 'Paid' or id like 'PL%'
GROUP BY 1) A ) bucket_check ON conversion_base.mobile=bucket_check.mobile) bucket ON install_base.active_user=bucket.mobile
AND date_part('YEAR',install_base.created_date)=bucket.acquistion_year
AND TO_CHAR(install_base.created_date,'month')=bucket.acquistion_month) bucket_final;
drop table if exists raena_analytics.OMREVENUE_RETENTION;
CREATE TABLE raena_analytics.OMREVENUE_RETENTION AS
SELECT *
FROM
(SELECT DISTINCT base_1.mobile,
cast(date_part('YEAR',base_1.created_date) AS varchar) AS acquisition_year,
TO_CHAR(base_1.created_date, 'Month') AS acquisition_month,
date_part('MONTH',base_1.created_date) AS month_sort,
base_1.created_date,
base_1.bucket,
case when datediff(month,base_1.created_date,base_2.created_date)=0 then base_2.payment_amount end as M0,
case when datediff(month,base_1.created_date,base_2.created_date)=1 then base_2.payment_amount end as M1,
case when datediff(month,base_1.created_date,base_2.created_date)=2 then base_2.payment_amount end as M2,
case when datediff(month,base_1.created_date,base_2.created_date)=3 then base_2.payment_amount end as M3,
case when datediff(month,base_1.created_date,base_2.created_date)=4 then base_2.payment_amount end as M4,
case when datediff(month,base_1.created_date,base_2.created_date)=5 then base_2.payment_amount end as M5,
case when datediff(month,base_1.created_date,base_2.created_date)=6 then base_2.payment_amount end as M6,
case when datediff(month,base_1.created_date,base_2.created_date)>6 then base_2.payment_amount end as GT_M6
From
(SELECT *,
CASE WHEN payment_amount < 2000000 THEN '<2M'
WHEN payment_amount BETWEEN 2000001 AND 10000000 THEN '2-10M'
WHEN payment_amount>10000000 THEN '+10M' END AS Bucket
FROM
(SELECT DISTINCT replace(json_extract_path_text(o.reseller_info,'mobile',true),'+','') AS mobile,
cast(created_at AS Date) created_date,
payment_amount,
row_number() over(partition BY replace(json_extract_path_text(o.reseller_info,'mobile',true),'+','')
ORDER BY cast(created_at AS Date)) created_rnk
FROM raena_order_management.order o
WHERE payment_status = 'Paid' or id like 'PL%') a
WHERE a.created_rnk=1) base_1
INNER JOIN
(SELECT mobile,
date_trunc('month',created_date) AS created_date,
sum(payment_amount) AS payment_amount
FROM
(SELECT DISTINCT replace(json_extract_path_text(o.reseller_info,'mobile',true),'+','') AS mobile,
cast(created_at AS Date) created_date,
payment_amount,
row_number() over(partition BY replace(json_extract_path_text(o.reseller_info,'mobile',true),'+',''), date_part('YEAR',cast(created_at AS Date)),date_part('MONTH',cast(created_at AS Date))
ORDER BY date_part('YEAR',cast(created_at AS Date)),date_part('MONTH',cast(created_at AS Date))) created_rnk
FROM raena_order_management.order o
WHERE payment_status = 'Paid' or id like 'PL%') A
GROUP BY 1,
2) base_2 ON base_1.mobile=base_2.mobile) bucket_final;
DROP TABLE IF EXISTS raena_analytics.RESELLER_RETENTION;
CREATE TABLE raena_analytics.RESELLER_RETENTION AS
SELECT *
FROM
(SELECT DISTINCT base_1.mobile,
cast(date_part('YEAR',base_1.created_date) AS varchar) AS acquisition_year,
TO_CHAR(base_1.created_date, 'Month') AS acquisition_month,
date_part('MONTH',base_1.created_date) AS month_sort,
base_1.created_date,
base_1.bucket,
case when datediff(month,base_1.created_date,base_2.created_date)=0 then base_2.payment_amount end as M0,
case when datediff(month,base_1.created_date,base_2.created_date)=1 then base_2.payment_amount end as M1,
case when datediff(month,base_1.created_date,base_2.created_date)=2 then base_2.payment_amount end as M2,
case when datediff(month,base_1.created_date,base_2.created_date)=3 then base_2.payment_amount end as M3,
case when datediff(month,base_1.created_date,base_2.created_date)=4 then base_2.payment_amount end as M4,
case when datediff(month,base_1.created_date,base_2.created_date)=5 then base_2.payment_amount end as M5,
case when datediff(month,base_1.created_date,base_2.created_date)=6 then base_2.payment_amount end as M6,
case when datediff(month,base_1.created_date,base_2.created_date)>6 then base_2.payment_amount end as GT_M6
FROM
(SELECT *,
CASE WHEN payment_amount < 2000000 THEN '<2M'
WHEN payment_amount BETWEEN 2000001 AND 10000000 THEN '2-10M'
WHEN payment_amount>10000000 THEN '+10M' END AS Bucket
FROM
(SELECT DISTINCT replace(json_extract_path_text(o.reseller_info,'mobile',true),'+','') AS mobile,
cast(created_at AS Date) created_date,
payment_amount,
row_number() over(partition BY replace(json_extract_path_text(o.reseller_info,'mobile',true),'+','')
ORDER BY cast(created_at AS Date)) created_rnk
FROM raena_order_management.order o
WHERE payment_status = 'Paid' or id like 'PL%') a
WHERE a.created_rnk=1) base_1
INNER JOIN
(SELECT mobile,
date_trunc('month',created_date) AS created_date,
sum(payment_amount) AS payment_amount
FROM
(SELECT DISTINCT replace(json_extract_path_text(o.reseller_info,'mobile',true),'+','') AS mobile,
cast(created_at AS Date) created_date,
payment_amount,
row_number() over(partition BY replace(json_extract_path_text(o.reseller_info,'mobile',true),'+',''), date_part('YEAR',cast(created_at AS Date)),date_part('MONTH',cast(created_at AS Date))
ORDER BY date_part('YEAR',cast(created_at AS Date)),date_part('MONTH',cast(created_at AS Date))) created_rnk
FROM raena_order_management.order o
WHERE payment_status = 'Paid' or id like 'PL%')A
GROUP BY 1,
2) base_2 ON base_1.mobile=base_2.mobile) bucket_final;
" > /home/ec2-user/cronjob/redshift/sql_code/retention_code_etl.sql
psql "host=redshift-cluster-1.cdqj58hfx4p7.ap-southeast-1.redshift.amazonaws.com user=dbadmin dbname=analytics port=5439 password=5qCif6eyY3Kmg4z" -f /home/ec2-user/cronjob/redshift/sql_code/retention_code_etl.sql