Merge pull request code-dot-org#24875 from code-dot-org/rp_redshift_updates

Rp redshift updates

Author: marybarnes37

aws/redshift/tables/csf_workshop_attendance.sql

@@ -140,7 +140,7 @@ UNION ALL
          0 as not_attended
     FROM dashboard_production.followers f
     JOIN dashboard_production.sections se 
-       ON se.id = f.section_id AND se.section_type = 'csf_workshop' 
+       ON se.id = f.section_id 
     JOIN analysis.training_school_years sy ON se.created_at BETWEEN sy.started_at AND sy.ended_at
     JOIN section_state_zip ssz 
       ON ssz.id = se.id

aws/redshift/tables/regional_partner_stats_csf.sql

@@ -1,3 +1,7 @@
+-- PLAN for adding CSF data
+-- add data on the facilitator, district sponsorship, workshop size, day of week, time of year
+-- create separate csf_facilitator csf_workshops table to connect to this one?
+
 --CHANGES
 -- addition of school years beyond 2017 --  includes teaching data from the same year they were trained and any subsequent years 
 -- removed 'location'
@@ -17,42 +21,6 @@ drop table if exists analysis_pii.regional_partner_stats_csf;
 create table analysis_pii.regional_partner_stats_csf AS
 
 with 
-csf_teachers_trained_temp as 
-(
-  select distinct
-  user_id, -- multiple entries per person (if they attended multiple workshops)
-  u.studio_p erson_id,
-  'CS Fundamentals'::varchar as course,
-  school_year as school_year,
-  regional_partner as regional_partner,
-  regional_partner_id as regional_partner_id, 
-  trained_at as trained_at, -- this is the 'min' date from the teachers trained at table (the first time they were trained)
-  workshop_date as workshop_date -- this is the date of the workshop (and the source of multiple entries per person) 
-  from
-  (
-    SELECT  
-        ctt.user_id, 
-        ctt.trained_at,
-        pds.start as workshop_date,
-        regional_partner_id::int, 
-        rp.name::varchar as regional_partner
-        FROM 
-        analysis.csf_teachers_trained ctt
-        LEFT JOIN dashboard_production_pii.pd_enrollments pde
-          ON pde.user_id = ctt.user_id
-         LEFT JOIN dashboard_production_pii.pd_attendances pda 
-           ON pda.pd_enrollment_id = pde.id
-         LEFT JOIN dashboard_production_pii.pd_workshops pdw 
-           ON pdw.id = pde.pd_workshop_id
-           AND course = 'CS Fundamentals'
-         LEFT JOIN dashboard_production_pii.pd_sessions pds 
-           ON pds.pd_workshop_id = pdw.id
-        LEFT JOIN dashboard_production_pii.regional_partners rp  
-           ON pdw.regional_partner_id = rp.id     
-  ) csf_train
-  JOIN analysis.training_school_years sy on csf_train.trained_at between sy.started_at and sy.ended_at
-  JOIN dashboard_production.users u on u.id = csf_train.user_id
-),
 completed as
 (
   select 
@@ -71,6 +39,24 @@ select
     script_name
   from analysis.csf_started_teachers 
 ),
+implementation_365 as
+(select 
+tt.user_id,
+date_part(month, trained_at) month_trained,
+date_part(dayofweek, trained_at) day_of_week, 
+min(datediff(day, tt.trained_at, st.started_at)) as days_to_start,
+min(datediff(day, tt.trained_at, ct.completed_at)) as days_to_complete,
+CASE WHEN days_to_start < 0 then 1 else 0 end as started_before_training,
+CASE WHEN days_to_complete < 0 then 1 else 0 end as completed_before_training,
+CASE WHEN days_to_start <= 365  and started_before_training = 0 then 1 else 0 end as started_365,
+CASE WHEN days_to_complete <= 365 and completed_before_training = 0 then 1 else 0 end as completed_365,
+CASE WHEN days_to_start <= 365  then 1 else 0 end as started_365_or_before,
+CASE WHEN days_to_complete <= 365  then 1 else 0 end as completed_365_or_before
+from csf_teachers_trained tt
+left join csf_started_teachers st on st.user_id = tt.user_id  
+left join csf_completed_teachers ct on ct.user_id = tt.user_id  
+group by 1, 2, 3
+),
 pd_enrollments_with_year as
 ( 
   select pd_workshop_id, first_name, last_name, email, user_id, school_year
@@ -90,16 +76,25 @@ pd_facilitators as
   JOIN dashboard_production_pii.users u2
         ON  pwf.user_id = u2.id
   group by 1
-
 )
   SELECT distinct 
          d.user_id,
-         d.studio_person_id,
+         u.studio_person_id,
          FIRST_VALUE(pde.first_name) OVER (PARTITION BY d.user_id  ORDER BY (CASE WHEN  pde.first_name IS NULL THEN 1 ELSE 2 END), pde.pd_workshop_id DESC ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING ) as first_name,
          FIRST_VALUE(pde.last_name) OVER (PARTITION BY d.user_id ORDER BY (CASE WHEN  pde.last_name IS NULL THEN 1 ELSE 2 END), pde.pd_workshop_id DESC ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING ) as last_name,
          FIRST_VALUE(pde.email) OVER (PARTITION BY d.user_id ORDER BY (CASE WHEN  pde.email IS NULL THEN 1 ELSE 2 END), pde.pd_workshop_id DESC ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING ) as email,
-         d.course,
-         d.school_year as school_year_trained,
+         'CS Fundamentals'::varchar as course,
+         sy.school_year as school_year_trained,
+         month_trained,
+         day_of_week, 
+         days_to_start,
+         days_to_complete,
+         started_before_training,
+         completed_before_training,
+         started_365,
+         completed_365,
+         started_365_or_before,
+         completed_365_or_before,
          s.school_year as school_year_taught,
          s.script_name,
          CASE WHEN rp.name is null THEN 'No Partner' ELSE rp.name END as regional_partner_name,        
@@ -117,7 +112,7 @@ pd_facilitators as
          CASE WHEN csfa.subject is null THEN 'Intro Workshop' else csfa.subject END as subject,
          CASE WHEN csfa.trained_by_regional_partner is null then 0 else csfa.trained_by_regional_partner END as trained_by_regional_partner,
          d.trained_at as trained_at,
-         coalesce(d.workshop_date, csfa.workshop_date, d.trained_at)  as workshop_date, 
+         coalesce(csfa.workshop_date, d.trained_at)  as workshop_date, 
          extract(month from csfa.workshop_date)::varchar(16) || '/'::varchar(2) || extract(day from csfa.workshop_date)::varchar(16) || '/'::varchar(2) || extract(year from csfa.workshop_date)::varchar(16) || ', id:'::varchar(2) || csfa.workshop_id::varchar(16)  as workshop_id_year,
          pwf.facilitator_names,
          -- started and completed
@@ -134,7 +129,8 @@ pd_facilitators as
           -- student gender
           sa.students_female as students_female_total,
           sa.students_gender as students_gender_total
-  FROM csf_teachers_trained_temp d 
+  FROM csf_teachers_trained d 
+  JOIN training_school_years sy on d.trained_at between sy.started_at and sy.ended_at
 -- school info
   LEFT JOIN dashboard_production_pii.users u  -- users needed to get school_info_id
          ON d.user_id = u.id
@@ -146,9 +142,7 @@ pd_facilitators as
  -- LEFT JOIN analysis.csf_workshop_attendance csfa -- functions mostly to get the regional partner's location info and to decide whether the person was 'trained_by_partner'
   LEFT JOIN analysis.csf_workshop_attendance csfa   
         ON csfa.user_id = d.user_id
-        AND csfa.course = d.course
-        AND csfa.school_year = d.school_year
-        -- AND trunc(csfa.workshop_date) = d.trained_at -- this must be the PROBLEM!! investigat it 
+        AND csfa.school_year = sy.school_year
         AND csfa.not_attended = 0 
 --pii tables (regional partner names, person names, emails, locations)
   LEFT JOIN pd_facilitators pwf
@@ -157,15 +151,17 @@ pd_facilitators as
        ON csfa.regional_partner_id = rp.id 
   LEFT JOIN pd_enrollments_with_year pde   -- only join pde if they are are trained by regional partner 
         ON pde.user_id = d.user_id
-        AND pde.school_year = d.school_year 
+        AND pde.school_year = sy.school_year 
 -- analysis tables 
   LEFT JOIN started s
        ON s.user_id = d.user_id
-      AND s.school_year >= d.school_year 
+      AND s.school_year >= sy.school_year 
   LEFT JOIN completed c
          ON c.user_id = d.user_id
          AND c.script_name = s.script_name
          AND c.school_year = s.school_year
+  LEFT JOIN implementation_365 i
+        ON i.user_id = d.user_id
   LEFT JOIN analysis.teacher_most_progress_csf tmp
          ON tmp.user_id = d.user_id
          and tmp.script_name = s.script_name