WWC standards – My Statistical tools

December 30, 2022

R code for deriving Hedge's g (or Cox Index)

#What works clearinghouse version of standardized program difference
#Hedges's g and Cox Index
#See page 15 and 16
#https://ies.ed.gov/ncee/wwc/Docs/referenceresources/WWC_Procedures_Handbook_V4_1_Draft.pdf
#Kaz has the web-based calculator -- to use for QC'ing my results here
#https://www.estat.us/file/calc_t_test1.php

CCC<-filter(match.data1,treat==0)
TTT<-filter(match.data1,treat==1)

#These are for raw database
#raw data
#CCC<-filter(studydata3,treatment==0)
#TTT<-filter(studydata3,treatment==1)
#CCC<-filter(psmdata,treat==0)
#TTT<-filter(psmdata,treat==1)

###############################################################################
#What works clearinghouse version of standardized program difference
#Hedges's g and Cox Index
#See page 15 and 16
#https://ies.ed.gov/ncee/wwc/Docs/referenceresources/WWC_Procedures_Handbook_V4_1_Draft.pdf
#Kaz has the web-based calculator -- to use for QC'ing my results here
#https://www.estat.us/file/calc_t_test1.php

#Hedges' g for continuous variables
#Kaz added the correct, sample size adjusted version of this while meeting with Samara for transition
kaz_macro_lin<-function(kaz1){
col_name <- deparse(substitute(kaz1))
C_mean<-mean(CCC[[col_name]])
T_mean<-mean(TTT[[col_name]])
C_sd<-sd(CCC[[col_name]])
T_sd<-sd(TTT[[col_name]])
C_n<-length(CCC[[col_name]])
T_n<-length(TTT[[col_name]])
total_n<-C_n+T_n
#linear
simple_gap=T_mean-C_mean
g1<- ((T_n-1)*(T_sd*T_sd))+((C_n-1)*(C_sd*C_sd))
g2= T_n + C_n -2
g3= sqrt(g1/g2)
wwc_effect= simple_gap/g3

#I didn't adjust for sample size
omega<-(1-3/( 4*total_n -9))
wwc_effect_n_adjusted= (omega*simple_gap)/g3
print(T_n)
print(C_n)
print(total_n)
print(T_mean)
print(C_mean)
print(T_sd)
print(C_sd)
print ("Hedges'g without sample size adjustment")
print(wwc_effect)
print ("Hedges'g with sample size adjustment (Use this)")
print(wwc_effect_n_adjusted)
print ("FYI: Adjustment factor")
print(omega)
}

#Cox Index
#Kaz added the correct, sample size adjusted version of this while meeting with Samara for transition
kaz_macro_bin<-function(kaz1){
col_name <- deparse(substitute(kaz1))
C_mean<-mean(CCC[[col_name]])
T_mean<-mean(TTT[[col_name]])
C_sd<-sd(CCC[[col_name]])
T_sd<-sd(TTT[[col_name]])
C_n<-length(CCC[[col_name]])
T_n<-length(TTT[[col_name]])

#binary
Odds_C<-(C_mean/(1-C_mean))
Odds_T<-(T_mean/(1-T_mean))

Odds_ratio<-Odds_T/Odds_C

LN_C<-log(Odds_C)
LN_T<-log(Odds_T)
LN_DIF<-LN_T-LN_C

# WWC_effect=(round(LN_DIF/1.65,0.001))
WWC_effect_binary<-(LN_DIF/1.65)

#sample size adjustment (Kaz is adding this on December 30 2022)
#I didn't use this for writing the report draft
total_n<-C_n+T_n
omega<-(1-3/( 4*total_n -9))
WWC_binary_effect_n_adjusted=(omega*LN_DIF)/1.65;
print(T_n)
print(C_n)
print(total_n)
print(T_mean)
print(C_mean)
print("Cox Index without sample size adjustment")
print(WWC_effect_binary)
print("Cox Index with sample size adjustment -- Use this")
print(WWC_binary_effect_n_adjusted)
print ("FYI: Adjustment factor")
print(omega)
}

###############################################################################
table(psmdata$treat)

kaz_macro_bin(male)
kaz_macro_bin(minority)
kaz_macro_bin(disadv)
kaz_macro_bin(binary_dualcredit)

kaz_macro_lin(GPA_12_GRADE)
kaz_macro_lin(SAT_TOTAL)

#Sam asked me to check this
kaz_macro_lin(TOTAL_DUALCREDIT)

kaz_macro_bin(enroll_FR_spring)
kaz_macro_bin(enroll_SP_fall)
kaz_macro_bin(enroll_SP_spring)

December 28, 2022

PHP syntax for calculating WWC standardized group difference (Hedges' g and Cox Index)

<?php

function compute()

{

$Tmean = $_POST['Tmean'];

$Cmean = $_POST['Cmean'];

$TSD = $_POST['TSD'];

$CSD = $_POST['CSD'];

$TN = $_POST['TN'];

$CN = $_POST['CN'];

$mean_dif=$Tmean-$Cmean;

$SE=sqrt(

(($TSD*$TSD) / $TN)+(($CSD*$CSD) / $CN)

);

$T=$mean_dif/$SE;

$DF=$TN+$CN-2;

$P="Under Development (Still working on this)";

/*$P=stats_dens_normal($T, 0,1);*/

/*$P=stats_dens_gamma(float $X, float $shape, float $scale);*/

/*$P= $T / 100 ;*/

/*Hedges g*/

/*g numerator*/

$g_numerator=($Tmean-$Cmean)*(1-3/((4*($TN+$CN))-9));

/*g demnominator*/

$g_denominator=SQRT(((($TN-1)* ($TSD**2) )+(($CN-1)* ($CSD**2) ))/($TN+$CN-2));

$hedges_d=$g_numerator/$g_denominator;

$hedges_d_abs=abs($hedges_d);

/*if binary variabels*/

$T_Odds=$Tmean/(1-$Tmean);

$C_Odds=$Cmean/(1-$Cmean);

$Odds_ratio=$T_Odds/$C_Odds;

$Tstep1=log($T_Odds);

$Cstep1=log($C_Odds);

$step2=$Tstep1-$Cstep1;

$WWC_binary_effect=$step2/1.65;

/*Use omega factor to adjust for data size*/

$total_num=$TN + $CN;

$omega=(1-3/( 4*$total_num -9));

$WWC_binary_effect_omega=($omega*$step2)/1.65;

if ($hedges_d >= 0.2) echo "Small Effect (Cohen)";

if ($hedges_d >= 0.5) echo "Medium Effect (Cohen)";

if ($hedges_d >= 0.8) echo "Large Effect (Cohen)";

echo " ";

echo "WWC group comparison of continuous variables";

echo " ";

echo "Values you entered:";

echo " ";

echo "Treatment N:" .$TN;

echo " ";

echo "Treatment mean:" .$Tmean;

echo " ";

echo "Treatment SD:" .$TSD;

echo " ";

echo "Comparison N:" .$CN;

echo " ";

echo "Comparison mean:" .$Cmean;

echo " ";

echo "Comparison SD:" .$CSD;

echo " ";

echo "The group mean difference:".round($mean_dif,2);

echo " ";

echo "without rounding:".round($mean_dif,5);

echo " ";

/*echo "Probability " .round($P,2);*/

echo "Probability: " .$P;

echo " ";

$abs_T=abs($T);

echo "T-score is: " .round($T,2);

echo " ";

echo "without rounding: " .round($T,5);

echo " ";

if($abs_T < 1.96 ) {

echo "Not significant at alpha 0.05 (two tail test;I used a z-test and ignored degree of freedom; threshold 1.96)";

}elseif($abs_T >=1.96){

echo "Significant at alpha 0.05 (two tail test;I used a z-test and ignored degree freedom; threshold 1.96)";

}

echo " ";

echo "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX";echo " ";

echo "If a variable is an interval scale:";

echo " ";

echo "Hedges g " .round($hedges_d,2);

echo " ";

echo "without rounding: " .round($hedges_d,5);

echo " ";

echo "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX";echo " ";

echo " ";

echo "Hedges g description";

echo " ";

echo "Page 16 of https://ies.ed.gov/ncee/wwc/Docs/referenceresources/WWC_Procedures_Handbook_V4_1_Draft.pdf";

echo " ";

echo "For continuous outcomes, the WWC has adopted the most commonly used effect size";

echo " ";

echo "index, the standardized mean difference known as Hedges’ g, with an adjustment for small";

echo " ";

echo "sample bias. For group design studies, this effect size is defined as the difference between the";

echo " ";

echo "mean outcome for the intervention group and the mean outcome for the comparison group, ";

echo " ";

echo "divided by the pooled within-group standard deviation of the outcome measure. Defining yi and ";

echo " ";

echo "yc as the means of the outcome for students in the intervention and comparison groups, ni and nc";

echo " ";

echo "as the student sample sizes, si and sc as the student-level standard deviations, given by ....";

echo " ";

echo "In addition, we define as the small sample size correction the effect size (Hedges 1981), which is given by ";

echo " ";

/*cohen's rule of thumb*/

echo "Cohen's rule of thumb";echo " ";

echo "if d >= 0.2 small effect -- if d >= 0.5 medium effect if --- d >= 0.8 large effect";echo " ";

if($hedges_d_abs < 0.2) {

echo "Close to zero and Not even small Effect (Cohen)";

}elseif($hedges_d_abs>=0.2 and $hedges_d_abs < 0.5){

echo "Small effect (Cohen)";

}elseif($hedges_d_abs>=0.5 and $hedges_d_abs < 0.8){

echo "Medium effect (Cohen)";

}elseif($hedges_d_abs>=0.8){

echo "Large effect (Cohen)";

}else {

echo "others";

}

echo " ";

echo "If a variable is obviously a continuous variable (e.g., MAX is greater than 1), ignore the result below (It says NAN)";

echo " ";

echo "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX";echo " ";

echo "If the variable is a binary variable (0 or 1), the Cox index value is " .round($WWC_binary_effect,2);

echo " ";

echo "with a lot of digits without rounding " .round($WWC_binary_effect,5);echo " ";

echo "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX";echo " ";

echo " ";

echo "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX";echo " ";

echo "Omega adjusted (data size taken into consideration) " .round($WWC_binary_effect_omega,2);

echo " ";

echo "with a lot of digits without rounding " .round($WWC_binary_effect_omega,5);

echo " ";

echo "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX";echo " ";

echo " ";

echo "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX";echo " ";

echo "Odds ratio:" .round($Odds_ratio,2);

echo " ";

echo "without rounding " .round($Odds_ratio,5);

echo " ";

echo "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX";echo " ";

echo " ";

echo "Cox Index description";

echo " ";

echo "Page 16 of https://ies.ed.gov/ncee/wwc/Docs/referenceresources/WWC_Procedures_Handbook_V4_1_Draft.pdf";

echo " ";

echo "For dichotomous outcomes, the difference in group means is calculated as the difference in";

echo " ";

echo "the probability of the occurrence of an event. The effect size measure of choice for dichotomous ";

echo " ";

echo "outcomes is the Cox index, which yields effect size values similar to the values of Hedges’ g that";

echo " ";

echo "one would obtain if group means, standard deviations, and sample sizes were available, assuming";

echo " ";

echo "the dichotomous outcome measure is based on an underlying logistic similar to a normal";

echo " ";

echo "distribution. Defining pi and pc as the probability of an outcome for students in the intervention";

echo " ";

echo "and comparison groups, the effect size is given by";

echo " ";

echo "";

echo " ";

}

/*echo "The result is: " . compute();*/

compute();

REFERENCE

Cohen's rule of thumb about effect sizes:

<li>If greater than 02, Small Effect

<li>If greater than 0.5, Medium Effect

<li>If greater 0.8 then Large Effect

Cohen, J. Statistical power for the behavioral sciences (2nd ed.). Hillsdale, NJ: Erlbaum (1988).

<ahref="https://wmich.edu/sites/default/files/attachments/u58/2015/Effect_Size_Substantive_Interpretation_Guidelines.pdf">

Effect Size Substantive Interpretation Guidelines: Issues in the Interpretation of Effect Sizes Jeff Valentine and Harris Cooper, Duke University(see page. 5)</a>

WWC related info:

<a href="https://ies.ed.gov/ncee/wwc/Docs/ReferenceResources/wwc_procedures_handbook_v4_draft.pdf">WWC procedures handbook (see page. 14)</a>

<a href="https://ies.ed.gov/ncee/wwc/Docs/OnlineTraining/wwc_training_m3.pdf">WWC standards slides (Definition of small sample size correction, slide 14)</a>

WWC considers the effect size greater than .25 substnatively important.

<a href="https://ies.ed.gov/ncee/wwc/Docs/referenceresources/wwc_procedures_handbook_v4.pdf">P.22 of WWC standards</a>

<a href="calc_t_test1.php">Back to the calculcator </a>

<a href="https://www.estat.us">My website</a>

Look at calc process t..1

April 8, 2021

Steps to Develop a Model to Estimate School- and District-Level Postsecondary Success

Steps to Develop a Model to Estimate School- and District-Level Postsecondary Success (ed.gov)

January 16, 2020

WWC version 4.0

January 14, 2020

WWC pretest effect size

<= 0.05 Satisfies the baseline equivalence requirement

0.05 < and <= 0.25 Requires statistical adjustment to satisfy BA the requirement.

>0.25 Does not satisfy.

July 22, 2019

WWC version 4

https://ies.ed.gov/ncee/wwc/Handbooks#draftV4_1

March 22, 2019March 22, 2019

WWC

WWC recertification video

https://ies.ed.gov/ncee/wwc/OnlineTraining/Cluster

2. Cluster design modules.

February 8, 2019

What pretest covariates are necessary for a graduation study

p 7 and 8

https://ies.ed.gov/ncee/wwc/Docs/ReferenceResources/wwc_dp_protocol_v3.0.pdf

Baseline Equivalence
For the Dropout Prevention topic area, RCTs with high attrition or QED studies must demonstrate
equivalence of the intervention and comparison groups before the intervention. The onus for
demonstrating equivalence in these studies rests with the authors. Sufficient reporting of preintervention data should be included in the study report (or obtained from the study authors) to allow
the review team to draw conclusions about the equivalence of the intervention and comparison
groups.
Important pre-intervention characteristics can include measures that are highly related to the
outcome measure(s). Other important pre-intervention characteristics can include outcome(s)
measured prior to the intervention. However, when the unit of analysis is the student, many
outcome(s) of interest for this review, such as dropout status or high school graduation status, are
not defined or are not informative when measured prior to the intervention.
Studies for which the unit of analysis is the student must show that the groups are equivalent in
terms of race/ethnicity and sex. Additionally, they must demonstrate equivalence of the research
groups in at least one measure of degree of disadvantage including:

• Free and reduced-price lunch status, poverty status, family income
• Being from a single-parent family
• Parent’s education
• Immigrant or English learner (EL) status

Special education or disability status
• Teen parent status

Finally, these studies must demonstrate equivalence of the research groups in at least one
measure of academic performance. These measures can include:

• Standardized test scores
• Whether behind in grade level (could be measured by age among students in the same
grade)
• Frequency of behavior or discipline incidents in school
• Rate of school attendance
• GPA

Because these measures of academic performance are not defined or typically not available for
students who have dropped out of school, studies of interventions for students who have dropped
out may demonstrate equivalence based on the proportion of students in each research group who
are dropouts.
Studies for which the unit of assignment is the school must show that the groups are equivalent
in terms of outcome(s) measured prior to the intervention. Additionally, they must demonstrate
equivalence in race/ethnicity and at least one measure of degree of disadvantage or academic
performance denoted above in bold text.
Groups are considered equivalent if the reported differences in pre-intervention data are less than
or equal to one-quarter of the pooled standard deviation in the sample, regardless of statistical
significance. However, if differences are greater than 0.05 standard deviations and less than or
equal to one-quarter of the pooled standard deviation in the sample, the analysis must control for
the pre-intervention outcome measure(s) on which the groups differ. If pre-intervention
differences are greater than 0.25 for any of the outcomes in the same domain, the study does not
meet standards. In addition, if there is evidence that comparison groups were drawn from very
different settings (such as rural vs. urban), the lead methodologist may decide that the
environments are too dissimilar to provide an adequate comparison.

January 19, 2019

WWC test of group differences

This program compares group means (treatment group vs. control group).

https://www.estat.us/file/calc_t_test1.php

January 18, 2019

Effect size 0.25 is considered "substantively important"

P. 22 of

https://ies.ed.gov/ncee/wwc/Docs/referenceresources/wwc_procedures_handbook_v4.pdf