Effect size: the new religon

The paper about the effect size metrics in software engineering is here: http://www.sciencedirect.com/science/article/pii/S0950584907000195

In case you need the bibtex:

@article{DBLP:journals/infsof/KampenesDHS07,
  author    = {Vigdis By Kampenes and
               Tore Dyb{\aa} and
               Jo Erskine Hannay and
               Dag I. K. Sj{\o}berg},
  title     = {A systematic review of effect size in software engineering
               experiments},
  journal   = {Information {\&} Software Technology},
  volume    = {49},
  number    = {11-12},
  year      = {2007},
  pages     = {1073-1086},
  ee        = {http://dx.doi.org/10.1016/j.infsof.2007.02.015},
  bibsource = {DBLP, http://dblp.uni-trier.de}
}

For code that implements the hedges g measure recommended by this paper, see below (the second last function: ghedge). Note that its kinda simple. For this to work, you need a value of "small". As per the recommendations of the above paper, I use 0.17.

function scottknott(datas,small,  data) {
  for(data in datas) 
    sk1(datas[data],small,1,1,length(datas[data]),0) 
}
function sk1(rxs,small,rank,lo,hi,  cut,i) { 
  cut = sk0(rxs,small,lo,hi)
  if (cut) {
    rank = sk1(rxs,small,rank,lo,cut-1) + 1
    rank = sk1(rxs,small,rank,cut,hi) 
  } else {
    for(i=lo;i<=hi;i++)
      rxs[i]["rank"] = rank
  }
  return rank
}
function sk0(a,small,lo,hi,\
             s,  s2, n, mu,                     \
             ls,ls2,ln,lmu,                     \
             rs,rs2,rn,rmu,                     \
             i,best,tmp,cut) {
  for(i=lo;i<=hi;i++) {
    s += a[i]["s"]; s2+= a[i]["s2"]; n += a[i]["n"]
  }
  mu= s/n
  best = -1; 
  for(i=lo+1;i<=hi;i++) {
    ls  += a[i-1]["s"]; 
    ls2 += a[i-1]["s2"]; 
    ln  += a[i-1]["n"]
    rs   = s - ls; 
    rs2  = s2- ls2; 
    rn   = n - ln; 
    rmu  = rs/rn
    lmu  = ls/ln
    tmp = ln/n * (mu - lmu)^2 + rn/n * (mu - rmu)^2 
    if (tmp > best) 
      if  (ghedge(rmu,lmu,rn,ln,rs,ls,rs2,ls2) > small) 
        if (significantlyDifferent(a,lo,i,hi)) {
            best = tmp; cut = i }}
  return cut
}
function ghedge(rmu,lmu,rn,ln,rs,ls,rs2,ls2,\
              n, ln1,rn1,rsd, lsd,sp,correct,g) {
  n       = rn + ln
  ln1     = ln - 1
  rn1     = rn - 1  
  rsd     = sd(rn,rs,rs2)
  lsd     = sd(ln,ls,ls2)
  sp      = sqrt((ln1*lsd^2 + rn1*rsd^2)/(ln1 + rn1))
  correct = 1 - 3/(4*(n - 2) - 1)
  return abs(lmu - rmu)/sp * correct
}
function significantlyDifferent(a,lo,cut,hi,\
                               i,j,x,r,l,rhs,lhs) {
  for(i=lo;i<=hi;i++) 
    for(j in  a[i]["="]) {
      x = a[i]["="][j]
      i < cut ?  lhs[++l] = x :  rhs[++r] = x
    }
  print ":lo",lo,":cut",cut,":hi",hi,":left",l,":right",r > "/dev/stderr"
  return bootstrap(lhs,rhs,500,0.05)
}