Outline

Statistical significance and practical significance

group1 <- rnorm(1000000, mean = 0.01)
group2 <- rnorm(1000000, mean = 0)
t.test(group1, group2)
## 
##  Welch Two Sample t-test
## 
## data:  group1 and group2
## t = 7.1219, df = 2e+06, p-value = 1.065e-12
## alternative hypothesis: true difference in means is not equal to 0
## 95 percent confidence interval:
##  0.007300852 0.012845106
## sample estimates:
##   mean of x   mean of y 
## 0.011239436 0.001166458
range <- data.frame(x = c(-2,2))
ggplot(range, aes(x)) +
  stat_function(fun = dnorm, args = list(mean = 0.01, sd = 1)) +
  stat_function(fun = dnorm, args = list(mean = 0, sd = 1)) +
  theme_tufte() +
  ggtitle("Two significantly different distributions?")

c_null <- qnorm(.95)
mu <- 0.001
powern <- function(n) {
  1 - pnorm(c_null - mu*sqrt(n))
}
range <- data.frame(n = 10^c(1:7))
ggplot(range, aes(n)) + 
  stat_function(fun = powern) + theme_tufte() +
  ylab("Power") +
  ggtitle("Power as a function of sample size, mu = 0.001")

powermu <- function(mu) 1 - pnorm(c_null - mu*10)
range <- data.frame(mu = seq(from = 0, to = 1, length.out = 100))
ggplot(range, aes(mu)) + 
  stat_function(fun = powermu) + theme_tufte() +
  ylab("Power") +
  ggtitle("Power as a function of true mean, n = 100")

c_null <- qnorm(.975)
powermu <- function(mu) pnorm(-c_null - mu*10) + pnorm(c_null - mu*10, lower.tail = F)
range <- data.frame(mu = seq(from = -.5, to = .5, length.out = 100))
ggplot(range, aes(mu)) + 
  stat_function(fun = powermu) + theme_tufte() +
  ylab("Power") +
  ggtitle("Power as a function of true mean (two-sided), n = 100")