visualizations.Rmd

---
title: "Visualizations"
author: "Grant"
date: "December 10, 2017"
output: html_document
---

Jared
```{R}
library(dplyr)
if(!require(mlbench)){install.packages("mlbench"); require(mlbench)} # common datasets to use
if(!require(caret)){install.packages("caret", dependencies = c("Depends", "Suggests")); require(caret)} # ML package and its dependencies. This will take awhile!
if(!require(dplyr)){install.packages("dplyr"); require(dplyr)}
set.seed(370)
if(!require(ModelMetrics)){install.packages("ModelMetrics"); require(ModelMetrics)} # common datasets to use
if(!require(recipes)){install.packages("recipes"); require(recipes)} # common datasets to use
if(!require(DEoptimR)){install.packages("DEoptimR"); require(DEoptimR)} # common datasets to use




```

```{r}
baseline <- read.csv(file="/Users/jaredpraino/Documents/University of Washington/Senior Year/INFO370/final_project/project-fragile-families/data/Baseline_Features.csv",header=TRUE,sep=",")
year1 <- read.csv(file="/Users/jaredpraino/Documents/University of Washington/Senior Year/INFO370/final_project/project-fragile-families/data/Year1_Features.csv", header=TRUE,sep=",")
year3 <- read.csv(file="/Users/jaredpraino/Documents/University of Washington/Senior Year/INFO370/final_project/project-fragile-families/data/Year3_Features.csv",header=TRUE,sep=",")
year5 <- read.csv(file="/Users/jaredpraino/Documents/University of Washington/Senior Year/INFO370/final_project/project-fragile-families/data/Year5_Features.csv",header=TRUE,sep=",")
year9 <- read.csv(file="/Users/jaredpraino/Documents/University of Washington/Senior Year/INFO370/final_project/project-fragile-families/data/Year9_Features.csv",header=TRUE,sep=",")

finalSet <- read.csv(file="/Users/grant/Documents/INFO370/fragile_families/project-fragile-families/data/FeaturesWithYear9Year5MartialStatus.csv")


library(dplyr)
year9grades <- filter(finalSet, t5c13a>-1, t5c13b>-1, t5c13c>-1)
year5grades <- filter(finalSet, kind_a5>-1, kind_a6>-1, kind_a7>-1)

finalSet$t5c13b
```



```{r}
temp_parent_involvement <- year9 %>% select(idnum,t5c13a,t5c13b,t5c13c) 

#View(year9)

combine0 <- baseline %>% select(idnum, m1a4, m1a11a, m1a11b) %>% 
  filter(idnum>-1, m1a4>-1, m1a11a>-1, m1a11b>-1)

combine1 <- year1 %>% select(idnum,m2b18a,m2b18b,m2b18c,m2b18d,m2b18e,m2b18f,m2b18g,m2b18h,m2c3a,m2c3b, m2c3c,m2c3d,m2c3e,m2c3f,m2c3h,m2c3i,m2c3j) %>%    filter(m2b18a>-1,m2b18b>-1,m2b18c>-1,m2b18d>-1,m2b18e>-1,m2b18f>-1,m2b18g>-1,m2b18h>-1,m2c3a>-1,m2c3b>-1, m2c3c>-1,m2c3d>-1,m2c3e>-1,m2c3f>-1,m2c3h>-1,m2c3i>-1,m2c3j>-1)

combine3 <- year3 %>% select(idnum,m3b4a,m3b4b,m3b4c,m3b4d,m3b4e,m3b4f,m3b4g,m3b4h,m3b4i,m3b4k,m3b4l,m3b5,m3c3a,m3c3b,m3c3c,m3c3d,m3c3e,m3c3f,m3c3g,m3c3h,m3c3i,m3c3j,m3c3k,m3c3l) %>% 
  filter(m3b4a>-1,m3b4b>-1,m3b4c>-1,m3b4d>-1,m3b4e>-1,m3b4f>-1,m3b4g>-1,m3b4h>-1,m3b4i>-1,m3b4k>-1,m3b4l>-1,m3b5>-1,m3c3a>-1,m3c3b>-1,m3c3c>-1,m3c3d>-1,m3c3e>-1,m3c3f>-1,m3c3g>-1,m3c3h>-1,m3c3i>-1,m3c3j>-1,m3c3k>-1,m3c3l>-1)

combine5 <- year5 %>% select(idnum,m4b4a1,m4b4a2,m4b4a3,m4b4a4,m4b4a5,m4b4a6,m4b4a7,m4b4a8) %>% 
  filter(m4b4a1>-1,m4b4a2>-1,m4b4a3>-1,m4b4a4>-1,m4b4a5>-1,m4b4a6>-1,m4b4a7>-1,m4b4a8>-1)

combine9 <- year9 %>% select(idnum,t5c13a,t5c13b,t5c13c) %>% 
  filter(t5c13a>-1,t5c13b>-1,t5c13c>-1)

join1 <- inner_join(combine0,combine1, by="idnum")
join2 <- inner_join(join1,combine3, by="idnum")
join3 <- inner_join(join2,combine5, by="idnum")
finalCombined <- inner_join(join3,combine9, by="idnum")

# prepare new columns
# total activities
parent_involvement <- finalCombined %>% mutate(tally = 0)

# total grade
year9grades <- year9grades %>% mutate(score = t5c13a + t5c13b + t5c13c)
year9grades <- year9grades %>% mutate(tally = 0)


# add outcome variables
parent_involvement$t5c13a <- as.factor(parent_involvement$t5c13a)
parent_involvement$t5c13b <- as.factor(parent_involvement$t5c13b)
parent_involvement$t5c13c <- as.factor(parent_involvement$t5c13c)

```


```{r}
####    Version 1:  Popularity of activites from Yuxing's feature selection

# ##  SET UP
# top  DPW - hand picked features in days/per/week format
#need more features
features <- c('m4b4a2','m3c3l','m4b4a8','m3b4l','m3b4a','m2c3j','m2c3f','m3c3g','m2c3e','m3b4d','m2c3a')

#

# top  YN - hand picked features in yes no format
#
# 

### View(parent_involvement)

# ##  FUNCTIONS

# COMPUTED MEAN
computed_mean <- function(column) {
  
  response <- parent_involvement[[column]]
  weighted_mean <- (sum(response==0)*0 + sum(response==1)*1 + sum(response==2)*2 + sum(response==3)*3 + sum(response==4)*4 +       sum(response==5)*5 + sum(response==6)*6 +sum(response==7)*7)/length(response)
  
  fltr <- response > weighted_mean
  parent_involvement[fltr,"tally"] <- parent_involvement[fltr,"tally"] + 1
  return(parent_involvement)
}


# ##  COMPUTATIONS
# Number of activities
weighted_means <- c(1:length(features))
for(i in 1:length(features)){
  column_name = features[i]
  parent_involvement <- computed_mean(column_name)
}


# Percentage of activities
length <- length(features)
parent_involvement <- parent_involvement %>% mutate(percentage = tally/11.0)
```

```{r}

library(ggplot2)
# ## GRAPHS
# Simple scatterplots 
# Involvement by math
ggplot(data = parent_involvement, aes(x = t5c13c, y = tally)) + geom_point() + geom_jitter(width = 0.2)

# Involvment by reading
ggplot(data = parent_involvement, aes(x = t5c13a, y = tally)) + geom_point() + geom_jitter(width = 0.2)

# Involvement by science
ggplot(data = parent_involvement, aes(x = t5c13b, y = tally)) + geom_point() + geom_jitter(width = 0.2)


#Involvement total score
ggplot(data = parent_involvement, aes(x = score, y = percentage)) + geom_point() + geom_jitter(width = 0.2)

```

```{r}
#Barplot for days per week playing imaginary games with child
test.baselineYear9 <- select(combine4, t5c13c, m3b4e) %>% filter(t5c13c > -1, m3b4e > -1, t5c13c < 7)


day1 <- filter(test.baselineYear9, m3b4e == 1)
dfDay1 <- table(day1)

day2 <- filter(test.baselineYear9, m3b4e == 2)
dfDay2 <- table(day2)

day3 <- filter(test.baselineYear9, m3b4e == 3)
dfDay3 <- table(day3)

day4 <- filter(test.baselineYear9, m3b4e == 4)
dfDay4 <- table(day4)

day5 <- filter(test.baselineYear9, m3b4e == 5)
dfDay5 <- table(day5)

day6 <- filter(test.baselineYear9, m3b4e == 6)
dfDay6 <- table(day6)

day7 <- filter(test.baselineYear9, m3b4e == 7)
dfDay7 <- table(day7)

View(newdf)

barplot(height = dfDay2, beside = T, col=c("aquamarine3","coral", "red", "blue", "green"))
legend("topleft", c("far below average", "below average", "average", "above average", "far above average"), pch=15, 
       col=c("aquamarine3","coral", "red", "blue", "green"), 
       bty="n")

barplot(height = dfDay3, beside = T, col=c("aquamarine3","coral", "red", "blue", "green"))
legend("topleft", c("far below average", "below average", "average", "above average", "far above average"), pch=15, 
       col=c("aquamarine3","coral", "red", "blue", "green"), 
       bty="n")

barplot(height = dfDay4, beside = T, col=c("aquamarine3","coral", "red", "blue", "green"))
legend("topleft", c("far below average", "below average", "average", "above average", "far above average"), pch=15, 
       col=c("aquamarine3","coral", "red", "blue", "green"), 
       bty="n")

barplot(height = dfDay5, beside = T, col=c("aquamarine3","coral", "red", "blue", "green"))
legend("topleft", c("far below average", "below average", "average", "above average", "far above average"), pch=15, 
       col=c("aquamarine3","coral", "red", "blue", "green"), 
       bty="n")

barplot(height = dfDay6, beside = T, col=c("aquamarine3","coral", "red", "blue", "green"))
legend("topleft", c("far below average", "below average", "average", "above average", "far above average"), pch=15, 
       col=c("aquamarine3","coral", "red", "blue", "green"), 
       bty="n")

barplot(height = dfDay7, beside = T, col=c("aquamarine3","coral", "red", "blue", "green"))
legend("topleft", c("far below average", "below average", "average", "above average", "far above average"), pch=15, 
       col=c("aquamarine3","coral", "red", "blue", "green"), 
       bty="n")

ggplot(data = as.data.frame(newdf), aes(m3b4e, t5c13c, size = Freq, color = Freq)) + geom_point() +
scale_color_gradient(low="blue", high="red") 

#linear model for best feature against math score
linear_model <- lm(t5c13c ~ m3c3h, data=test.baselineYear9)
linear_model
```


```{r}
####    Verison 2: Popularity of all activities in the days/week format


# total grade


# Our predictor variables
features2 <- c('f1g9a',	'f1g9b',	'f1g9c',	'f1g9d',	'f1g9e',	'f1g9f',	'f1g9g',	'f1g9h',	'f1g9i',	'f1g9j',	'f1g9k',	'f1g9l','m2b18a',	'm2b18b',	'm2b18c',	'm2b18d',	'm2b18e',	'm2b18f',	'm2b18g',	'm2b18h','m2c3a','m3b4a',	'm3b4b',	'm3b4c',	'm3b4d',	'm3b4e',	'm3b4f',	'm3b4g',	'm3b4h',	'm3b4i',	'm3b4j',	'm3b4k',	'm3b4l',	'm3b4m','m3c3a','m3c3b',	'm3c3c',	'm3c3d',	'm3c3e',	'm3c3f',	'm3c3g',	'm3c3h',	'm3c3i',	'm3c3j',	'm3c3k',	'm3c3l',	'm3c3m','f3c3a',	'f3c3b',	'f3c3c',	'f3c3d',	'f3c3e',	'f3c3f',	'f3c3g',	'f3c3h',	'f3c3i',	'f3c3j',	'f3c3k',	'f3c3l',	'f3c3m','f3b4a',	'f3b4b',	'f3b4c',	'f3b4d',	'f3b4e',	'f3b4f',	'f3b4g',	'f3b4h',	'f3b4i',	'f3b4j',	'f3b4k',	'f3b4l',	'f3b4m','m4c3a',	'm4c3b',	'm4c3c',	'm4c3d',	'm4c3e',	'm4c3f','m4c3g',	'm4c3h','f4b4a1',	'f4b4a2',	'f4b4a3',	'f4b4a4',	'f4b4a5',	'f4b4a6',	'f4b4a7',	'f4b4a8',	'f4c3a',	'f4c3b',	'f4c3c',	'f4c3d',	'f4c3e',	'f4c3f',	'f4c3g',	'f4c3h','f2b36a',	'f2b36b',	'f2b36c',	'f2b36d',	'f2b36e',	'f2b36f',	'f2b36g',	'f2b36h',	'f2c3a',	'f2c3b',	'f2c3c',	'f2c3d',	'f2c3e',	'f2c3f',	'f2c3g',	'f2c3h',	'f2c3i',	'f2c3j','m2c3b',	'm2c3c',	'm2c3d',	'm2c3e',	'm2c3f',	'm2c3g',	'm2c3h',	'm2c3i',	'm2c3j')


features2 <- c("m3c3e","m4b4a1","m3b4h", "m4b4a5")


parent_involvement2 <- read.csv("/Users/jaredpraino/Documents/University of Washington/Senior Year/INFO370/final_project/project-fragile-families/data/FeaturesSelected.csv")

## convert 200's values
# 201 => 7
# 202 => 3
# 203 => 2
# 204 => 1
# 205 => 0
parent_involvement2[parent_involvement2==205] <- 0
parent_involvement2[parent_involvement2==204] <- 1
parent_involvement2[parent_involvement2==203] <- 2
parent_involvement2[parent_involvement2==202] <- 3
parent_involvement2[parent_involvement2==201] <- 7
# remove negative values
year9grades[year9grades<0] <- NA

bargraph <- ggplot(year9grades, aes(t5c13c, m3b4a)) + geom_bar()

bargraph


# grades and total grade
combine9_2 <- year9 %>% select(idnum,t5c13a,t5c13b,t5c13c) %>% 
  filter(t5c13a>-1,t5c13b>-1,t5c13c>-1)
year9grades <- year9grades %>% mutate(score=t5c13a + t5c13b + t5c13c)


combine9_2$t5c13a <- as.factor(combine9_2$t5c13a)
combine9_2$t5c13b <- as.factor(combine9_2$t5c13b)
combine9_2$t5c13c <- as.factor(combine9_2$t5c13c)
combine9_2$score <- as.factor(combine9_2$score)

# merge
parent_involvement2 <- inner_join(combine9_2, parent_involvement2, by="idnum")
parent_involvement2 <- parent_involvement2 %>% mutate(tally = 0)



View(finalSet)
View(year5grades)



# ##  FUNCTIONS
# convert 
# -6 to 0
# apply to c()
##c('m2b28','f2b24','m3b11','f3b11')


computed_mean2 <- function(column) {
   
 response <- na.omit(year5grades[[column_name]])
 weighted_mean <- (sum(response==0)*0 + sum(response==1)*1 + sum(response==2)*2 + sum(response==3)*3 + sum(response==4)*4 +       sum(response==5)*5 + sum(response==6)*6 +sum(response==7)*7)/length(response)
  
  fltr <- response > 3.5
  year5grades[fltr,"tally2"] <- year5grades[fltr,"tally2"] + 1
  print(weighted_mean)
  return(year5grades)
}


for(i in 1:length(features2)){
  column_name = features2[i]
  year5grades <- computed_mean2(column_name)
}

computed_mean2 <- function(column) {
   
  response <- na.omit(year9grades[[column_name]])
  weighted_mean <- (sum(response==0)*0 + sum(response==1)*1 + sum(response==2)*2 + sum(response==3)*3 + sum(response==4)*4 +       sum(response==5)*5 + sum(response==6)*6 +sum(response==7)*7)/length(response)
  
  fltr <- response > weighted_mean
  year9grades[fltr,"tally"] <- year9grades[fltr,"tally"] + 1
  print(weighted_mean)
  return(year9grades)
}


for(i in 1:length(features2)){
  column_name = features2[i]
  year9grades <- computed_mean2(column_name)
}



parent_involvement2$score
length <- length(features2)
parent_involvement2 <- parent_involvement2 %>% mutate(percentage = tally/124.0)


```

```{r}
#load required libraries
library(reshape2)
library(ggplot2)
```



```{r}
#Year 9 visualizations

#Boxplots
ggplot(data = parent_involvement2, aes(x = t5c13c, y = tally)) + geom_point() + geom_jitter(width = 0.2) + geom_boxplot() +labs(title="Math Performance v Parent Activity Count", x="Performance in Math at Year 9", y="# of Parent Activities") + scale_x_discrete(labels=c("Far Below Average","Below Average","Average","Above Average","Far Above Average")) +
  geom_smooth(method = "lm", se=FALSE, color="black", aes(group=1))


# Involvment by reading
ggplot(data = parent_involvement2, aes(x = t5c13a, y = tally)) + geom_point() + geom_jitter(width = 0.2) + geom_boxplot() +labs(title="Reading Performance v Parent Activity Count", x="Performance in Reading at Year 9", y="# of Parent Activities")+  scale_x_discrete(labels=c("Far Below Average","Below Average","Average","Above Average","Far Above Average"))+
  geom_smooth(method = "lm", se=FALSE, color="black", aes(group=1))

# Involvement by science
ggplot(data = parent_involvement2, aes(x = t5c13b, y = tally)) + geom_point() + geom_jitter(width = 0.2) + geom_boxplot() +labs(title="Science Performance v Parent Activity Count", x="Performance in Science at Year 9", y="# of Parent Activities")+ scale_x_discrete(labels=c("Far Below Average","Below Average","Average","Above Average","Far Above Average"))+
  geom_smooth(method = "lm", se=FALSE, color="black", aes(group=1))


#Scatterplots with linear regression
#Involvement total score
ggplot(data = parent_involvement2, aes(x = score, y = tally)) + geom_point() + geom_jitter(width = 0.2) + geom_boxplot() +labs(title="Total Performance v Parent Activity Count", x="School Performance at Year 9", y="# of Parent Activities")  + scale_x_discrete(labels=c("Far Below Average","","","Below Average","","","Average","","","Above Average","","","Far Above Average"))+ geom_smooth(method = "lm", se=FALSE, color="black", aes(group=1))

#involvement by math
ggplot(data = year9grades, aes(x = t5c13c, y = tally)) + geom_point() + geom_jitter(width = 0.2) + labs(title="Math Performance v Parent Activity Count", x="Performance in Math", y="# of Parent Activities") + scale_x_continuous(labels=c("Far Below Average","Below Average","Average","Above Average","Far Above Average"), breaks = c(1, 2, 3, 4, 5))

# Involvment by reading
ggplot(data = year9grades, aes(x = t5c13a, y = tally, color=marrystat)) + geom_point() + geom_jitter(width = 0.2) + geom_smooth(method=lm, se=FALSE, group=1) +labs(title="Reading Performance v Parent Activity Count", x="Performance in Reading", y="# of Parent Activities", color = "Marriage Status") + scale_x_continuous(labels=c("Far Below Average","Below Average","Average","Above Average","Far Above Average"), breaks = c(1, 2, 3, 4, 5))

# Involvement by science
ggplot(data = year9grades, aes(x = t5c13b, y = tally, color=marrystat)) + geom_point() + geom_jitter(width = 0.2) + geom_smooth(method=lm, se=FALSE, group=1) +labs(title="Science Performance v Parent Activity Count", x="Science Performance", y="# of Parent Activities", color = "Marriage Status") + scale_x_continuous(labels=c("Far Below Average","Below Average","Average","Above Average","Far Above Average"), breaks = c(1, 2, 3, 4, 5))


#Involvement total score
ggplot(data = year9grades, aes(x = score, y = tally, color=marrystat)) + geom_point() + geom_jitter(width = 0.2) + geom_smooth(method=lm, se=FALSE, group=1) + labs(title="Total Performance v Parent Activity Count", x="Total Performance", y="# of Parent Activities", color = "Marriage Status") + scale_x_continuous(labels=c("Far Below Average","Below Average","Average","Above Average","Far Above Average"), breaks = c(3, 6, 9, 12, 15))

#Involvement total score with marital status
ggplot(data = year9grades, (aes(x= score, color = marrystat, fill = marrystat))) + geom_bar(position = "dodge", width = 0.5)

#Violin Plot
install.packages("devtools")
library(devtools)
install_github("easyGgplot2", "kassambara")

library(easyGgplot2)

#violin plot for total score
ggplot2.violinplot(data=parent_involvement2, xName='score',yName='tally', addDot=FALSE, addMean=TRUE) + labs(title="Total Performance v Parent Activity Count", x="Total Performance", y="# of Parent Activities") + scale_x_discrete(labels=c("Far Below Average","","","Below Average","","","Average","","","Above Average","","","Far Above Average"))

```

```{r}
#Year 5 visualizations


#involvement by math
ggplot(data = year5grades, aes(x = kind_a7, y = tally2)) + geom_point() + geom_jitter(width = 0.2) + geom_boxplot() +labs(title="Math Performance v Parent Activity Count", x="Math Performance", y="# of Parent Activities", color = "Marriage Status") + scale_x_continuous(labels=c("Far Below Average","Below Average","Average","Above Average","Far Above Average"), breaks = c(1, 2, 3, 4, 5))

cor(year5grades$kind_a7, year5grades$tally2)

# Involvment by reading
ggplot(data = year5grades, aes(x = kind_a5, y = tally2, color = marrystat)) + geom_point() + geom_jitter(width = 0.2) + geom_smooth(method=lm, group=1, se=FALSE) +labs(title="Reading Performance v Parent Activity Count", x="Reading Performance", y="# of Parent Activities", color = "Marriage Status") + scale_x_continuous(labels=c("Far Below Average","Below Average","Average","Above Average","Far Above Average"), breaks = c(1, 2, 3, 4, 5))

cor(year5grades$kind_a5, year5grades$tally2)

# Involvement by science
ggplot(data = year5grades, aes(x = kind_a6, y = tally2, color = marrystat)) + geom_point() + geom_jitter(width = 0.2) + geom_smooth(method=lm, group=1, se=FALSE) +labs(title="Science Performance v Parent Activity Count", x="Science Performance", y="# of Parent Activities", color = "Marriage Status") + scale_x_continuous(labels=c("Far Below Average","Below Average","Average","Above Average","Far Above Average"), breaks = c(1, 2, 3, 4, 5))

cor(year5grades$kind_a6, year5grades$tally2)

#Involvement total score
ggplot(data = year9grades, aes(x = score, y = tally)) + geom_violin() + geom_smooth(method=lm, group=1, se=FALSE) +labs(title="Total Performance v Parent Activity Count", x="Total Performance", y="# of Parent Activities", color = "Marriage Status") + scale_x_continuous(labels=c("Far Below Average","Below Average","Average","Above Average","Far Above Average"), breaks = c(3, 6, 9, 12, 15))

cor(year5grades$scoreKind, year5grades$tally2)

#Math scores according to binary values
ggplot(data = year9gradesHighLow, aes(x = as.factor(abovebelowMath), y = tally)) + geom_violin()  + geom_smooth(method=lm, se=FALSE, group=1) +labs(title="Math Performance v Parent Activity Count", x="Performance in Math", y="# of Parent Activities") + scale_x_discrete(labels=c("Far Below Average","Below Average","Average","Above Average","Far Above Average"))

```