.Rhistory

levels(d$Disease)[41] <- "Strep toxic shock synd"
levels(d$Disease)[42] <- "Syphilis congenital <1yr"
levels(d$Disease)[42] <- "Toxic shock synd staph"
levels(d$Disease)[47] <- "Vanco Interm Staph A"
levels(d$Disease)[48] <- "Vanco Resist Staph A"
d$threshold[is.na(d$threshold)]<-d$c[is.na(d$threshold)]
d$alert[is.na(d$alert)]<-"N"
write.table(d, file="infreq.txt", row.names=FALSE, col.names=TRUE)
write.table(unique(d$Disease), file="inf_dis.txt",row.names=FALSE, col.names=TRUE)
shiny::runApp('Documents/PhD/Spring 15/mmwr scrape/CDCPlot')
getwd()
setwd("/home/nick/Documents/PhD/Spring 15/mmwr scrape/CDCPlot"")
setwd("/home/nick/Documents/PhD/Spring 15/mmwr scrape/CDCPlot")
all <- matrix(c(
c("Cryptosporidiosis","Cryptosporidiosis", "b36e-ru3r", "2014"),
c("Cryptosporidiosis","Cryptosporidiosis", "9n3x-apcd", "2015"),
c("Salmonellosis", "Salmonellosis", "52cr-rw4k", "2014"),
c("Salmonellosis", "Salmonellosis", "d6kj-devz", "2105"),
c("Shigellosis","Shigellosis", "52cr-rw4k", "2014"),
c("Shigellosis","Shigellosis", "n3wf-wtep", "2015"),
c("Pertussis","Pertussis", "8rkx-vimh", "2014"),
c("Pertussis","Pertussis", "d69q-iyrb", "2015"),
c("Malaria","Malaria", "y6uv-t34t", "2014"),
c("Malaria","Malaria", "7pb7-w9us", "2015"),
c("Legionellosis","Legionellosis", "23gt-ssfe", "2014"),
c("Legionellosis","Legionellosis", "ydsy-yh5w", "2015"),
c("Hepatitis A", "Hepatitis..viral..acute...type.A","rg4j-6mcc", "2014"),
c("Hepatitis A", "Hepatitis..viral..acute...type.A","65xe-6neq", "2015"),
c("Hepatitis B, Acute", "Hepatitis..viral..acute...type.B","rg4j-6mcc","2014"),
c("Hepatitis B, Acute", "Hepatitis..viral..acute...type.B","65xe-6neq","2015"),
c("Hepatitis C, Acute", "Hepatitis..viral..acute...type.C","rg4j-6mcc","2014"),
c("Hepatitis C, Acute", "Hepatitis..viral..acute...type.C","65xe-6neq","2015"),
c("Giardiasis", "Giardiasis", "9ix3-ryt6","2014"),
c("Giardiasis", "Giardiasis", "mpdg-hf57","2015"),
c("Meningococcal Disease Invasive (all serogroups)", "Meningococcal.disease..invasive...All.serogroups", "y6uv-t34t","2014"),
c("Meningococcal Disease Invasive (all serogroups)", "Meningococcal.disease..invasive...All.serogroups", "7pb7-w9us","2015"),
c("Mumps", "Mumps", "8rkx-vimh","2014"),
c("Mumps", "Mumps", "d69q-iyrb","2015"),
#c("Pneumonia and Influenza Mortality Reports by City/Region, 2014", "P.I..Total","qpap-3u8w"),
#leave out pneumonia for now, format is too different
c("Shiga toxin-producing E. coli (STEC)", "Shiga.toxin.producing.E..coli..STEC..", "52cr-rw4k","2014"),
c("Shiga toxin-producing E. coli (STEC)", "Shiga.toxin.producing.E..coli..STEC..", "n3wf-wtep","2015"),
c("P&I MORT", "P&I MORT", "qpap-3u8w", "2014"),
c("P&I MORT", "P&I MORT", "7esm-uptm", "2015")
)
,ncol=4, byrow=T)
#Name matrix columns and write to csv file
URL_NAMES <- data.frame(display_name=all[,1],data_name=all[,2],url=all[,3],year=all[,4])
write.table(URL_NAMES, file="urldat.txt", row.names=FALSE, col.names=TRUE)
# Read in data with disease names and corresponding urls.  This data is created from the url_names.R file, which should be run first.
urldat <- read.table("urldat.txt", header=T)
# A function to help deal with NA values when calculating thresholds.  NA's occur when we try to
# calculate running standard deviations with only one data point, and cause an error in the cumsum function.
# Args:
#   x: A vector of disease occurance data that we wish to calculate an alert threshold for
#   days: an integer for the number of days to calculate the threshold over
newthresh <- function(x,days){
thresh <-runmean(x, days,endrule="NA")+2*runsd(x, days,endrule="sd",align="right")
thresh[is.na(thresh)]<-x[is.na(thresh)]
return(thresh)
}
# This function takes each url and corresponding disease name and gets data from CDC.  It then combines multiple years worth of data,
# calculates alert thresholds and cumulative sums and returns the columns of interest from the CDC data.
# Args:
#   url_data: the rows of the url_data.txt file which contain the urls for a given disease
url_func <- function(url_data){
# Construct actual CDC website url name and get data for 2014 and 2015
curl <- url_data$url
URL <- paste( "https://data.cdc.gov/api/views/",curl, "/rows.csv?accessType=DOWNLOAD",sep="")
nndss14 <-read.csv(textConnection(getURL(URL[1],ssl.verifypeer=FALSE)),strip.white=T,stringsAsFactors=F)
nndss15 <- read.csv(textConnection(getURL(URL[2],ssl.verifypeer=FALSE)),strip.white=T,stringsAsFactors=F)
# Some diseases have a slightly different name for MMWR.Week and MMWR.Year, so we standardize the names here
if("MMWRWeek"%in%names(nndss14)){nndss14<- dplyr::rename(nndss14,MMWR.Week=MMWRWeek )}
if("MMWR.WEEK"%in%names(nndss14)){nndss14<- dplyr::rename(nndss14,MMWR.Week=MMWR.WEEK )}
if("MMWRYear"%in%names(nndss14)){nndss14<- dplyr::rename(nndss14, MMWR.Year=MMWRYear )}
if("MMWR.YEAR"%in%names(nndss14)){nndss14<- dplyr::rename(nndss14, MMWR.Year=MMWR.YEAR )}
if("MMWRWeek"%in%names(nndss15)){nndss15<- dplyr::rename(nndss15, MMWR.Week=MMWRWeek  )}
if("MMWR.WEEK"%in%names(nndss15)){nndss15<- dplyr::rename(nndss15, MMWR.Week=MMWR.WEEK  )}
if("MMWRYear"%in%names(nndss15)){nndss15<- dplyr::rename(nndss15,  MMWR.Year=MMWRYear )}
if("MMWR.YEAR"%in%names(nndss15)){nndss15<- dplyr::rename(nndss15,  MMWR.Year=MMWR.YEAR )}
# dname is the name of the column in the nndss file which contains weekly data for the disease of interest
dname <- c(paste(url_data$data_name[1],"..Current.week",sep=""))
#special column name for P&I mortality data
if(url_data$data_name[1]=="P&I MORT")dname <- "P.I..Total"
# Select relevant columns from both the 2014 and 2015 data and rbind them together
nndss <- rbind(select(nndss14, contains(dname), contains("MMWR"), contains("Reporting"), -contains("flag")),
select(nndss15, contains(dname), contains("MMWR"), contains("Reporting"), -contains("flag")))
# set NA values to 0, maybe not a great idea, but useful for calculating thresholds and cumulative sums
names(nndss)[which(dname==names(nndss))] <- "c"
nndss$c <- as.numeric(nndss$c)
nndss$c[is.na(nndss$c)]<-0
nndss$display_name <- url_data$display_name[1]
# Create columns for 10 and 14 week thresholds and 10 and 14 week alerts, grouping by reporting area.
nndss <- nndss %>% group_by(Reporting.Area) %>% mutate(fourteenwk.thresh=newthresh(c,14),
tenwk.thresh=newthresh(c,10),
fourteenwk.alert=c>fourteenwk.thresh,
tenwk.alert=c>tenwk.thresh)
# Create columns for cumulative sum along with cumulative threshold values, grouping both by reporting area and year
nndss <- group_by(nndss, Reporting.Area, MMWR.Year) %>% mutate(cumulate=cumsum(c),
cumu10=cumulate+(tenwk.thresh-c),
cumu14=cumulate+(fourteenwk.thresh-c))
#select and return relevant columns of data table
nndss<- select(nndss, one_of("c","Reporting.Area", "MMWR.Year", "MMWR.Week","display_name"), contains("thresh"),contains("cumu"),contains("alert"))
return(nndss)
}
# Run the url_func function for each different disease name in our urldat.txt data file
output <- ddply(urldat, .(data_name), url_func)
Encoding(output$Reporting.Area) <- "latin1"
output$Reporting.Area <- iconv(output$Reporting.Area, "latin1", "ASCII", sub="")
# Write output as plotdat.csv
write.table(output, file="plotdat.txt", row.names=FALSE, col.names=TRUE)
# Separate output file which contains all disease names called disease_names.csv
write.table(unique(output$display_name), file="disease_names.txt", row.names=FALSE, col.names=TRUE)
# Separate output file which contains locations and location types (state, region, or country) called location_names.cd, doesn't include p&i data
regions <-c("NEW ENGLAND", "MID. ATLANTIC", "E.N. CENTRAL", "W.N. CENTRAL", "S. ATLANTIC",
"E.S. CENTRAL", "W.S. CENTRAL", "MOUNTAIN", "PACIFIC", "TERRITORIES")
loc_type <- rep("state", length(unique(output$Reporting.Area[output$data_name!="P&I MORT"])))
loc_type[which(unique(output$Reporting.Area[output$data_name!="P&I MORT"])%in%regions)] <- "region"
loc_type[1] <- "country"
# Also include, for state locations, which region the state falls under.  Thankfully, the CDC data table is ordered so that it first lists a region, then
# all the states in that region, then the next region, and so on.  So, between each region name, all states will be in the same region
region_num=0
loc_reg <- rep("NONE", length(loc_type))
for(i in 1:62){
if(loc_type[i]=="region"){
region_num = region_num+1
}
if(loc_type[i]=="state"){
loc_reg[i]=regions[region_num]
}
}
loc_reg[63:67] <- "TERRITORIES"
all_locs<-data.frame(location=unique(output$Reporting.Area[output$data_name!="P&I MORT"]),type=loc_type, region=loc_reg)
write.table(all_locs, file="location_names.txt", row.names=FALSE, col.names=TRUE)
pi_loc <- rep("city", length(unique(output$Reporting.Area[output$data_name=="P&I MORT"])))
pi_loc[which(tolower(unique(output$Reporting.Area[output$data_name=="P&I MORT"]))%in%tolower(regions))] <- "region"
pi_loc[length(pi_loc)] <- "country"
region_num=0
pi_reg <- rep("NONE", length(pi_loc))
for(i in 1:length(pi_loc)){
if(pi_loc[i]=="region"){
region_num = region_num+1
}
if(pi_loc[i]=="city"){
pi_reg[i]=regions[region_num]
}
}
pi_locs<-data.frame(location=unique(output$Reporting.Area[output$data_name=="P&I MORT"]),type=pi_loc, region=pi_reg)
pi_locs <- data.frame(lapply(pi_locs, as.character), stringsAsFactors=FALSE)
write.table(pi_locs, file="pi_names.txt", row.names=FALSE, col.names=TRUE)
#separate code for infrequent diseases.
URL <- c("https://data.cdc.gov/api/views/wcwi-x3uk/rows.csv?accessType=DOWNLOAD",
"https://data.cdc.gov/api/views/pb4z-432k/rows.csv?accessType=DOWNLOAD")
nndss14 <-read.csv(textConnection(getURL(URL[1],ssl.verifypeer=FALSE)),strip.white=T,stringsAsFactors=F)
nndss15 <- read.csv(textConnection(getURL(URL[2],ssl.verifypeer=FALSE)),strip.white=T,stringsAsFactors=F)
nndss <- rbind(select(nndss14, contains("Current.week"), contains("MMWR"), contains("Disease"), -contains("flag")),
select(nndss15,  contains("Current.week"),contains("MMWR"), contains("Disease"), -contains("flag")))
#disease names are different bewteen years, try to clean some disease names up
Encoding(nndss$Disease) <- "latin1"
nndss$Disease <- iconv(nndss$Disease, "latin1", "ASCII", sub="")
nndss$Disease <- gsub(":","",nndss$Disease)
nndss$Disease <- gsub(",","",nndss$Disease)
nndss$Disease <- gsub("\\*","",nndss$Disease)
#remove all disease names which aren't present in both years
nndss <- nndss[-which(nndss$Disease%in%names(which(table(nndss$Disease)<54))),]
d <- nndss %>% group_by(Disease, MMWR.week, MMWR.year)%>%summarise(c=Current.week)
d$c <- as.numeric(d$c)
d <- d %>% mutate(c = ifelse(is.na(c),0,c))
d <- d %>% group_by(Disease) %>% mutate("fourteenweekmean"=runmean(c, 2, align="right"))
d <- d %>% mutate(fourteenweekmean = ifelse(is.na(fourteenweekmean),0,fourteenweekmean))
d <- d%>% group_by(Disease) %>% mutate ("fourteenweeksd"=runsd(c, 14,center=runmean(c,14),align="right"))
d$sd2 <- d$fourteenweeksd*2
d$threshold <- d$fourteenweekmean + d$sd2
d$alert <- ifelse(d$c > d$threshold, "Y", "N")
d$Disease <- as.factor(d$Disease)
levels(d$Disease)[3] <- "Arbo,EEE"
levels(d$Disease)[2] <- "Arbo,CA serogroup"
levels(d$Disease)[4] <- "Arbo,Powassan"
levels(d$Disease)[5] <- "Arbo,St Louis"
levels(d$Disease)[6] <- "Arbo,WEE"
levels(d$Disease)[9] <- "Botulism other"
levels(d$Disease)[14] <- "Cyclosporiasis"
levels(d$Disease)[16] <- "H flu <5 non-b"
levels(d$Disease)[17] <- "H flu <5 b"
levels(d$Disease)[18] <- "H flu <5 unknown"
levels(d$Disease)[19] <- "Hansen Disease"
levels(d$Disease)[20] <- "HUS,postdiarrheal"
levels(d$Disease)[21] <- "HBV,perinatal"
levels(d$Disease)[22] <- "Influenza ped mort"
levels(d$Disease)[25] <- "Measles"
levels(d$Disease)[26] <- "Mening a,c,y,w-135"
levels(d$Disease)[27] <- "Mening other"
levels(d$Disease)[28] <- "Mening serogroup b"
levels(d$Disease)[29] <- "Mening unknown"
levels(d$Disease)[30] <- "Novel influenza A"
levels(d$Disease)[32] <- "Polio nonparalytic"
levels(d$Disease)[34] <- "Psittacosis"
levels(d$Disease)[37] <- "Q Fever, Total"
levels(d$Disease)[39] <- "SARS-CoV"
levels(d$Disease)[40] <- "Smallpox"
levels(d$Disease)[41] <- "Strep toxic shock synd"
levels(d$Disease)[42] <- "Syphilis congenital <1yr"
levels(d$Disease)[42] <- "Toxic shock synd staph"
levels(d$Disease)[47] <- "Vanco Interm Staph A"
levels(d$Disease)[48] <- "Vanco Resist Staph A"
d$threshold[is.na(d$threshold)]<-d$c[is.na(d$threshold)]
d$alert[is.na(d$alert)]<-"N"
write.table(d, file="infreq.txt", row.names=FALSE, col.names=TRUE)
write.table(unique(d$Disease), file="inf_dis.txt",row.names=FALSE, col.names=TRUE)
shiny::runApp()
pi_names <- read.table("pi_names.txt", header=T,colClasses=c("character","character"))
pi_names$region
shiny::runApp()
sort(filter(pi_names, type=="region")$location)
sort(filter(location_names, type=="region")$location)
location_names <- read.table("location_names.txt", header=T, colClasses=c("character","character"))
sort(filter(location_names, type=="region")$location)
shiny::runApp()
?toupper
toupper(pi_names$location[which(pi_names$type=="region")])
pi_names$location[which(pi_names$type=="region")]
shiny::runApp()
pi_names$type
pi_names$location
pi_names$location[which(pi_names$type=="region")]<- toupper(pi_names$location[which(pi_names$type=="region")])
pi_names$location
filter(cdcdata, display_name == "P&I MORT", Reporting.Area %in% pi_names$location[which(pi_names$type=="region")])
cdcdata <- read.table("plotdat.txt", header=T)
filter(cdcdata, display_name == "P&I MORT", Reporting.Area %in% pi_names$location[which(pi_names$type=="region")])
pi_names$location[which(pi_names$type=="region")]
unique(cdcdata$Reporting.Area)
pi_names <- read.table("pi_names.txt", header=T,colClasses=c("character","character"))
pi_names
shiny::runApp()
cdcdata$Reporting.Area
location_names
toupper(cdcdata$Reporting.Area[toupper(cdcdata$Reporting.Area)%in%location_names$region])
shiny::runApp()
infreq <- read.table("infreq.txt", header=T)
infreq
head(infreq)
head()
head(d)
d <- d %>% group_by(Disease, MMWR.Year) %>% mutate(cumulate=cumsum(c),
cumu14=cumulate+(threshold-c))
?mutate
d <- d %>% group_by(Disease, MMWR.year) %>% mutate(cumulate=cumsum(c),
cumu14=cumulate+(threshold-c))
d$Disease <- as.factor(d$Disease)
levels(d$Disease)[3] <- "Arbo,EEE"
levels(d$Disease)[2] <- "Arbo,CA serogroup"
levels(d$Disease)[4] <- "Arbo,Powassan"
levels(d$Disease)[5] <- "Arbo,St Louis"
levels(d$Disease)[6] <- "Arbo,WEE"
levels(d$Disease)[9] <- "Botulism other"
levels(d$Disease)[14] <- "Cyclosporiasis"
levels(d$Disease)[16] <- "H flu <5 non-b"
levels(d$Disease)[17] <- "H flu <5 b"
levels(d$Disease)[18] <- "H flu <5 unknown"
levels(d$Disease)[19] <- "Hansen Disease"
levels(d$Disease)[20] <- "HUS,postdiarrheal"
levels(d$Disease)[21] <- "HBV,perinatal"
levels(d$Disease)[22] <- "Influenza ped mort"
levels(d$Disease)[25] <- "Measles"
levels(d$Disease)[26] <- "Mening a,c,y,w-135"
levels(d$Disease)[27] <- "Mening other"
levels(d$Disease)[28] <- "Mening serogroup b"
levels(d$Disease)[29] <- "Mening unknown"
levels(d$Disease)[30] <- "Novel influenza A"
levels(d$Disease)[32] <- "Polio nonparalytic"
levels(d$Disease)[34] <- "Psittacosis"
levels(d$Disease)[37] <- "Q Fever, Total"
levels(d$Disease)[39] <- "SARS-CoV"
levels(d$Disease)[40] <- "Smallpox"
levels(d$Disease)[41] <- "Strep toxic shock synd"
levels(d$Disease)[42] <- "Syphilis congenital <1yr"
levels(d$Disease)[42] <- "Toxic shock synd staph"
levels(d$Disease)[47] <- "Vanco Interm Staph A"
levels(d$Disease)[48] <- "Vanco Resist Staph A"
d$threshold[is.na(d$threshold)]<-d$c[is.na(d$threshold)]
d$alert[is.na(d$alert)]<-"N"
write.table(d, file="infreq.txt", row.names=FALSE, col.names=TRUE)
write.table(unique(d$Disease), file="inf_dis.txt",row.names=FALSE, col.names=TRUE)
shiny::runApp()
d <- nndss %>% group_by(Disease, MMWR.week, MMWR.year)%>%summarise(c=Current.week)
d$c <- as.numeric(d$c)
d <- d %>% mutate(c = ifelse(is.na(c),0,c))
d <- d %>% group_by(Disease) %>% mutate("fourteenweekmean"=runmean(c, 14, align="right"))
d <- d %>% mutate(fourteenweekmean = ifelse(is.na(fourteenweekmean),0,fourteenweekmean))
d <- d%>% group_by(Disease) %>% mutate ("fourteenweeksd"=runsd(c, 14,center=runmean(c,14),align="right"))
d$sd2 <- d$fourteenweeksd*2
d$threshold <- d$fourteenweekmean + d$sd2
d$alert <- ifelse(d$c > d$threshold, "Y", "N")
d <- d %>% group_by(Disease, MMWR.year) %>% mutate(cumulate=cumsum(c),
cumu14=cumulate+(threshold-c))
d$Disease <- as.factor(d$Disease)
levels(d$Disease)[3] <- "Arbo,EEE"
levels(d$Disease)[2] <- "Arbo,CA serogroup"
levels(d$Disease)[4] <- "Arbo,Powassan"
levels(d$Disease)[5] <- "Arbo,St Louis"
levels(d$Disease)[6] <- "Arbo,WEE"
levels(d$Disease)[9] <- "Botulism other"
levels(d$Disease)[14] <- "Cyclosporiasis"
levels(d$Disease)[16] <- "H flu <5 non-b"
levels(d$Disease)[17] <- "H flu <5 b"
levels(d$Disease)[18] <- "H flu <5 unknown"
levels(d$Disease)[19] <- "Hansen Disease"
levels(d$Disease)[20] <- "HUS,postdiarrheal"
levels(d$Disease)[21] <- "HBV,perinatal"
levels(d$Disease)[22] <- "Influenza ped mort"
levels(d$Disease)[25] <- "Measles"
levels(d$Disease)[26] <- "Mening a,c,y,w-135"
levels(d$Disease)[27] <- "Mening other"
levels(d$Disease)[28] <- "Mening serogroup b"
levels(d$Disease)[29] <- "Mening unknown"
levels(d$Disease)[30] <- "Novel influenza A"
levels(d$Disease)[32] <- "Polio nonparalytic"
levels(d$Disease)[34] <- "Psittacosis"
levels(d$Disease)[37] <- "Q Fever, Total"
levels(d$Disease)[39] <- "SARS-CoV"
levels(d$Disease)[40] <- "Smallpox"
levels(d$Disease)[41] <- "Strep toxic shock synd"
levels(d$Disease)[42] <- "Syphilis congenital <1yr"
levels(d$Disease)[42] <- "Toxic shock synd staph"
levels(d$Disease)[47] <- "Vanco Interm Staph A"
levels(d$Disease)[48] <- "Vanco Resist Staph A"
d$threshold[is.na(d$threshold)]<-d$c[is.na(d$threshold)]
d$alert[is.na(d$alert)]<-"N"
write.table(d, file="infreq.txt", row.names=FALSE, col.names=TRUE)
write.table(unique(d$Disease), file="inf_dis.txt",row.names=FALSE, col.names=TRUE)
shiny::runApp()
pi_names
shiny::runApp()
?h4
shiny::runApp()
shiny::runApp()
URL <- "https://ibis.health.state.nm.us/resources/MMWRWeekCalendar.html"
getURL(URL,ssl.verifypeer=FALSE))
getURL(URL,ssl.verifypeer=FALSE)
library(XML)
readHTMLTable(URL)
htmlParse(URL)
URL <- "https://ibis.health.state.nm.us/resources/MMWRWeekCalendar.html"
htmlParse(URL)
getURL(URL)
dates <-getURL(URL)
readHTMLTable(dates)
htmlParse(dates)
readHTMLTable(dates)
readHTMLTable(dates)$'NULL'
str(readHTMLTable(dates))
readHTMLTable(dates)[[7]]
URL <- "https://ibis.health.state.nm.us/resources/MMWRWeekCalendar.html"
dates <-getURL(URL)
dates <- readHTMLTable(dates)[[7]]
str(dates)
library(lubridate)
?readHTMLTable
dates <- readHTMLTable(dates, header=T)[[7]]
dates <-getURL(URL)
dates <- readHTMLTable(dates, header=T)[[7]]
dates
names(dates) <- c("MMWR.Week", "2011", "2012", "2013", "2014", "2015")
dates
dates <- dates[-1,]
dates
dates$2014
names(dates) <- c("MMWR.Week", "y2011", "y2012", "y2013", "y2014", "y2015")
dates$y2014
?which
getwd()
write.table(dates, file="dates.txt", row.names=FALSE, col.names=TRUE)
dates <- read.table("dates.txt", header=T)
dates
newthresh <- function(x,days){
thresh <-runmean(x, days,endrule="NA")+2*runsd(x, days,endrule="sd",align="right")
thresh[is.na(thresh)]<-x[is.na(thresh)]
return(thresh)
}
# This function takes each url and corresponding disease name and gets data from CDC.  It then combines multiple years worth of data,
# calculates alert thresholds and cumulative sums and returns the columns of interest from the CDC data.
# Args:
#   url_data: the rows of the url_data.txt file which contain the urls for a given disease
url_func <- function(url_data){
# Construct actual CDC website url name and get data for 2014 and 2015
curl <- url_data$url
URL <- paste( "https://data.cdc.gov/api/views/",curl, "/rows.csv?accessType=DOWNLOAD",sep="")
nndss14 <-read.csv(textConnection(getURL(URL[1],ssl.verifypeer=FALSE)),strip.white=T,stringsAsFactors=F)
nndss15 <- read.csv(textConnection(getURL(URL[2],ssl.verifypeer=FALSE)),strip.white=T,stringsAsFactors=F)
# Some diseases have a slightly different name for MMWR.Week and MMWR.Year, so we standardize the names here
if("MMWRWeek"%in%names(nndss14)){nndss14<- dplyr::rename(nndss14,MMWR.Week=MMWRWeek )}
if("MMWR.WEEK"%in%names(nndss14)){nndss14<- dplyr::rename(nndss14,MMWR.Week=MMWR.WEEK )}
if("MMWRYear"%in%names(nndss14)){nndss14<- dplyr::rename(nndss14, MMWR.Year=MMWRYear )}
if("MMWR.YEAR"%in%names(nndss14)){nndss14<- dplyr::rename(nndss14, MMWR.Year=MMWR.YEAR )}
if("MMWRWeek"%in%names(nndss15)){nndss15<- dplyr::rename(nndss15, MMWR.Week=MMWRWeek  )}
if("MMWR.WEEK"%in%names(nndss15)){nndss15<- dplyr::rename(nndss15, MMWR.Week=MMWR.WEEK  )}
if("MMWRYear"%in%names(nndss15)){nndss15<- dplyr::rename(nndss15,  MMWR.Year=MMWRYear )}
if("MMWR.YEAR"%in%names(nndss15)){nndss15<- dplyr::rename(nndss15,  MMWR.Year=MMWR.YEAR )}
# dname is the name of the column in the nndss file which contains weekly data for the disease of interest
dname <- c(paste(url_data$data_name[1],"..Current.week",sep=""))
#special column name for P&I mortality data
if(url_data$data_name[1]=="P&I MORT")dname <- "P.I..Total"
# Select relevant columns from both the 2014 and 2015 data and rbind them together
nndss <- rbind(select(nndss14, contains(dname), contains("MMWR"), contains("Reporting"), -contains("flag")),
select(nndss15, contains(dname), contains("MMWR"), contains("Reporting"), -contains("flag")))
# set NA values to 0, maybe not a great idea, but useful for calculating thresholds and cumulative sums
names(nndss)[which(dname==names(nndss))] <- "c"
nndss$c <- as.numeric(nndss$c)
nndss$c[is.na(nndss$c)]<-0
nndss$display_name <- url_data$display_name[1]
# Create columns for 10 and 14 week thresholds and 10 and 14 week alerts, grouping by reporting area.
nndss <- nndss %>% group_by(Reporting.Area) %>% mutate(fourteenwk.thresh=newthresh(c,14),
tenwk.thresh=newthresh(c,10),
fourteenwk.alert=c>fourteenwk.thresh,
tenwk.alert=c>tenwk.thresh)
# Create columns for cumulative sum along with cumulative threshold values, grouping both by reporting area and year
nndss <- group_by(nndss, Reporting.Area, MMWR.Year) %>% mutate(cumulate=cumsum(c),
cumu10=cumulate+(tenwk.thresh-c),
cumu14=cumulate+(fourteenwk.thresh-c))
#select and return relevant columns of data table
nndss<- select(nndss, one_of("c","Reporting.Area", "MMWR.Year", "MMWR.Week","display_name"), contains("thresh"),contains("cumu"),contains("alert"))
nndss$date <- apply(nndss,1, function(x){return(dates[which(dates$MMWR.Week==x$MMWR.Week),5+(x$MMWR.Year-2014)])}
return(nndss)
}
# Run the url_func function for each different disease name in our urldat.txt data file
output <- ddply(urldat, .(data_name), url_func)
url_func <- function(url_data){
# Construct actual CDC website url name and get data for 2014 and 2015
curl <- url_data$url
URL <- paste( "https://data.cdc.gov/api/views/",curl, "/rows.csv?accessType=DOWNLOAD",sep="")
nndss14 <-read.csv(textConnection(getURL(URL[1],ssl.verifypeer=FALSE)),strip.white=T,stringsAsFactors=F)
nndss15 <- read.csv(textConnection(getURL(URL[2],ssl.verifypeer=FALSE)),strip.white=T,stringsAsFactors=F)
# Some diseases have a slightly different name for MMWR.Week and MMWR.Year, so we standardize the names here
if("MMWRWeek"%in%names(nndss14)){nndss14<- dplyr::rename(nndss14,MMWR.Week=MMWRWeek )}
if("MMWR.WEEK"%in%names(nndss14)){nndss14<- dplyr::rename(nndss14,MMWR.Week=MMWR.WEEK )}
if("MMWRYear"%in%names(nndss14)){nndss14<- dplyr::rename(nndss14, MMWR.Year=MMWRYear )}
if("MMWR.YEAR"%in%names(nndss14)){nndss14<- dplyr::rename(nndss14, MMWR.Year=MMWR.YEAR )}
if("MMWRWeek"%in%names(nndss15)){nndss15<- dplyr::rename(nndss15, MMWR.Week=MMWRWeek  )}
if("MMWR.WEEK"%in%names(nndss15)){nndss15<- dplyr::rename(nndss15, MMWR.Week=MMWR.WEEK  )}
if("MMWRYear"%in%names(nndss15)){nndss15<- dplyr::rename(nndss15,  MMWR.Year=MMWRYear )}
if("MMWR.YEAR"%in%names(nndss15)){nndss15<- dplyr::rename(nndss15,  MMWR.Year=MMWR.YEAR )}
# dname is the name of the column in the nndss file which contains weekly data for the disease of interest
dname <- c(paste(url_data$data_name[1],"..Current.week",sep=""))
#special column name for P&I mortality data
if(url_data$data_name[1]=="P&I MORT")dname <- "P.I..Total"
# Select relevant columns from both the 2014 and 2015 data and rbind them together
nndss <- rbind(select(nndss14, contains(dname), contains("MMWR"), contains("Reporting"), -contains("flag")),
select(nndss15, contains(dname), contains("MMWR"), contains("Reporting"), -contains("flag")))
# set NA values to 0, maybe not a great idea, but useful for calculating thresholds and cumulative sums
names(nndss)[which(dname==names(nndss))] <- "c"
nndss$c <- as.numeric(nndss$c)
nndss$c[is.na(nndss$c)]<-0
nndss$display_name <- url_data$display_name[1]
# Create columns for 10 and 14 week thresholds and 10 and 14 week alerts, grouping by reporting area.
nndss <- nndss %>% group_by(Reporting.Area) %>% mutate(fourteenwk.thresh=newthresh(c,14),
tenwk.thresh=newthresh(c,10),
fourteenwk.alert=c>fourteenwk.thresh,
tenwk.alert=c>tenwk.thresh)
# Create columns for cumulative sum along with cumulative threshold values, grouping both by reporting area and year
nndss <- group_by(nndss, Reporting.Area, MMWR.Year) %>% mutate(cumulate=cumsum(c),
cumu10=cumulate+(tenwk.thresh-c),
cumu14=cumulate+(fourteenwk.thresh-c))
#select and return relevant columns of data table
nndss<- select(nndss, one_of("c","Reporting.Area", "MMWR.Year", "MMWR.Week","display_name"), contains("thresh"),contains("cumu"),contains("alert"))
nndss$date <- apply(nndss,1, function(x){return(dates[which(dates$MMWR.Week==x$MMWR.Week),5+(x$MMWR.Year-2014)])})
return(nndss)
}
output <- ddply(urldat, .(data_name), url_func)
dates$MMWR.Week
?apply
url_func <- function(url_data){
# Construct actual CDC website url name and get data for 2014 and 2015
curl <- url_data$url
URL <- paste( "https://data.cdc.gov/api/views/",curl, "/rows.csv?accessType=DOWNLOAD",sep="")
nndss14 <-read.csv(textConnection(getURL(URL[1],ssl.verifypeer=FALSE)),strip.white=T,stringsAsFactors=F)
nndss15 <- read.csv(textConnection(getURL(URL[2],ssl.verifypeer=FALSE)),strip.white=T,stringsAsFactors=F)
# Some diseases have a slightly different name for MMWR.Week and MMWR.Year, so we standardize the names here
if("MMWRWeek"%in%names(nndss14)){nndss14<- dplyr::rename(nndss14,MMWR.Week=MMWRWeek )}
if("MMWR.WEEK"%in%names(nndss14)){nndss14<- dplyr::rename(nndss14,MMWR.Week=MMWR.WEEK )}
if("MMWRYear"%in%names(nndss14)){nndss14<- dplyr::rename(nndss14, MMWR.Year=MMWRYear )}
if("MMWR.YEAR"%in%names(nndss14)){nndss14<- dplyr::rename(nndss14, MMWR.Year=MMWR.YEAR )}
if("MMWRWeek"%in%names(nndss15)){nndss15<- dplyr::rename(nndss15, MMWR.Week=MMWRWeek  )}
if("MMWR.WEEK"%in%names(nndss15)){nndss15<- dplyr::rename(nndss15, MMWR.Week=MMWR.WEEK  )}
if("MMWRYear"%in%names(nndss15)){nndss15<- dplyr::rename(nndss15,  MMWR.Year=MMWRYear )}
if("MMWR.YEAR"%in%names(nndss15)){nndss15<- dplyr::rename(nndss15,  MMWR.Year=MMWR.YEAR )}
# dname is the name of the column in the nndss file which contains weekly data for the disease of interest
dname <- c(paste(url_data$data_name[1],"..Current.week",sep=""))
#special column name for P&I mortality data
if(url_data$data_name[1]=="P&I MORT")dname <- "P.I..Total"
# Select relevant columns from both the 2014 and 2015 data and rbind them together
nndss <- rbind(select(nndss14, contains(dname), contains("MMWR"), contains("Reporting"), -contains("flag")),
select(nndss15, contains(dname), contains("MMWR"), contains("Reporting"), -contains("flag")))
# set NA values to 0, maybe not a great idea, but useful for calculating thresholds and cumulative sums
names(nndss)[which(dname==names(nndss))] <- "c"
nndss$c <- as.numeric(nndss$c)
nndss$c[is.na(nndss$c)]<-0
nndss$display_name <- url_data$display_name[1]
# Create columns for 10 and 14 week thresholds and 10 and 14 week alerts, grouping by reporting area.
nndss <- nndss %>% group_by(Reporting.Area) %>% mutate(fourteenwk.thresh=newthresh(c,14),
tenwk.thresh=newthresh(c,10),
fourteenwk.alert=c>fourteenwk.thresh,
tenwk.alert=c>tenwk.thresh)
# Create columns for cumulative sum along with cumulative threshold values, grouping both by reporting area and year
nndss <- group_by(nndss, Reporting.Area, MMWR.Year) %>% mutate(cumulate=cumsum(c),
cumu10=cumulate+(tenwk.thresh-c),
cumu14=cumulate+(fourteenwk.thresh-c))
#select and return relevant columns of data table
nndss<- select(nndss, one_of("c","Reporting.Area", "MMWR.Year", "MMWR.Week","display_name"), contains("thresh"),contains("cumu"),contains("alert"))
nndss$date <- apply(nndss,1, function(x){return(dates[which(datesMMWR.Week==x['MMWR.Week']),5+(x['MMWR.Year']-2014)])})
return(nndss)
}
output <- ddply(urldat, .(data_name), url_func)