# Kołczyńska, M. (2022). Combining multiple survey sources: A reproducible workflow and toolbox 
# for survey data harmonization. Methodological Innovations, 15(1), 62-72. 
# https://doi.org/10.1177/20597991221077923


# 1. Functions ---------

# creates a dictionary from labelled data with response frequencies (takes a while to run)
create_codebook <- function(data) {
  var_labels <- data.frame(cbind(names(var_label(data)), do.call(rbind, var_label(data)))) %>%
    rename(var_name = X1, var_label = X2)
  
  freqs <- lapply(data, function(x) { return(questionr::freq(x)) }) %>%
    keep(function(x) nrow(x) < 1000) %>%
    do.call(rbind, .) %>% 
    tibble::rownames_to_column(var = "var_name_value") %>%
    mutate(var_name = gsub("(.+?)(\\..*)", "\\1", var_name_value),
           value = gsub("^[^.]*.","",var_name_value)) %>%
    group_by(var_name) %>%
    mutate(npos = row_number(),
           value_n = paste(value, n, sep = ": ")) %>%
    select(var_name, value_n, npos) %>%
    spread(npos, value_n) %>%
    mutate_at(vars(-var_name), funs(ifelse(is.na(.), "", .))) %>%
    unite("val_freqs", c(2:ncol(.)), sep = "\n") %>%
    mutate(val_freqs = sub("\\s+$", "", val_freqs))
  
  full_join(var_labels, freqs, by = "var_name")
}

# creates a dictionary without response frequencies (very fast)
create_codebook_short <- function(data) {
  var_labels <- data.frame(cbind(names(var_label(data)), do.call(rbind, var_label(data)))) %>%
    rename(var_name = X1, var_label = X2) %>%
    mutate(var_name = as.character(var_name),
           var_label = as.character(var_label))%>% 
    rename(var_source = var_name, var_label_source = var_label) #* NEW
}

# creates a crosswalk from labelled data (mapping table)
create_cwt <- function(data) {
  
  # data = eb_small
  # head(data)

  var_labels <- data.frame(cbind(names(var_label(data)), do.call(rbind, var_label(data)))) %>%
    rename(var_name = X1, var_label = X2)
  var_labels
  
  freqs_cwt0 <- lapply(data, function(x) { return(questionr::freq(x)) }) %>% # this is a list with elements=varb
    keep(function(x) nrow(x) < 1000) %>% # 1000 is the nrow of each element of the list, which is number of distinct values for the varb
    do.call(rbind, .) %>% 
    tibble::rownames_to_column(var = "var_name_value") %>%
    mutate(var_name = gsub("(.+?)(\\..*)", "\\1", var_name_value),
           value = gsub("^[^.]*.","",var_name_value))
  head(freqs_cwt0)
  
  (x <- freqs_cwt0[freqs_cwt0$value== "", "var_name"]) #new*
  
  freqs_cwt <- freqs_cwt0 %>% 
    mutate(value = ifelse(value == "", unique(data[, x]), value), #new*
           value_code = sub(".*\\[(\\S+?)\\].*$", "\\1", var_name_value, perl = TRUE),
           # value_code = ifelse(str_sub(var_name_value, -2, -1) == "NA", "NA", value_code),
           value_code = ifelse(gsub(" ", "", fixed = TRUE, var_name_value) == var_name_value, 
                               gsub("^[^.]*.","",var_name_value), value_code),
           value_code = ifelse(value_code == "", value, value_code)) %>% #new*
    group_by(var_name) %>%
    mutate(npos = row_number(),
           value_n = paste(value, n, sep = ": ")) %>%
    select(npos, var_name, value_n, value, value_code) 
  head(freqs_cwt)
  #view(freqs_cwt)
  
  full_join(var_labels, freqs_cwt, by = "var_name") %>% 
    rename(var_source = var_name, var_label_source = var_label, value_label_source = value) #new*
  
}


# ess <- readRDS("DATA/ess_1_10.rds")

# codebook_ess <- create_codebook_short(ess) %>%
#   mutate(table_name = "ESS_1_10",
#          target_var = ifelse( var_name == "cntry", "country", NA),
#          target_var = ifelse( var_name == "essround", "wave", target_var),
#          target_var = ifelse( grepl("Trust in", var_label), 
#                               gsub("Trust in |Trust in the ", "", var_label), 
#                               target_var),
#          target_var = gsub("country's", "national", target_var),
#          target_var = gsub(" ", "_", target_var),
#          target_var = tolower(target_var))