rm(list=ls(all=t))

Setup filenames

filename <- "Admin Data - Ecuador_Public Use" # !!!Update filename
functions_vers <-  "functions_1.7.R" # !!!Update helper functions file

Setup data, functions and create dictionary for dataset review

source (functions_vers)
## --------
## This is sdcMicro v5.6.0.
## For references, please have a look at citation('sdcMicro')
## Note: since version 5.0.0, the graphical user-interface is a shiny-app that can be started with sdcApp().
## Please submit suggestions and bugs at: https://github.com/sdcTools/sdcMicro/issues
## --------
## 
## Attaching package: 'dplyr'
## The following objects are masked from 'package:stats':
## 
##     filter, lag
## The following objects are masked from 'package:base':
## 
##     intersect, setdiff, setequal, union
## Loading required package: sp
## Checking rgeos availability: TRUE
## 
## Attaching package: 'raster'
## The following object is masked from 'package:dplyr':
## 
##     select
## The following object is masked from 'package:sdcMicro':
## 
##     freq
## rgdal: version: 1.5-23, (SVN revision 1121)
## Geospatial Data Abstraction Library extensions to R successfully loaded
## Loaded GDAL runtime: GDAL 3.2.1, released 2020/12/29
## Path to GDAL shared files: C:/Users/C_Pablo_Diego-Rosell/Documents/R/R-3.6.3/library/rgdal/gdal
## GDAL binary built with GEOS: TRUE 
## Loaded PROJ runtime: Rel. 7.2.1, January 1st, 2021, [PJ_VERSION: 721]
## Path to PROJ shared files: C:/Users/C_Pablo_Diego-Rosell/Documents/R/R-3.6.3/library/rgdal/proj
## PROJ CDN enabled: FALSE
## Linking to sp version:1.4-5
## To mute warnings of possible GDAL/OSR exportToProj4() degradation,
## use options("rgdal_show_exportToProj4_warnings"="none") before loading rgdal.
## Overwritten PROJ_LIB was C:/Users/C_Pablo_Diego-Rosell/Documents/R/R-3.6.3/library/rgdal/proj
## Loading required package: spatstat.data
## Loading required package: spatstat.geom
## spatstat.geom 2.0-1
## 
## Attaching package: 'spatstat.geom'
## The following objects are masked from 'package:raster':
## 
##     area, rotate, shift
## Loading required package: spatstat.core
## Loading required package: nlme
## 
## Attaching package: 'nlme'
## The following object is masked from 'package:raster':
## 
##     getData
## The following object is masked from 'package:dplyr':
## 
##     collapse
## Loading required package: rpart
## spatstat.core 2.0-0
## Loading required package: spatstat.linnet
## spatstat.linnet 2.1-1
## 
## spatstat 2.0-1       (nickname: 'Caution: contains small parts') 
## For an introduction to spatstat, type 'beginner'
## rgeos version: 0.5-5, (SVN revision 640)
##  GEOS runtime version: 3.8.0-CAPI-1.13.1 
##  Linking to sp version: 1.4-4 
##  Polygon checking: TRUE
## 
## Spatial Point Pattern Analysis Code in S-Plus
##  
##  Version 2 - Spatial and Space-Time analysis
## 
## Attaching package: 'splancs'
## The following object is masked from 'package:raster':
## 
##     zoom
## The following object is masked from 'package:dplyr':
## 
##     tribble
## Loading required package: spam
## Loading required package: dotCall64
## Loading required package: grid
## Spam version 2.6-0 (2020-12-14) is loaded.
## Type 'help( Spam)' or 'demo( spam)' for a short introduction 
## and overview of this package.
## Help for individual functions is also obtained by adding the
## suffix '.spam' to the function name, e.g. 'help( chol.spam)'.
## 
## Attaching package: 'spam'
## The following objects are masked from 'package:base':
## 
##     backsolve, forwardsolve
## See https://github.com/NCAR/Fields for
##  an extensive vignette, other supplements and source code
## 
## Attaching package: 'geosphere'
## The following object is masked from 'package:spatstat.geom':
## 
##     perimeter
## 
## Attaching package: 'tibble'
## The following object is masked from 'package:splancs':
## 
##     tribble

Visually inspect variables in "dictionary.csv" and flag for risk, using the following flags:

# Direct PII: Respondent Names, Addresses, Identification Numbers, Phone Numbers
# Direct PII-team: Interviewer Names, other field team names 
# Indirect PII-ordinal: Date of birth, Age, income, education, household composition. 
# Indirect PII-categorical: Gender, education, ethnicity, nationality,
# occupation, employer, head of household, marital status
# GPS: Longitude, Latitude
# Small Location: Location (<100,000) 
# Large Location (>100,000)
# Weight: weightVar
# Household ID:  hhId, 
# Open-ends: Review responses for any sensitive information, redact as necessary 

#!!!Save flagged dictionary in .csv format, add "DatasetReview" to name and continue processing data with subset of flagged variables

Direct PII: variables to be removed

# !!!No Direct PII

Direct PII-team: Encode field team names

# !!!No Direct PII-team

Small locations: Encode locations with pop <100,000 using random large numbers

# !!!No small location

Indirect PII - Ordinal: Global recode or Top/bottom coding for extreme values

# Focus on variables with a "Lowest Freq" in dictionary of 30 or less. 
# Top code household composition variables with large and unusual numbers 

mydata <- top_recode ("b_hhsize_2016_masked", break_point=10, missing=c(888, 999999))  # Topcode cases with 10 or more individuals.
## [1] "Frequency table before encoding"
## b_hhsize_2016_masked. Household Size
## 2 or less         3         4         5         6         7         8         9        10       11+      <NA> 
##        27        81       117       186       112        73        33        18        14        18        47

## [1] "Frequency table after encoding"
## b_hhsize_2016_masked. Household Size
##  2 or less          3          4          5          6          7          8          9 10 or more       <NA> 
##         27         81        117        186        112         73         33         18         32         47

mydata <- top_recode ("b_hhsize_2016_imp", break_point=10, missing=c(888, 999999)) # Topcode cases with 10 or more individuals.
## [1] "Frequency table before encoding"
## b_hhsize_2016_imp. Imputed Version Of B_Hhsize_2016
##                1                2                3                4                5 5.42120742797852 
##                2               25               81              117              186               47 
##                6                7                8                9               10               11 
##              112               73               33               18               14                6 
##               12               13               14               16               17               29 
##                7                1                1                1                1                1

## [1] "Frequency table after encoding"
## b_hhsize_2016_imp. Imputed Version Of B_Hhsize_2016
##                1                2                3                4                5 5.42120742797852 
##                2               25               81              117              186               47 
##                6                7                8                9       10 or more 
##              112               73               33               18               32

Indirect PII - Categorical: Recode, encode, or Top/bottom coding for extreme values

# !!!Include relevant variables in list below (Indirect PII - Categorical, and Ordinal if not processed yet)

indirect_PII <- c("female",
                  "age_at_base",
                  "e_fasis",
                  "e_ffinjust",
                  "e_ffjust",
                  "e_pdfinal",
                  "e_promotion_status",
                  "e_qanual",
                  "b_white_2016",
                  "b_mestizo_2016",
                  "b_afro_2016",
                  "b_montub_2016",
                  "b_indig_2016",
                  "b_ownchild_2016",
                  "b_married_2016",
                  "b_mspan_2016",
                  "b_mnat_2016",
                  "b_fspan_2016",
                  "b_fnat_2016",
                  "b_sameage_2016",
                  "b_younger_2016",
                  "b_older_2016",
                  "b_yearsout_2016",
                  "b_othrace_2016",
                  "b_afro_2016_imp",
                  "b_fspan_2016_imp",
                  "b_married_2016_imp",
                  "b_mestizo_2016_imp",
                  "b_mspan_2016_imp",
                  "b_older_2016_imp",
                  "b_othrace_2016_imp",
                  "b_ownchild_2016_imp",
                  "b_sameage_2016_imp",
                  "b_white_2016_imp",
                  "b_yearsout_2016_imp",
                  "b_younger_2016_imp")

capture_tables (indirect_PII)


# Recode those with very specific values. 

# Combine races' variables in  other races' variable in order to reduce risk of re-identification.

mydata$b_othrace_2016 <- cbind(mydata$b_montub_2016,mydata$b_indig_2016, mydata$b_othrace_2016)
mydata <- mydata[!names(mydata) %in% c("b_indig_2016", "b_montub_2016")]

Matching and crosstabulations: Run automated PII check

# Based on dictionary inspection, select variables for creating sdcMicro object
# See: https://sdcpractice.readthedocs.io/en/latest/anon_methods.html
# All variable names should correspond to the names in the data file
# selected categorical key variables: gender, occupation/education and age
selectedKeyVars = c('female', 'age_at_base', 'b_white_2016') ##!!! Replace with candidate categorical demo vars

# weight variable (add if available)
# selectedWeightVar = c('projwt') ##!!! Replace with weight var

# household id variable (cluster)
# selectedHouseholdID = c('wpid') ##!!! Replace with household id

# creating the sdcMicro object with the assigned variables
sdcInitial <- createSdcObj(dat = mydata, keyVars = selectedKeyVars)
sdcInitial
## The input dataset consists of 726 rows and 74 variables.
##   --> Categorical key variables: female, age_at_base, b_white_2016
## ----------------------------------------------------------------------
## Information on categorical key variables:
## 
## Reported is the number, mean size and size of the smallest category >0 for recoded variables.
## In parenthesis, the same statistics are shown for the unmodified data.
## Note: NA (missings) are counted as seperate categories!
##  Key Variable Number of categories     Mean size           Size of smallest (>0)      
##        female                    2 (2)   363.000 (363.000)                   275 (275)
##   age_at_base                    4 (4)   181.500 (181.500)                    44  (44)
##  b_white_2016                    3 (3)   341.500 (341.500)                    51  (51)
## ----------------------------------------------------------------------
## Infos on 2/3-Anonymity:
## 
## Number of observations violating
##   - 2-anonymity: 0 (0.000%)
##   - 3-anonymity: 0 (0.000%)
##   - 5-anonymity: 0 (0.000%)
## 
## ----------------------------------------------------------------------
# All cases meet 2-anonimity

Open-ends: review responses for any sensitive information, redact as necessary

# !!! Identify open-end variables here: 
open_ends <- c("e_fecom")
report_open (list_open_ends = open_ends)
mydata <- mydata[!names(mydata) %in% open_ends]

# Review "verbatims.csv". Identify variables to be deleted or redacted and their row number 

GPS data: Displace

# !!!No GPS data

Save processed data in Stata and SPSS format

Adds "_PU" (Public Use) to the end of the name

haven::write_dta(mydata, paste0(filename, "_PU.dta"))
haven::write_sav(mydata, paste0(filename, "_PU.sav"))

# Add report title dynamically
title_var <- paste0("DOL-ILAB SDC - ", filename)