rm(list=ls(all=t))

Setup filenames

filename <- "bcsection3_relabelled" # !!!Update filename
functions_vers <-  "functions_1.7.R" # !!!Update helper functions file

Setup data, functions and create dictionary for dataset review

source (functions_vers)

Visually inspect variables in "dictionary.csv" and flag for risk, using the following flags:

# Direct PII: Respondent Names, Addresses, Identification Numbers, Phone Numbers
# Direct PII-team: Interviewer Names, other field team names 
# Indirect PII-ordinal: Date of birth, Age, income, education, household composition. 
# Indirect PII-categorical: Gender, education, ethnicity, nationality,
# occupation, employer, head of household, marital status
# GPS: Longitude, Latitude
# Small Location: Location (<100,000) 
# Large Location (>100,000)
# Weight: weightVar
# Household ID:  hhId, 
# Open-ends: Review responses for any sensitive information, redact as necessary

Direct PII: variables to be removed

# !!! No Direct PII

Direct PII-team: Encode field team names

# !!! No Direct PII-team

Small locations: Encode locations with pop <100,000 using random large numbers

dropvars <- c("dise") 
mydata <- mydata[!names(mydata) %in% dropvars]

locvars <- c("q002_blckid", "q003_vill_id") 
mydata <- encode_location (variables= locvars, missing=999999)
## [1] "Frequency table before encoding"
## q002_blckid. 2. Tehsil/Block ID
##   1   2   3   4   5   6   7   8   9 
## 210 152 192 414 100 197 158 426 550 
## [1] "Frequency table after encoding"
## q002_blckid. 2. Tehsil/Block ID
## 279 280 281 282 283 284 285 286 287 
## 426 152 192 550 414 100 210 158 197 
## [1] "Frequency table before encoding"
## q003_vill_id. 3. Village ID
##   1   2   3   4   5   6   7   9  10  11  12  13  14  15  16  17  18  19  20  21  22 
##  17  12  16  16  19  31  27  17  15  20  26  23   2  13  19  21  17  18  20  30  22 
##  23  24  25  26  27  28  29  30  31  32  33  34  35  36  37  38  39  40  41  42  43 
##  17  16  27  27  28  18  15  15  24  27  21  13  30  17  17  21  28  16  16  18  16 
##  44  45  46  47  48  49  50  51  52  53  54  55  56  57  58  59  60  61  62  63  64 
##  26  21  24  21  17  17  17  13  18  28  25  27  19  14  21  12  22   9  17  19  18 
##  65  66  67  68  69  70  71  72  73  74  75  76  77  78  80  81  82  83  84  85  86 
##  28  15  19  22  26  14  16  21  16  25  23  17  23  31  30  17  22  17  16  13  14 
##  87  88  89  90  91  92  93  94  95  96  97  98  99 100 101 102 103 104 105 106 107 
##  17  18  16  19  19  20  21  14  20  24  29  29  26  22  25  21  11  23  17  30  14 
## 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 
##  28  20  16  20  27  41  22  20  12  24  19  21  13  13  10 
## [1] "Frequency table after encoding"
## q003_vill_id. 3. Village ID
## 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 
##  29  16  12  21  23  18  16  23  10  25  18  15  31  19  28  17  20  17  17  17  21 
## 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 
##  20  19  19  25  22  15  31  17  22  28  22  24  17  14  24  26  30  21  20  26  13 
## 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 
##  29  11  19  27  28  15  14  24  21  19  14  20  21  13  15  27  16  13   2  18  17 
## 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 
##  20  17  21  17  13  20  30  23  25  18  27  18  22  27  12  16   9  18  17  26  27 
## 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 
##  17  17  17  21  21  41  17  21  28  20  12  14  16  21  24  13  30  30  19  16  17 
## 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 
##  27  22  14  19  16  19  16  13  22  16  23  26  28  16  16

Indirect PII - Ordinal: Global recode or Top/bottom coding for extreme values

# !!! No Direct PII - Ordinal

Indirect PII - Categrical: Recode, encode, or Top/bottom coding for extreme values

indirect_PII <- c("q301_own_farm",
"q302_help_farm",
"q303_paid",
"q304_own_busi",
"q305_spend_busi",
"q306_income",
"q307_work_not_fly",
"q308_paid",
"q309_accomp_mem",
"q310_type_work",
"q311_activity1",
"q311_activity2",
"q311_activity3",
"q311_activity4",
"q311_activity5",
"q311_activity6",
"q311_activity7",
"q311_activity8",
"q311_activity9",
"q311_activity10",
"q311_activity11",
"q311_activity12",
"q311_activity13",
"q311_activity14",
"q311_activity15",
"q311_activity16",
"q311_activity17",
"q311_activity18",
"q311_activity19",
"q311_activity20",
"q311_activity21",
"q311_activity22",
"q311_activity23",
"q311_activity24",
"q311_activity25",
"q311_activity26",
"q311_activity27",
"q311_activity28",
"q311_activity29",
"q311_activity30",
"q311_activity31",
"q311_activity32",
"q330_help_farm",
"q331_spend_busi",
"q332_work_not_fly")

capture_tables (indirect_PII)

Matching and crosstabulations: Run automated PII check

#!!!Demographic information available is insufficient for matching analysis.

Open-ends: review responses for any sensitive information, redact as necessary

open_ends <- c("q310_type_work_oth",
               "q311_act_oth1",
               "q311_act_oth2",
               "q311_act_oth3",
               "q311_act_oth4",
               "q311_act_oth5",
               "q311_act_oth6",
               "q311_act_oth7",
               "q311_act_oth8",
               "q311_act_oth9",
               "q311_act_oth10",
               "q311_act_oth11",
               "q311_act_oth12",
               "q311_act_oth13",
               "q311_act_oth14",
               "q311_act_oth15",
               "q311_act_oth16",
               "q311_act_oth17",
               "q311_act_oth18",
               "q311_act_oth19",
               "q311_act_oth20",
               "q311_act_oth21",
               "q311_act_oth22",
               "q311_act_oth23",
               "q311_act_oth24",
               "q311_act_oth25",
               "q311_act_oth26",
               "q311_act_oth27",
               "q311_act_oth28",
               "q311_act_oth29",
               "q311_act_oth30",
               "q311_act_oth31",
               "q311_act_oth32")

report_open (list_open_ends = open_ends)

# Review "verbatims.csv". Identify variables to be deleted or redacted and their row number 
# Drop all, as actually verbatim data in Hindi
mydata <- mydata[!names(mydata) %in% "q310_type_work_oth"]
mydata <- mydata[!names(mydata) %in% "q311_act_oth1"]
mydata <- mydata[!names(mydata) %in% "q311_act_oth2"]
mydata <- mydata[!names(mydata) %in% "q311_act_oth3"]
mydata <- mydata[!names(mydata) %in% "q311_act_oth4"]
mydata <- mydata[!names(mydata) %in% "q311_act_oth5"]
mydata <- mydata[!names(mydata) %in% "q311_act_oth6"]
mydata <- mydata[!names(mydata) %in% "q311_act_oth7"]
mydata <- mydata[!names(mydata) %in% "q311_act_oth8"]
mydata <- mydata[!names(mydata) %in% "q311_act_oth9"]
mydata <- mydata[!names(mydata) %in% "q311_act_oth10"]
mydata <- mydata[!names(mydata) %in% "q311_act_oth11"]
mydata <- mydata[!names(mydata) %in% "q311_act_oth12"]
mydata <- mydata[!names(mydata) %in% "q311_act_oth13"]
mydata <- mydata[!names(mydata) %in% "q311_act_oth14"]
mydata <- mydata[!names(mydata) %in% "q311_act_oth15"]
mydata <- mydata[!names(mydata) %in% "q311_act_oth16"]
mydata <- mydata[!names(mydata) %in% "q311_act_oth17"]
mydata <- mydata[!names(mydata) %in% "q311_act_oth18"]
mydata <- mydata[!names(mydata) %in% "q311_act_oth19"]
mydata <- mydata[!names(mydata) %in% "q311_act_oth20"]
mydata <- mydata[!names(mydata) %in% "q311_act_oth21"]
mydata <- mydata[!names(mydata) %in% "q311_act_oth22"]
mydata <- mydata[!names(mydata) %in% "q311_act_oth23"]
mydata <- mydata[!names(mydata) %in% "q311_act_oth24"]
mydata <- mydata[!names(mydata) %in% "q311_act_oth25"]
mydata <- mydata[!names(mydata) %in% "q311_act_oth26"]
mydata <- mydata[!names(mydata) %in% "q311_act_oth27"]
mydata <- mydata[!names(mydata) %in% "q311_act_oth28"]
mydata <- mydata[!names(mydata) %in% "q311_act_oth29"]
mydata <- mydata[!names(mydata) %in% "q311_act_oth30"]
mydata <- mydata[!names(mydata) %in% "q311_act_oth31"]
mydata <- mydata[!names(mydata) %in% "q311_act_oth32"]

Save processed data in Stata and SPSS format

haven::write_dta(mydata, paste0(filename, "_PU.dta"))
haven::write_sav(mydata, paste0(filename, "_PU.sav"))

# Add report title dynamically
title_var <- paste0("DOL-ILAB SDC - ", filename)