Data Processing in R

# Data Processing in R
## With the {tidyverse}
### Michael Jones
### 2021-08-25

---

/* custom.css */
.left-code {
  color: #777;
  width: 38%;
  height: 92%;
  float: left;
}
.right-plot {
  width: 60%;
  float: right;
  padding-left: 1%;
}
</style>

# A quick note<br>on **Composition**

---
class: center, middle

# `$f(g(x)) = (f \circ g)(x)$`

---
class: highlight-last-item, center

- `f(g(x)) = g(x) %>% f()`

--
- `f(g(x)) = x %>% g() %>% f()`

--
- `f(x, y) = x %>% f(y)`

--
- `f(x, y) = x |> f(y)`

---

# Traditional Cake

```r
sprinkle(
  sandwich(
    bake(
        mix(
          what = ingredients,
          in = "bowl"
        )
      in = "oven",
      at = 180
    )
  between = "jam",
  )
  with = "sugar"
)
```

---

# Saving your intermediate steps

```r
batter <- mix(what = ingredients, in = "bowl")
sponge <- bake(batter, in = "oven", at = 180)
assembled_cake <- sandwich(sponge, between = "jam")
decorated_cake <- sprinkle(assembled_cake, with = "sugar")
```

---

# Piped Cake

```r
mix(what = ingredients, in = "bowl") %>% 
  bake(in = "oven", at = 180) %>%
  sandwich(between = "jam") %>%
  sprinkle(with = "sugar")
```
 
---

# {tidyverse}

---

```r
library(tidyverse)
```

```
## ── Attaching packages ─────────────────────────────────────── tidyverse 1.3.1 ──
```

```
## ✓ ggplot2 3.3.5     ✓ purrr   0.3.4
## ✓ tibble  3.1.4     ✓ dplyr   1.0.7
## ✓ tidyr   1.1.3     ✓ stringr 1.4.0
## ✓ readr   2.0.1     ✓ forcats 0.5.1
```

```
## ── Conflicts ────────────────────────────────────────── tidyverse_conflicts() ──
## x dplyr::filter() masks stats::filter()
## x dplyr::lag()    masks stats::lag()
```

---

# Other useful packages

```r
library(readxl)
library(janitor)
```

```
## 
## Attaching package: 'janitor'
```

```
## The following objects are masked from 'package:stats':
## 
##     chisq.test, fisher.test
```

---
class: inverse, middle, center

# Reading in<br>a CSV

---

# Old Fashioned

```r
read.csv("my_data.csv")
```

```
##   id_column group        fruit rating    sampled
## 1         1     A        apple      4 2021-07-05
## 2         2     B      apricot      3 2021-07-13
## 3         3     A      avocado      5 2021-07-09
## 4         4     B       banana      1 2021-07-26
## 5         5     A  bell pepper      4 2021-07-22
## 6         6     B     bilberry      1 2021-07-31
## 7         7     A   blackberry      5 2021-07-01
## 8         8     B blackcurrant      1 2021-07-18
```
---

# Tidyverse Way

```r
read_csv("my_data.csv")
```

```
## Rows: 8 Columns: 5
```

```
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr  (2): group, fruit
## dbl  (2): id_column, rating
## date (1): sampled
```

```
## 
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
```

```
## # A tibble: 8 × 5
##   id_column group fruit        rating sampled   
##       <dbl> <chr> <chr>         <dbl> <date>    
## 1         1 A     apple             4 2021-07-05
## 2         2 B     apricot           3 2021-07-13
## 3         3 A     avocado           5 2021-07-09
## 4         4 B     banana            1 2021-07-26
## 5         5 A     bell pepper       4 2021-07-22
## 6         6 B     bilberry          1 2021-07-31
## 7         7 A     blackberry        5 2021-07-01
## 8         8 B     blackcurrant      1 2021-07-18
```

---

# Tell R what you expect

```r
ratings <- read_csv("my_data.csv",
         col_types = cols(
           id_column = col_double(),
           group = col_character(),
           fruit = col_character(),
           rating = col_double(),
           sampled = col_date(format = "")
         ))
```

---

# Tell R what you expect

```r
ratings_bad <- read_csv("my_data.csv",
         col_types = cols(
           id_column = col_double(),
           group = col_character(),
           fruit = col_double(),
           rating = col_double(),
           sampled = col_date(format = "")
         ))
```

---

# Taking a quick look at the data

```r
ratings
```

---

# Taking a quick look at the data

```r
glimpse(ratings)
```

```
## Rows: 8
## Columns: 5
## $ id_column <dbl> 1, 2, 3, 4, 5, 6, 7, 8
## $ group     <chr> "A", "B", "A", "B", "A", "B", "A", "B"
## $ fruit     <chr> "apple", "apricot", "avocado", "banana", "bell pepper", "bil…
## $ rating    <dbl> 4, 3, 5, 1, 4, 1, 5, 1
## $ sampled   <date> 2021-07-05, 2021-07-13, 2021-07-09, 2021-07-26, 2021-07-22, …
```

---
class: center, inverse, middle

# Key idea of<br>the tidyverse

---

# Functions **accept** a data frame
# Functions **return** a data frame

---
# Selecting Columns

## <span class="hl kwd">select</span>(&lt;dataframe&gt;, &lt;columns&gt;) `$\to$` `data frame`

---

```r
ratings %>%
* select(id_column)
```
]
 
.panel2-selecting-rotate[

```
## # A tibble: 8 × 1
##   id_column
##       <dbl>
## 1         1
## 2         2
## 3         3
## 4         4
## 5         5
## 6         6
## 7         7
## 8         8
```
]

---
count: false
 
# Selecting Columns
.panel1-selecting-rotate[

```r
ratings %>%
* select(id_column, group)
```
]
 
.panel2-selecting-rotate[

```
## # A tibble: 8 × 2
##   id_column group
##       <dbl> <chr>
## 1         1 A    
## 2         2 B    
## 3         3 A    
## 4         4 B    
## 5         5 A    
## 6         6 B    
## 7         7 A    
## 8         8 B
```
]

---
count: false
 
# Selecting Columns
.panel1-selecting-rotate[

```r
ratings %>%
* select(id_column:rating)
```
]
 
.panel2-selecting-rotate[

```
## # A tibble: 8 × 4
##   id_column group fruit        rating
##       <dbl> <chr> <chr>         <dbl>
## 1         1 A     apple             4
## 2         2 B     apricot           3
## 3         3 A     avocado           5
## 4         4 B     banana            1
## 5         5 A     bell pepper       4
## 6         6 B     bilberry          1
## 7         7 A     blackberry        5
## 8         8 B     blackcurrant      1
```
]

---
count: false
 
# Selecting Columns
.panel1-selecting-rotate[

```r
ratings %>%
* select(-id_column)
```
]
 
.panel2-selecting-rotate[

```
## # A tibble: 8 × 4
##   group fruit        rating sampled   
##   <chr> <chr>         <dbl> <date>    
## 1 A     apple             4 2021-07-05
## 2 B     apricot           3 2021-07-13
## 3 A     avocado           5 2021-07-09
## 4 B     banana            1 2021-07-26
## 5 A     bell pepper       4 2021-07-22
## 6 B     bilberry          1 2021-07-31
## 7 A     blackberry        5 2021-07-01
## 8 B     blackcurrant      1 2021-07-18
```
]

---

- *Yes*: `ratings %>% select(id_column)`
- **No**: `ratings %>% select("id_column")`

---

# Filtering Columns

## <span class="hl kwd">filter</span>(&lt;dataframe&gt;, &lt;predicates&gt;) `$\to$` `data frame`

---

```r
ratings %>%
* filter(group == "A")
```
]
 
.panel2-filtering-rotate[

```
## # A tibble: 4 × 5
##   id_column group fruit       rating sampled   
##       <dbl> <chr> <chr>        <dbl> <date>    
## 1         1 A     apple            4 2021-07-05
## 2         3 A     avocado          5 2021-07-09
## 3         5 A     bell pepper      4 2021-07-22
## 4         7 A     blackberry       5 2021-07-01
```
]

---
count: false
 
# Filtering Columns
.panel1-filtering-rotate[

```r
ratings %>%
* filter(rating >= 3)
```
]
 
.panel2-filtering-rotate[

```
## # A tibble: 5 × 5
##   id_column group fruit       rating sampled   
##       <dbl> <chr> <chr>        <dbl> <date>    
## 1         1 A     apple            4 2021-07-05
## 2         2 B     apricot          3 2021-07-13
## 3         3 A     avocado          5 2021-07-09
## 4         5 A     bell pepper      4 2021-07-22
## 5         7 A     blackberry       5 2021-07-01
```
]

---
count: false
 
# Filtering Columns
.panel1-filtering-rotate[

```r
ratings %>%
* filter(sampled > as.Date("2021-07-15"))
```
]
 
.panel2-filtering-rotate[

```
## # A tibble: 4 × 5
##   id_column group fruit        rating sampled   
##       <dbl> <chr> <chr>         <dbl> <date>    
## 1         4 B     banana            1 2021-07-26
## 2         5 A     bell pepper       4 2021-07-22
## 3         6 B     bilberry          1 2021-07-31
## 4         8 B     blackcurrant      1 2021-07-18
```
]

---

# Sorting Columns

## <span class="hl kwd">arrange</span>(&lt;dataframe&gt;, &lt;columns&gt;) `$\to$` `data frame`

---

```r
ratings %>%
* arrange(rating)
```
]
 
.panel2-arranging-rotate[

```
## # A tibble: 8 × 5
##   id_column group fruit        rating sampled   
##       <dbl> <chr> <chr>         <dbl> <date>    
## 1         4 B     banana            1 2021-07-26
## 2         6 B     bilberry          1 2021-07-31
## 3         8 B     blackcurrant      1 2021-07-18
## 4         2 B     apricot           3 2021-07-13
## 5         1 A     apple             4 2021-07-05
## 6         5 A     bell pepper       4 2021-07-22
## 7         3 A     avocado           5 2021-07-09
## 8         7 A     blackberry        5 2021-07-01
```
]

---
count: false
 
# Sorting Columns
.panel1-arranging-rotate[

```r
ratings %>%
* arrange(desc(rating))
```
]
 
.panel2-arranging-rotate[

```
## # A tibble: 8 × 5
##   id_column group fruit        rating sampled   
##       <dbl> <chr> <chr>         <dbl> <date>    
## 1         3 A     avocado           5 2021-07-09
## 2         7 A     blackberry        5 2021-07-01
## 3         1 A     apple             4 2021-07-05
## 4         5 A     bell pepper       4 2021-07-22
## 5         2 B     apricot           3 2021-07-13
## 6         4 B     banana            1 2021-07-26
## 7         6 B     bilberry          1 2021-07-31
## 8         8 B     blackcurrant      1 2021-07-18
```
]

---
count: false
 
# Sorting Columns
.panel1-arranging-rotate[

```r
ratings %>%
* arrange(group, sampled)
```
]
 
.panel2-arranging-rotate[

```
## # A tibble: 8 × 5
##   id_column group fruit        rating sampled   
##       <dbl> <chr> <chr>         <dbl> <date>    
## 1         7 A     blackberry        5 2021-07-01
## 2         1 A     apple             4 2021-07-05
## 3         3 A     avocado           5 2021-07-09
## 4         5 A     bell pepper       4 2021-07-22
## 5         2 B     apricot           3 2021-07-13
## 6         8 B     blackcurrant      1 2021-07-18
## 7         4 B     banana            1 2021-07-26
## 8         6 B     bilberry          1 2021-07-31
```
]

---

# Changing Columns

## <span class="hl kwd">mutate</span>(&lt;dataframe&gt;, &lt;new_column&gt; = &lt;<span class="hl kwa">function</span>&gt;) `$\to$` `data frame`

---

```r
ratings %>%
* mutate(rating = rating * 20)
```
]
 
.panel2-mutating-rotate[

```
## # A tibble: 8 × 5
##   id_column group fruit        rating sampled   
##       <dbl> <chr> <chr>         <dbl> <date>    
## 1         1 A     apple            80 2021-07-05
## 2         2 B     apricot          60 2021-07-13
## 3         3 A     avocado         100 2021-07-09
## 4         4 B     banana           20 2021-07-26
## 5         5 A     bell pepper      80 2021-07-22
## 6         6 B     bilberry         20 2021-07-31
## 7         7 A     blackberry      100 2021-07-01
## 8         8 B     blackcurrant     20 2021-07-18
```
]

---
count: false
 
# Changing Columns
.panel1-mutating-rotate[

```r
ratings %>%
* mutate(fruit = str_to_upper(fruit))
```
]
 
.panel2-mutating-rotate[

```
## # A tibble: 8 × 5
##   id_column group fruit        rating sampled   
##       <dbl> <chr> <chr>         <dbl> <date>    
## 1         1 A     APPLE             4 2021-07-05
## 2         2 B     APRICOT           3 2021-07-13
## 3         3 A     AVOCADO           5 2021-07-09
## 4         4 B     BANANA            1 2021-07-26
## 5         5 A     BELL PEPPER       4 2021-07-22
## 6         6 B     BILBERRY          1 2021-07-31
## 7         7 A     BLACKBERRY        5 2021-07-01
## 8         8 B     BLACKCURRANT      1 2021-07-18
```
]

---
count: false
 
# Changing Columns
.panel1-mutating-rotate[

```r
ratings %>%
* mutate(sampled = format(sampled, "%b %d"), rating = rating * 20)
```
]
 
.panel2-mutating-rotate[

```
## # A tibble: 8 × 5
##   id_column group fruit        rating sampled
##       <dbl> <chr> <chr>         <dbl> <chr>  
## 1         1 A     apple            80 Jul 05 
## 2         2 B     apricot          60 Jul 13 
## 3         3 A     avocado         100 Jul 09 
## 4         4 B     banana           20 Jul 26 
## 5         5 A     bell pepper      80 Jul 22 
## 6         6 B     bilberry         20 Jul 31 
## 7         7 A     blackberry      100 Jul 01 
## 8         8 B     blackcurrant     20 Jul 18
```
]

---
count: false
 
# Changing Columns
.panel1-mutating-rotate[

```r
ratings %>%
* mutate(new_col = 1:8)
```
]
 
.panel2-mutating-rotate[

```
## # A tibble: 8 × 6
##   id_column group fruit        rating sampled    new_col
##       <dbl> <chr> <chr>         <dbl> <date>       <int>
## 1         1 A     apple             4 2021-07-05       1
## 2         2 B     apricot           3 2021-07-13       2
## 3         3 A     avocado           5 2021-07-09       3
## 4         4 B     banana            1 2021-07-26       4
## 5         5 A     bell pepper       4 2021-07-22       5
## 6         6 B     bilberry          1 2021-07-31       6
## 7         7 A     blackberry        5 2021-07-01       7
## 8         8 B     blackcurrant      1 2021-07-18       8
```
]

---

```r
*ratings
```
]
 
.panel2-chaining-auto[

---
count: false
 
# Chaining Functions Together
.panel1-chaining-auto[

```r
ratings %>%
* filter(group == "A")
```
]
 
.panel2-chaining-auto[

---
count: false
 
# Chaining Functions Together
.panel1-chaining-auto[

```r
ratings %>%
  filter(group == "A") %>%
* select(fruit, sampled)
```
]
 
.panel2-chaining-auto[

```
## # A tibble: 4 × 2
##   fruit       sampled   
##   <chr>       <date>    
## 1 apple       2021-07-05
## 2 avocado     2021-07-09
## 3 bell pepper 2021-07-22
## 4 blackberry  2021-07-01
```
]

---
count: false
 
# Chaining Functions Together
.panel1-chaining-auto[

```r
ratings %>%
  filter(group == "A") %>%
  select(fruit, sampled) %>%
* mutate(month_name = format(sampled, "%b"))
```
]
 
.panel2-chaining-auto[

```
## # A tibble: 4 × 3
##   fruit       sampled    month_name
##   <chr>       <date>     <chr>     
## 1 apple       2021-07-05 Jul       
## 2 avocado     2021-07-09 Jul       
## 3 bell pepper 2021-07-22 Jul       
## 4 blackberry  2021-07-01 Jul
```
]

---
count: false
 
# Chaining Functions Together
.panel1-chaining-auto[

```r
ratings %>%
  filter(group == "A") %>%
  select(fruit, sampled) %>%
  mutate(month_name = format(sampled, "%b")) %>%
* arrange(desc(fruit))
```
]
 
.panel2-chaining-auto[

```
## # A tibble: 4 × 3
##   fruit       sampled    month_name
##   <chr>       <date>     <chr>     
## 1 blackberry  2021-07-01 Jul       
## 2 bell pepper 2021-07-22 Jul       
## 3 avocado     2021-07-09 Jul       
## 4 apple       2021-07-05 Jul
```
]

---
class: center, middle, inverse

# Grouping and Summarising

---

# Grouping and Summarising

## <span class="hl kwd">group_by</span>(&lt;dataframe&gt;, &lt;columns&gt;) `$\to$` `data frame`
## <span class="hl kwd">summarise</span>(&lt;dataframe&gt;, &lt;new column&gt; = &lt;<span class="hl kwa">function</span>&gt;) `$\to$` `data frame`

---

# Grouping and Summarising

```r
ratings %>% head(n = 2)
```

```
## # A tibble: 2 × 5
##   id_column group fruit   rating sampled   
##       <dbl> <chr> <chr>    <dbl> <date>    
## 1         1 A     apple        4 2021-07-05
## 2         2 B     apricot      3 2021-07-13
```

```r
ratings %>% group_by(group) %>% head(n = 2)
```

```
## # A tibble: 2 × 5
## # Groups:   group [2]
##   id_column group fruit   rating sampled   
##       <dbl> <chr> <chr>    <dbl> <date>    
## 1         1 A     apple        4 2021-07-05
## 2         2 B     apricot      3 2021-07-13
```

---

# Grouping and Summarising

```r
ratings %>% summarise(mean = mean(rating))
```

```
## # A tibble: 1 × 1
##    mean
##   <dbl>
## 1     3
```

```r
ratings %>% group_by(group) %>% summarise(mean = mean(rating))
```

```
## # A tibble: 2 × 2
##   group  mean
##   <chr> <dbl>
## 1 A       4.5
## 2 B       1.5
```

---

# Counting

```r
ratings %>% group_by(group) %>% summarise(count = n())
```

```
## # A tibble: 2 × 2
##   group count
##   <chr> <int>
## 1 A         4
## 2 B         4
```

```r
ratings %>% count(group, name = "count")
```

```
## # A tibble: 2 × 2
##   group count
##   <chr> <int>
## 1 A         4
## 2 B         4
```

---
class: inverse, middle, center

# tibbles within tibbles

---

```r
*ratings
```
]
 
.panel2-nesting-auto[

---
count: false
 
# Nesting Data
.panel1-nesting-auto[

```r
ratings %>%
* group_by(group)
```
]
 
.panel2-nesting-auto[

```
## # A tibble: 8 × 5
## # Groups:   group [2]
##   id_column group fruit        rating sampled   
##       <dbl> <chr> <chr>         <dbl> <date>    
## 1         1 A     apple             4 2021-07-05
## 2         2 B     apricot           3 2021-07-13
## 3         3 A     avocado           5 2021-07-09
## 4         4 B     banana            1 2021-07-26
## 5         5 A     bell pepper       4 2021-07-22
## 6         6 B     bilberry          1 2021-07-31
## 7         7 A     blackberry        5 2021-07-01
## 8         8 B     blackcurrant      1 2021-07-18
```
]

---
count: false
 
# Nesting Data
.panel1-nesting-auto[

```r
ratings %>%
  group_by(group) %>%
* nest()
```
]
 
.panel2-nesting-auto[

```
## # A tibble: 2 × 2
## # Groups:   group [2]
##   group data            
##   <chr> <list>          
## 1 A     <tibble [4 × 4]>
## 2 B     <tibble [4 × 4]>
```
]

---

```r
*(subset <- ratings %>%
*  filter(group == "A"))
```
]
 
.panel2-nest_explanation-user[

---
count: false
 
# Nesting Data
.panel1-nest_explanation-user[

```r
(subset <- ratings %>%
   filter(group == "A"))

*my_func <- function(df) {
* f <- df %>%
*   filter(rating == max(rating)) %>%
*   slice(1) %>%
*   pull(fruit)
* paste("The highest rated fruit is", f)
*} 
```
]
 
.panel2-nest_explanation-user[

---
count: false
 
# Nesting Data
.panel1-nest_explanation-user[

```r
(subset <- ratings %>%
   filter(group == "A"))

my_func <- function(df) {
  f <- df %>%
    filter(rating == max(rating)) %>%
    slice(1) %>%
    pull(fruit)
  paste("The highest rated fruit is", f)
}

*my_func(subset)
```
]
 
.panel2-nest_explanation-user[

```
## [1] "The highest rated fruit is avocado"
```
]

---

```r
*my_func <- function(df) {
* f <- df %>%
*   filter(rating == max(rating)) %>%
*   slice(1) %>%
*   pull(fruit)
* paste("The highest rated fruit is", f)
*} 
```
]
 
.panel2-nest_explanation2-user[

]

---
count: false
 
# Nesting Data
.panel1-nest_explanation2-user[

```r
my_func <- function(df) {
  f <- df %>%
    filter(rating == max(rating)) %>%
    slice(1) %>%
    pull(fruit)
  paste("The highest rated fruit is", f)
}

*ratings %>%
* group_by(group) %>%
* nest()
```
]
 
.panel2-nest_explanation2-user[

```
## # A tibble: 2 × 2
## # Groups:   group [2]
##   group data            
##   <chr> <list>          
## 1 A     <tibble [4 × 4]>
## 2 B     <tibble [4 × 4]>
```
]

---
count: false
 
# Nesting Data
.panel1-nest_explanation2-user[

```r
my_func <- function(df) {
  f <- df %>%
    filter(rating == max(rating)) %>%
    slice(1) %>%
    pull(fruit)
  paste("The highest rated fruit is", f)
}

ratings %>%
  group_by(group) %>%
  nest() %>%
* mutate(description = map(data, my_func))
```
]
 
.panel2-nest_explanation2-user[

```
## # A tibble: 2 × 3
## # Groups:   group [2]
##   group data             description
##   <chr> <list>           <list>     
## 1 A     <tibble [4 × 4]> <chr [1]>  
## 2 B     <tibble [4 × 4]> <chr [1]>
```
]

---
count: false
 
# Nesting Data
.panel1-nest_explanation2-user[

```r
my_func <- function(df) {
  f <- df %>%
    filter(rating == max(rating)) %>%
    slice(1) %>%
    pull(fruit)
  paste("The highest rated fruit is", f)
}

ratings %>%
  group_by(group) %>%
  nest() %>%
  mutate(description = map(data, my_func)) %>%
* unnest(description)
```
]
 
.panel2-nest_explanation2-user[

```
## # A tibble: 2 × 3
## # Groups:   group [2]
##   group data             description                       
##   <chr> <list>           <chr>                             
## 1 A     <tibble [4 × 4]> The highest rated fruit is avocado
## 2 B     <tibble [4 × 4]> The highest rated fruit is apricot
```
]

---

```r
*library(gapminder)
```
]
 
.panel2-gapminder1-user[

]