Skip to content

Introduction

We make sure that the timer is not TRUE, by setting it to ten hours.

library(cli)
# 10 hours
cli:::cli_tick_set(10 * 60 * 60 * 1000)
cli_tick_reset()
#> NULL
`__cli_update_due`
#> [1] FALSE

R benchmarks

The timer

fun <- function() NULL
ben_st <- bench::mark(
  `__cli_update_due`,
  fun(),
  .Call(ccli_tick_reset),
  interactive(),
  check = FALSE
)
ben_st
#> # A tibble: 4 × 6
#>   expression                  min   median  `itr/sec` mem_alloc `gc/sec`
#>   <bch:expr>             <bch:tm> <bch:tm>      <dbl> <bch:byt>    <dbl>
#> 1 __cli_update_due              0        0 102344749.        0B        0
#> 2 fun()                    99.9ns    100ns   4322955.        0B        0
#> 3 .Call(ccli_tick_reset)   99.9ns    200ns   5547162.        0B        0
#> 4 interactive()                 0        0  46908992.        0B        0
ben_st2 <- bench::mark(
  if (`__cli_update_due`) foobar()
)
ben_st2
#> # A tibble: 1 × 6
#>   expression                            min median itr/s…¹ mem_a…² gc/se…³
#>   <bch:expr>                       <bch:tm> <bch:>   <dbl> <bch:b>   <dbl>
#> 1 if (`__cli_update_due`) foobar()        0      0  3.73e7      0B       0
#> # … with abbreviated variable names ¹​`itr/sec`, ²​mem_alloc, ³​`gc/sec`

cli_progress_along()

seq <- 1:100000
ta <- cli_progress_along(seq)
bench::mark(seq[[1]], ta[[1]])
#> # A tibble: 2 × 6
#>   expression      min   median `itr/sec` mem_alloc `gc/sec`
#>   <bch:expr> <bch:tm> <bch:tm>     <dbl> <bch:byt>    <dbl>
#> 1 seq[[1]]     99.9ns    200ns  4754797.        0B     476.
#> 2 ta[[1]]       100ns    200ns  4884731.        0B       0

for loop

This is the baseline:

f0 <- function(n = 1e5) {
  x <- 0
  seq <- 1:n
  for (i in seq) {
    x <- x + i %% 2
  }
  x
}

With progress bars:

fp <- function(n = 1e5) {
  x <- 0
  seq <- 1:n
  for (i in cli_progress_along(seq)) {
    x <- x + seq[[i]] %% 2
  }
  x
}

Overhead per iteration:

ben_taf <- bench::mark(f0(), fp())
ben_taf
#> # A tibble: 2 × 6
#>   expression      min   median `itr/sec` mem_alloc `gc/sec`
#>   <bch:expr> <bch:tm> <bch:tm>     <dbl> <bch:byt>    <dbl>
#> 1 f0()         18.4ms   18.4ms      54.3    25.2KB    1140.
#> 2 fp()         26.2ms   26.2ms      38.1    83.2KB     610.
(ben_taf$median[2] - ben_taf$median[1]) / 1e5
#> [1] 78.3ns
ben_taf2 <- bench::mark(f0(1e6), fp(1e6))
#> Warning: Some expressions had a GC in every iteration; so filtering is
#> disabled.
ben_taf2
#> # A tibble: 2 × 6
#>   expression      min   median `itr/sec` mem_alloc `gc/sec`
#>   <bch:expr> <bch:tm> <bch:tm>     <dbl> <bch:byt>    <dbl>
#> 1 f0(1e+06)     224ms    237ms      4.29        0B     40.1
#> 2 fp(1e+06)     244ms    288ms      3.47    1.85KB     31.3
(ben_taf2$median[2] - ben_taf2$median[1]) / 1e6
#> [1] 51ns
ben_taf3 <- bench::mark(f0(1e7), fp(1e7))
#> Warning: Some expressions had a GC in every iteration; so filtering is
#> disabled.
ben_taf3
#> # A tibble: 2 × 6
#>   expression      min   median `itr/sec` mem_alloc `gc/sec`
#>   <bch:expr> <bch:tm> <bch:tm>     <dbl> <bch:byt>    <dbl>
#> 1 f0(1e+07)      2.3s     2.3s     0.436        0B     40.9
#> 2 fp(1e+07)     2.52s    2.52s     0.397    1.85KB     36.9
(ben_taf3$median[2] - ben_taf3$median[1]) / 1e7
#> [1] 22.4ns
ben_taf4 <- bench::mark(f0(1e8), fp(1e8))
#> Warning: Some expressions had a GC in every iteration; so filtering is
#> disabled.
ben_taf4
#> # A tibble: 2 × 6
#>   expression      min   median `itr/sec` mem_alloc `gc/sec`
#>   <bch:expr> <bch:tm> <bch:tm>     <dbl> <bch:byt>    <dbl>
#> 1 f0(1e+08)     21.7s    21.7s    0.0462        0B     27.4
#> 2 fp(1e+08)     24.1s    24.1s    0.0415    1.85KB     23.9
(ben_taf4$median[2] - ben_taf4$median[1]) / 1e8
#> [1] 24.5ns

Mapping with lapply()

This is the baseline:

f0 <- function(n = 1e5) {
  seq <- 1:n
  ret <- lapply(seq, function(x) {
    x %% 2
  })
  invisible(ret)
}

With an index vector:

f01 <- function(n = 1e5) {
  seq <- 1:n
  ret <- lapply(seq_along(seq), function(i) {
    seq[[i]] %% 2
  })
  invisible(ret)
}

With progress bars:

fp <- function(n = 1e5) {
  seq <- 1:n
  ret <- lapply(cli_progress_along(seq), function(i) {
    seq[[i]] %% 2
  })
  invisible(ret)
}

Overhead per iteration:

ben_tam <- bench::mark(f0(), f01(), fp())
#> Warning: Some expressions had a GC in every iteration; so filtering is
#> disabled.
ben_tam
#> # A tibble: 3 × 6
#>   expression      min   median `itr/sec` mem_alloc `gc/sec`
#>   <bch:expr> <bch:tm> <bch:tm>     <dbl> <bch:byt>    <dbl>
#> 1 f0()          104ms    122ms      7.76     781KB    21.7 
#> 2 f01()         127ms    142ms      5.32     781KB     7.10
#> 3 fp()          133ms    144ms      6.92     783KB    10.4
(ben_tam$median[3] - ben_tam$median[1]) / 1e5
#> [1] 226ns
ben_tam2 <- bench::mark(f0(1e6), f01(1e6), fp(1e6))
#> Warning: Some expressions had a GC in every iteration; so filtering is
#> disabled.
ben_tam2
#> # A tibble: 3 × 6
#>   expression      min   median `itr/sec` mem_alloc `gc/sec`
#>   <bch:expr> <bch:tm> <bch:tm>     <dbl> <bch:byt>    <dbl>
#> 1 f0(1e+06)      2.2s     2.2s     0.454    7.63MB     3.63
#> 2 f01(1e+06)    1.17s    1.17s     0.855    7.63MB     1.71
#> 3 fp(1e+06)     1.45s    1.45s     0.692    7.63MB     2.77
(ben_tam2$median[3] - ben_tam2$median[1]) / 1e6
#> [1] 1ns
(ben_tam2$median[3] - ben_tam2$median[2]) / 1e6
#> [1] 274ns

Mapping with purrr

This is the baseline:

f0 <- function(n = 1e5) {
  seq <- 1:n
  ret <- purrr::map(seq, function(x) {
    x %% 2
  })
  invisible(ret)
}

With index vector:

f01 <- function(n = 1e5) {
  seq <- 1:n
  ret <- purrr::map(seq_along(seq), function(i) {
    seq[[i]] %% 2
  })
  invisible(ret)
}

With progress bars:

fp <- function(n = 1e5) {
  seq <- 1:n
  ret <- purrr::map(cli_progress_along(seq), function(i) {
    seq[[i]] %% 2
  })
  invisible(ret)
}

Overhead per iteration:

ben_pur <- bench::mark(f0(), f01(), fp())
ben_pur
#> # A tibble: 3 × 6
#>   expression      min   median `itr/sec` mem_alloc `gc/sec`
#>   <bch:expr> <bch:tm> <bch:tm>     <dbl> <bch:byt>    <dbl>
#> 1 f0()         86.7ms   92.1ms     10.8      795KB     2.16
#> 2 f01()       103.5ms  104.5ms      9.51     781KB     2.38
#> 3 fp()        102.6ms  107.7ms      9.25     783KB     2.31
(ben_pur$median[3] - ben_pur$median[1]) / 1e5
#> [1] 157ns
(ben_pur$median[3] - ben_pur$median[2]) / 1e5
#> [1] 32.7ns
ben_pur2 <- bench::mark(f0(1e6), f01(1e6), fp(1e6))
#> Warning: Some expressions had a GC in every iteration; so filtering is
#> disabled.
ben_pur2
#> # A tibble: 3 × 6
#>   expression      min   median `itr/sec` mem_alloc `gc/sec`
#>   <bch:expr> <bch:tm> <bch:tm>     <dbl> <bch:byt>    <dbl>
#> 1 f0(1e+06)     1.07s    1.07s     0.931    7.63MB     1.86
#> 2 f01(1e+06)    1.38s    1.38s     0.724    7.63MB     2.17
#> 3 fp(1e+06)     1.46s    1.46s     0.685    7.63MB     2.06
(ben_pur2$median[3] - ben_pur2$median[1]) / 1e6
#> [1] 385ns
(ben_pur2$median[3] - ben_pur2$median[2]) / 1e6
#> [1] 78.5ns

ticking()

f0 <- function(n = 1e5) {
  i <- 0
  x <- 0 
  while (i < n) {
    x <- x + i %% 2
    i <- i + 1
  }
  x
}
fp <- function(n = 1e5) {
  i <- 0
  x <- 0 
  while (ticking(i < n)) {
    x <- x + i %% 2
    i <- i + 1
  }
  x
}
ben_tk <- bench::mark(f0(), fp())
#> Warning: Some expressions had a GC in every iteration; so filtering is
#> disabled.
ben_tk
#> # A tibble: 2 × 6
#>   expression      min   median `itr/sec` mem_alloc `gc/sec`
#>   <bch:expr> <bch:tm> <bch:tm>     <dbl> <bch:byt>    <dbl>
#> 1 f0()        19.32ms  19.99ms    49.1      40.3KB    1.96 
#> 2 fp()          7.32s    7.32s     0.137    96.9KB    0.957
(ben_tk$median[2] - ben_tk$median[1]) / 1e5
#> [1] 73µs

Traditional API

f0 <- function(n = 1e5) {
  x <- 0
  for (i in 1:n) {
    x <- x + i %% 2
  }
  x
}
fp <- function(n = 1e5) {
  cli_progress_bar(total = n)
  x <- 0
  for (i in 1:n) {
    x <- x + i %% 2
    cli_progress_update()
  }
  x
}
ff <- function(n = 1e5) {
  cli_progress_bar(total = n)
  x <- 0
  for (i in 1:n) {
    x <- x + i %% 2
    if (`__cli_update_due`) cli_progress_update()
  }
  x
}
ben_api <- bench::mark(f0(), ff(), fp())
#> Warning: Some expressions had a GC in every iteration; so filtering is
#> disabled.
ben_api
#> # A tibble: 3 × 6
#>   expression      min   median `itr/sec` mem_alloc `gc/sec`
#>   <bch:expr> <bch:tm> <bch:tm>     <dbl> <bch:byt>    <dbl>
#> 1 f0()        17.95ms  19.06ms    46.8      19.9KB     3.90
#> 2 ff()         26.8ms  30.18ms    31.7      28.6KB     1.98
#> 3 fp()          3.86s    3.86s     0.259      26KB     1.04
(ben_api$median[3] - ben_api$median[1]) / 1e5
#> [1] 38.4µs
(ben_api$median[2] - ben_api$median[1]) / 1e5
#> [1] 111ns
ben_api2 <- bench::mark(f0(1e6), ff(1e6), fp(1e6))
#> Warning: Some expressions had a GC in every iteration; so filtering is
#> disabled.
ben_api2
#> # A tibble: 3 × 6
#>   expression      min   median `itr/sec` mem_alloc `gc/sec`
#>   <bch:expr> <bch:tm> <bch:tm>     <dbl> <bch:byt>    <dbl>
#> 1 f0(1e+06)   205.8ms  237.8ms    4.37          0B     2.92
#> 2 ff(1e+06)     288ms    297ms    3.37      1.85KB     3.37
#> 3 fp(1e+06)     40.9s    40.9s    0.0245    1.85KB     1.03
(ben_api2$median[3] - ben_api2$median[1]) / 1e6
#> [1] 40.6µs
(ben_api2$median[2] - ben_api2$median[1]) / 1e6
#> [1] 59.2ns

C benchmarks

Baseline function:

SEXP test_baseline() {
  int i;
  int res = 0;
  for (i = 0; i < 2000000000; i++) {
    res += i % 2;
  }
  return ScalarInteger(res);
}

Switch + modulo check:

SEXP test_modulo(SEXP progress) {
  int i;
  int res = 0;
  int progress_ = LOGICAL(progress)[0];
  for (i = 0; i < 2000000000; i++) {
    if (i % 10000 == 0 && progress_) cli_progress_set(R_NilValue, i);
    res += i % 2;
  }
  return ScalarInteger(res);
}

cli progress bar API:

SEXP test_cli() {
  int i;
  int res = 0;
  SEXP bar = PROTECT(cli_progress_bar(2000000000, NULL));
  for (i = 0; i < 2000000000; i++) {
    if (CLI_SHOULD_TICK) cli_progress_set(bar, i);
    res += i % 2;
  }
  cli_progress_done(bar);
  UNPROTECT(1);
  return ScalarInteger(res);
}
SEXP test_cli_unroll() {
  int i = 0;
  int res = 0;
  SEXP bar = PROTECT(cli_progress_bar(2000000000, NULL));
  int s, final, step = 2000000000 / 100000;
  for (s = 0; s < 100000; s++) {
    if (CLI_SHOULD_TICK) cli_progress_set(bar, i);
    final = (s + 1) * step;
    for (i = s * step; i < final; i++) {
      res += i % 2;
    }
  }
  cli_progress_done(bar);
  UNPROTECT(1);
  return ScalarInteger(res);
}
library(progresstest)
ben_c <- bench::mark(
  test_baseline(),
  test_modulo(),
  test_cli(),
  test_cli_unroll()
)
ben_c
#> # A tibble: 4 × 6
#>   expression             min   median `itr/sec` mem_alloc `gc/sec`
#>   <bch:expr>        <bch:tm> <bch:tm>     <dbl> <bch:byt>    <dbl>
#> 1 test_baseline()   910.45ms 910.45ms     1.10     2.08KB        0
#> 2 test_modulo()        1.77s    1.77s     0.565    2.24KB        0
#> 3 test_cli()            1.6s     1.6s     0.625   23.69KB        0
#> 4 test_cli_unroll()  881.2ms  881.2ms     1.13     3.36KB        0
(ben_c$median[3] - ben_c$median[1]) / 2000000000
#> [1] 1ns

Display update

We only update the display a fixed number of times per second. (Currently maximum five times per second.)

Let’s measure how long a single update takes.

Iterator with a bar

cli_progress_bar(total = 100000)
bench::mark(cli_progress_update(force = TRUE), max_iterations = 10000)
#>    0% | ETA:  8m
#>    0% | ETA:  3h
#>    0% | ETA:  2h
#>    0% | ETA:  2h
#>    0% | ETA:  1h
#>    0% | ETA:  1h
#>    0% | ETA:  1h
#>    0% | ETA:  1h
#>    0% | ETA:  1h
#>    0% | ETA: 48m
#>    0% | ETA: 45m
#>    0% | ETA: 42m
#>    0% | ETA: 40m
#>    0% | ETA: 39m
#>    0% | ETA: 37m
#>    0% | ETA: 36m
#>    0% | ETA: 35m
#>    0% | ETA: 34m
#>    0% | ETA: 33m
#>    0% | ETA: 32m
#>    0% | ETA: 31m
#>    0% | ETA: 30m
#>    0% | ETA: 30m
#>    0% | ETA: 29m
#>    0% | ETA: 29m
#>    0% | ETA: 28m
#>    0% | ETA: 28m
#>    0% | ETA: 27m
#>    0% | ETA: 27m
#>    0% | ETA: 26m
#>    0% | ETA: 26m
#>    0% | ETA: 26m
#>    0% | ETA: 25m
#>    0% | ETA: 25m
#>    0% | ETA: 25m
#>    0% | ETA: 25m
#>    0% | ETA: 24m
#>    0% | ETA: 24m
#>    0% | ETA: 24m
#>    0% | ETA: 24m
#>    0% | ETA: 23m
#>    0% | ETA: 23m
#>    0% | ETA: 23m
#>    0% | ETA: 23m
#>    0% | ETA: 23m
#>    0% | ETA: 23m
#>    0% | ETA: 23m
#>    0% | ETA: 22m
#>    0% | ETA: 22m
#>    0% | ETA: 22m
#>    0% | ETA: 22m
#>    0% | ETA: 22m
#>    0% | ETA: 22m
#>    0% | ETA: 22m
#> # A tibble: 1 × 6
#>   expression                            min median itr/s…¹ mem_a…² gc/se…³
#>   <bch:expr>                        <bch:t> <bch:>   <dbl> <bch:b>   <dbl>
#> 1 cli_progress_update(force = TRUE)  8.84ms 9.29ms    104.  1.27MB       0
#> # … with abbreviated variable names ¹​`itr/sec`, ²​mem_alloc, ³​`gc/sec`
cli_progress_done()

Iterator without a bar

cli_progress_bar(total = NA)
bench::mark(cli_progress_update(force = TRUE), max_iterations = 10000)
#> ⠙ 1 done (275/s) | 4ms
#> ⠹ 2 done (41/s) | 49ms
#> ⠸ 3 done (50/s) | 61ms
#> ⠼ 4 done (57/s) | 71ms
#> ⠴ 5 done (62/s) | 82ms
#> ⠦ 6 done (65/s) | 93ms
#> ⠧ 7 done (68/s) | 104ms
#> ⠇ 8 done (70/s) | 114ms
#> ⠏ 9 done (73/s) | 125ms
#> ⠋ 10 done (73/s) | 137ms
#> ⠙ 11 done (75/s) | 148ms
#> ⠹ 12 done (76/s) | 159ms
#> ⠸ 13 done (77/s) | 170ms
#> ⠼ 14 done (78/s) | 181ms
#> ⠴ 15 done (79/s) | 191ms
#> ⠦ 16 done (79/s) | 202ms
#> ⠧ 17 done (80/s) | 213ms
#> ⠇ 18 done (78/s) | 231ms
#> ⠏ 19 done (79/s) | 243ms
#> ⠋ 20 done (79/s) | 254ms
#> ⠙ 21 done (79/s) | 265ms
#> ⠹ 22 done (79/s) | 277ms
#> ⠸ 23 done (80/s) | 289ms
#> ⠼ 24 done (80/s) | 300ms
#> ⠴ 25 done (81/s) | 311ms
#> ⠦ 26 done (81/s) | 322ms
#> ⠧ 27 done (81/s) | 333ms
#> ⠇ 28 done (81/s) | 345ms
#> ⠏ 29 done (82/s) | 356ms
#> ⠋ 30 done (82/s) | 368ms
#> ⠙ 31 done (82/s) | 378ms
#> ⠹ 32 done (82/s) | 389ms
#> ⠸ 33 done (83/s) | 400ms
#> ⠼ 34 done (82/s) | 415ms
#> ⠴ 35 done (82/s) | 427ms
#> ⠦ 36 done (82/s) | 438ms
#> ⠧ 37 done (83/s) | 449ms
#> ⠇ 38 done (83/s) | 459ms
#> ⠏ 39 done (83/s) | 470ms
#> ⠋ 40 done (83/s) | 480ms
#> ⠙ 41 done (84/s) | 491ms
#> ⠹ 42 done (84/s) | 501ms
#> ⠸ 43 done (84/s) | 512ms
#> ⠼ 44 done (84/s) | 523ms
#> ⠴ 45 done (84/s) | 534ms
#> ⠦ 46 done (85/s) | 545ms
#> # A tibble: 1 × 6
#>   expression                            min median itr/s…¹ mem_a…² gc/se…³
#>   <bch:expr>                        <bch:t> <bch:>   <dbl> <bch:b>   <dbl>
#> 1 cli_progress_update(force = TRUE)  10.2ms 10.9ms    90.2   202KB    2.05
#> # … with abbreviated variable names ¹​`itr/sec`, ²​mem_alloc, ³​`gc/sec`
cli_progress_done()