cli progress bar benchmark
Gábor Csárdi
2025-06-01
Source:vignettes/progress-benchmark.Rmd
progress-benchmark.Rmd
Introduction
We make sure that the timer is not TRUE
, by setting it
to ten hours.
library(cli)
# 10 hours
cli:::cli_tick_set(10 * 60 * 60 * 1000)
cli_tick_reset()
#> NULL
`__cli_update_due`
#> [1] FALSE
R benchmarks
The timer
fun <- function() NULL
ben_st <- bench::mark(
`__cli_update_due`,
fun(),
.Call(ccli_tick_reset),
interactive(),
check = FALSE
)
ben_st
#> # A tibble: 4 × 6
#> expression min median `itr/sec` mem_alloc `gc/sec`
#> <bch:expr> <bch:tm> <bch:tm> <dbl> <bch:byt> <dbl>
#> 1 __cli_update_due 0 10ns 1.20e8 0B 0
#> 2 fun() 130.04ns 151.1ns 4.67e6 0B 0
#> 3 .Call(ccli_tick_reset) 100ns 120ns 8.21e6 0B 0
#> 4 interactive() 8.96ns 10.1ns 6.61e7 0B 0
ben_st2 <- bench::mark(
if (`__cli_update_due`) foobar()
)
ben_st2
#> # A tibble: 1 × 6
#> expression min median `itr/sec` mem_alloc `gc/sec`
#> <bch:expr> <bch> <bch:> <dbl> <bch:byt> <dbl>
#> 1 if (`__cli_update_due`) fo… 40ns 50.1ns 20358045. 0B 0
cli_progress_along()
seq <- 1:100000
ta <- cli_progress_along(seq)
bench::mark(seq[[1]], ta[[1]])
#> # A tibble: 2 × 6
#> expression min median `itr/sec` mem_alloc `gc/sec`
#> <bch:expr> <bch:tm> <bch:tm> <dbl> <bch:byt> <dbl>
#> 1 seq[[1]] 110ns 130ns 7061974. 0B 0
#> 2 ta[[1]] 130ns 150ns 6176402. 0B 0
for
loop
This is the baseline:
f0 <- function(n = 1e5) {
x <- 0
seq <- 1:n
for (i in seq) {
x <- x + i %% 2
}
x
}
With progress bars:
fp <- function(n = 1e5) {
x <- 0
seq <- 1:n
for (i in cli_progress_along(seq)) {
x <- x + seq[[i]] %% 2
}
x
}
Overhead per iteration:
ben_taf <- bench::mark(f0(), fp())
ben_taf
#> # A tibble: 2 × 6
#> expression min median `itr/sec` mem_alloc `gc/sec`
#> <bch:expr> <bch:tm> <bch:tm> <dbl> <bch:byt> <dbl>
#> 1 f0() 22ms 22ms 45.4 21.6KB 272.
#> 2 fp() 24.7ms 24.9ms 38.0 82.3KB 142.
(ben_taf$median[2] - ben_taf$median[1]) / 1e5
#> [1] 29.1ns
ben_taf2 <- bench::mark(f0(1e6), fp(1e6))
#> Warning: Some expressions had a GC in every iteration; so filtering is
#> disabled.
ben_taf2
#> # A tibble: 2 × 6
#> expression min median `itr/sec` mem_alloc `gc/sec`
#> <bch:expr> <bch:tm> <bch:tm> <dbl> <bch:byt> <dbl>
#> 1 f0(1e+06) 235ms 238ms 4.20 0B 35.0
#> 2 fp(1e+06) 266ms 293ms 3.41 1.87KB 29.0
(ben_taf2$median[2] - ben_taf2$median[1]) / 1e6
#> [1] 54.6ns
ben_taf3 <- bench::mark(f0(1e7), fp(1e7))
#> Warning: Some expressions had a GC in every iteration; so filtering is
#> disabled.
ben_taf3
#> # A tibble: 2 × 6
#> expression min median `itr/sec` mem_alloc `gc/sec`
#> <bch:expr> <bch:tm> <bch:tm> <dbl> <bch:byt> <dbl>
#> 1 f0(1e+07) 2.49s 2.49s 0.402 0B 33.8
#> 2 fp(1e+07) 2.58s 2.58s 0.388 1.87KB 32.2
(ben_taf3$median[2] - ben_taf3$median[1]) / 1e7
#> [1] 9.28ns
ben_taf4 <- bench::mark(f0(1e8), fp(1e8))
#> Warning: Some expressions had a GC in every iteration; so filtering is
#> disabled.
ben_taf4
#> # A tibble: 2 × 6
#> expression min median `itr/sec` mem_alloc `gc/sec`
#> <bch:expr> <bch:tm> <bch:tm> <dbl> <bch:byt> <dbl>
#> 1 f0(1e+08) 23.2s 23.2s 0.0432 0B 21.1
#> 2 fp(1e+08) 25.3s 25.3s 0.0396 1.87KB 19.3
(ben_taf4$median[2] - ben_taf4$median[1]) / 1e8
#> [1] 21.1ns
Mapping with lapply()
This is the baseline:
With an index vector:
f01 <- function(n = 1e5) {
seq <- 1:n
ret <- lapply(seq_along(seq), function(i) {
seq[[i]] %% 2
})
invisible(ret)
}
With progress bars:
fp <- function(n = 1e5) {
seq <- 1:n
ret <- lapply(cli_progress_along(seq), function(i) {
seq[[i]] %% 2
})
invisible(ret)
}
Overhead per iteration:
ben_tam <- bench::mark(f0(), f01(), fp())
#> Warning: Some expressions had a GC in every iteration; so filtering is
#> disabled.
ben_tam
#> # A tibble: 3 × 6
#> expression min median `itr/sec` mem_alloc `gc/sec`
#> <bch:expr> <bch:tm> <bch:tm> <dbl> <bch:byt> <dbl>
#> 1 f0() 84.1ms 89.6ms 10.9 781KB 14.6
#> 2 f01() 105ms 122.7ms 7.90 781KB 13.8
#> 3 fp() 111.1ms 118.2ms 7.84 783KB 13.7
(ben_tam$median[3] - ben_tam$median[1]) / 1e5
#> [1] 285ns
ben_tam2 <- bench::mark(f0(1e6), f01(1e6), fp(1e6))
#> Warning: Some expressions had a GC in every iteration; so filtering is
#> disabled.
ben_tam2
#> # A tibble: 3 × 6
#> expression min median `itr/sec` mem_alloc `gc/sec`
#> <bch:expr> <bch:tm> <bch:tm> <dbl> <bch:byt> <dbl>
#> 1 f0(1e+06) 998.48ms 998.48ms 1.00 7.63MB 4.01
#> 2 f01(1e+06) 1.14s 1.14s 0.880 7.63MB 5.28
#> 3 fp(1e+06) 1.49s 1.49s 0.674 7.63MB 4.72
(ben_tam2$median[3] - ben_tam2$median[1]) / 1e6
#> [1] 486ns
(ben_tam2$median[3] - ben_tam2$median[2]) / 1e6
#> [1] 348ns
Mapping with purrr
This is the baseline:
f0 <- function(n = 1e5) {
seq <- 1:n
ret <- purrr::map(seq, function(x) {
x %% 2
})
invisible(ret)
}
With index vector:
f01 <- function(n = 1e5) {
seq <- 1:n
ret <- purrr::map(seq_along(seq), function(i) {
seq[[i]] %% 2
})
invisible(ret)
}
With progress bars:
fp <- function(n = 1e5) {
seq <- 1:n
ret <- purrr::map(cli_progress_along(seq), function(i) {
seq[[i]] %% 2
})
invisible(ret)
}
Overhead per iteration:
ben_pur <- bench::mark(f0(), f01(), fp())
ben_pur
#> # A tibble: 3 × 6
#> expression min median `itr/sec` mem_alloc `gc/sec`
#> <bch:expr> <bch:tm> <bch:tm> <dbl> <bch:byt> <dbl>
#> 1 f0() 80.2ms 81.4ms 12.3 1.4MB 6.14
#> 2 f01() 98.6ms 99.5ms 10.1 781.3KB 6.72
#> 3 fp() 98.7ms 99ms 10.1 783.2KB 15.2
(ben_pur$median[3] - ben_pur$median[1]) / 1e5
#> [1] 176ns
(ben_pur$median[3] - ben_pur$median[2]) / 1e5
#> [1] 1ns
ben_pur2 <- bench::mark(f0(1e6), f01(1e6), fp(1e6))
#> Warning: Some expressions had a GC in every iteration; so filtering is
#> disabled.
ben_pur2
#> # A tibble: 3 × 6
#> expression min median `itr/sec` mem_alloc `gc/sec`
#> <bch:expr> <bch:tm> <bch:tm> <dbl> <bch:byt> <dbl>
#> 1 f0(1e+06) 900ms 900ms 1.11 7.63MB 2.22
#> 2 f01(1e+06) 2.25s 2.25s 0.445 7.63MB 1.34
#> 3 fp(1e+06) 1.17s 1.17s 0.858 7.63MB 2.57
(ben_pur2$median[3] - ben_pur2$median[1]) / 1e6
#> [1] 265ns
(ben_pur2$median[3] - ben_pur2$median[2]) / 1e6
#> [1] 1ns
ticking()
f0 <- function(n = 1e5) {
i <- 0
x <- 0
while (i < n) {
x <- x + i %% 2
i <- i + 1
}
x
}
ben_tk <- bench::mark(f0(), fp())
#> Warning: Some expressions had a GC in every iteration; so filtering is
#> disabled.
ben_tk
#> # A tibble: 2 × 6
#> expression min median `itr/sec` mem_alloc `gc/sec`
#> <bch:expr> <bch:tm> <bch:tm> <dbl> <bch:byt> <dbl>
#> 1 f0() 25.23ms 30ms 33.3 39.3KB 1.96
#> 2 fp() 4.25s 4.25s 0.235 100.4KB 2.35
(ben_tk$median[2] - ben_tk$median[1]) / 1e5
#> [1] 42.2µs
Traditional API
f0 <- function(n = 1e5) {
x <- 0
for (i in 1:n) {
x <- x + i %% 2
}
x
}
fp <- function(n = 1e5) {
cli_progress_bar(total = n)
x <- 0
for (i in 1:n) {
x <- x + i %% 2
cli_progress_update()
}
x
}
ff <- function(n = 1e5) {
cli_progress_bar(total = n)
x <- 0
for (i in 1:n) {
x <- x + i %% 2
if (`__cli_update_due`) cli_progress_update()
}
x
}
ben_api <- bench::mark(f0(), ff(), fp())
#> Warning: Some expressions had a GC in every iteration; so filtering is
#> disabled.
ben_api
#> # A tibble: 3 × 6
#> expression min median `itr/sec` mem_alloc `gc/sec`
#> <bch:expr> <bch:tm> <bch:tm> <dbl> <bch:byt> <dbl>
#> 1 f0() 22.09ms 42.35ms 25.7 18.7KB 3.96
#> 2 ff() 31.13ms 50.62ms 21.1 27.6KB 1.92
#> 3 fp() 2.38s 2.38s 0.419 25.1KB 2.10
(ben_api$median[3] - ben_api$median[1]) / 1e5
#> [1] 23.4µs
(ben_api$median[2] - ben_api$median[1]) / 1e5
#> [1] 82.7ns
ben_api2 <- bench::mark(f0(1e6), ff(1e6), fp(1e6))
#> Warning: Some expressions had a GC in every iteration; so filtering is
#> disabled.
ben_api2
#> # A tibble: 3 × 6
#> expression min median `itr/sec` mem_alloc `gc/sec`
#> <bch:expr> <bch:tm> <bch:tm> <dbl> <bch:byt> <dbl>
#> 1 f0(1e+06) 373ms 391.2ms 2.56 0B 2.56
#> 2 ff(1e+06) 462.1ms 465.5ms 2.15 1.88KB 2.15
#> 3 fp(1e+06) 23.3s 23.3s 0.0430 1.88KB 2.28
(ben_api2$median[3] - ben_api2$median[1]) / 1e6
#> [1] 22.9µs
(ben_api2$median[2] - ben_api2$median[1]) / 1e6
#> [1] 74.3ns
C benchmarks
Baseline function:
SEXP test_baseline() {
int i;
int res = 0;
for (i = 0; i < 2000000000; i++) {
res += i % 2;
}
return ScalarInteger(res);
}
Switch + modulo check:
SEXP test_modulo(SEXP progress) {
int i;
int res = 0;
int progress_ = LOGICAL(progress)[0];
for (i = 0; i < 2000000000; i++) {
if (i % 10000 == 0 && progress_) cli_progress_set(R_NilValue, i);
res += i % 2;
}
return ScalarInteger(res);
}
cli progress bar API:
SEXP test_cli() {
int i;
int res = 0;
SEXP bar = PROTECT(cli_progress_bar(2000000000, NULL));
for (i = 0; i < 2000000000; i++) {
if (CLI_SHOULD_TICK) cli_progress_set(bar, i);
res += i % 2;
}
cli_progress_done(bar);
UNPROTECT(1);
return ScalarInteger(res);
}
SEXP test_cli_unroll() {
int i = 0;
int res = 0;
SEXP bar = PROTECT(cli_progress_bar(2000000000, NULL));
int s, final, step = 2000000000 / 100000;
for (s = 0; s < 100000; s++) {
if (CLI_SHOULD_TICK) cli_progress_set(bar, i);
final = (s + 1) * step;
for (i = s * step; i < final; i++) {
res += i % 2;
}
}
cli_progress_done(bar);
UNPROTECT(1);
return ScalarInteger(res);
}
library(progresstest)
ben_c <- bench::mark(
test_baseline(),
test_modulo(),
test_cli(),
test_cli_unroll()
)
ben_c
#> # A tibble: 4 × 6
#> expression min median `itr/sec` mem_alloc `gc/sec`
#> <bch:expr> <bch:tm> <bch:tm> <dbl> <bch:byt> <dbl>
#> 1 test_baseline() 621.99ms 621.99ms 1.61 2.08KB 0
#> 2 test_modulo() 1.25s 1.25s 0.803 2.24KB 0
#> 3 test_cli() 1.24s 1.24s 0.804 23.89KB 0
#> 4 test_cli_unroll() 639.62ms 639.62ms 1.56 3.56KB 0
(ben_c$median[3] - ben_c$median[1]) / 2000000000
#> [1] 1ns
Display update
We only update the display a fixed number of times per second. (Currently maximum five times per second.)
Let’s measure how long a single update takes.
Iterator with a bar
cli_progress_bar(total = 100000)
bench::mark(cli_progress_update(force = TRUE), max_iterations = 10000)
#> ■ 0% | ETA: 4m
#> ■ 0% | ETA: 2h
#> ■ 0% | ETA: 1h
#> ■ 0% | ETA: 1h
#> ■ 0% | ETA: 1h
#> ■ 0% | ETA: 44m
#> ■ 0% | ETA: 39m
#> ■ 0% | ETA: 36m
#> ■ 0% | ETA: 33m
#> ■ 0% | ETA: 31m
#> ■ 0% | ETA: 29m
#> ■ 0% | ETA: 27m
#> ■ 0% | ETA: 26m
#> ■ 0% | ETA: 25m
#> ■ 0% | ETA: 24m
#> ■ 0% | ETA: 23m
#> ■ 0% | ETA: 22m
#> ■ 0% | ETA: 22m
#> ■ 0% | ETA: 21m
#> ■ 0% | ETA: 20m
#> ■ 0% | ETA: 20m
#> ■ 0% | ETA: 20m
#> ■ 0% | ETA: 19m
#> ■ 0% | ETA: 19m
#> ■ 0% | ETA: 18m
#> ■ 0% | ETA: 18m
#> ■ 0% | ETA: 18m
#> ■ 0% | ETA: 18m
#> ■ 0% | ETA: 17m
#> ■ 0% | ETA: 17m
#> ■ 0% | ETA: 17m
#> ■ 0% | ETA: 17m
#> ■ 0% | ETA: 17m
#> ■ 0% | ETA: 16m
#> ■ 0% | ETA: 16m
#> ■ 0% | ETA: 16m
#> ■ 0% | ETA: 16m
#> ■ 0% | ETA: 16m
#> ■ 0% | ETA: 16m
#> ■ 0% | ETA: 15m
#> ■ 0% | ETA: 15m
#> ■ 0% | ETA: 15m
#> ■ 0% | ETA: 15m
#> ■ 0% | ETA: 15m
#> ■ 0% | ETA: 15m
#> ■ 0% | ETA: 15m
#> ■ 0% | ETA: 15m
#> ■ 0% | ETA: 15m
#> ■ 0% | ETA: 15m
#> ■ 0% | ETA: 14m
#> ■ 0% | ETA: 14m
#> ■ 0% | ETA: 14m
#> ■ 0% | ETA: 14m
#> ■ 0% | ETA: 14m
#> ■ 0% | ETA: 14m
#> ■ 0% | ETA: 14m
#> ■ 0% | ETA: 14m
#> ■ 0% | ETA: 14m
#> ■ 0% | ETA: 14m
#> ■ 0% | ETA: 14m
#> ■ 0% | ETA: 14m
#> ■ 0% | ETA: 14m
#> ■ 0% | ETA: 14m
#> ■ 0% | ETA: 14m
#> ■ 0% | ETA: 14m
#> ■ 0% | ETA: 13m
#> ■ 0% | ETA: 14m
#> ■ 0% | ETA: 14m
#> ■ 0% | ETA: 13m
#> ■ 0% | ETA: 13m
#> ■ 0% | ETA: 13m
#> ■ 0% | ETA: 13m
#> ■ 0% | ETA: 13m
#> ■ 0% | ETA: 13m
#> ■ 0% | ETA: 13m
#> ■ 0% | ETA: 13m
#> ■ 0% | ETA: 13m
#> ■ 0% | ETA: 13m
#> ■ 0% | ETA: 13m
#> ■ 0% | ETA: 13m
#> # A tibble: 1 × 6
#> expression min median `itr/sec` mem_alloc `gc/sec`
#> <bch:expr> <bch:> <bch:> <dbl> <bch:byt> <dbl>
#> 1 cli_progress_update(force… 6.11ms 6.24ms 158. 1.4MB 2.02
cli_progress_done()
Iterator without a bar
cli_progress_bar(total = NA)
bench::mark(cli_progress_update(force = TRUE), max_iterations = 10000)
#> ⠙ 1 done (528/s) | 3ms
#> ⠹ 2 done (71/s) | 29ms
#> ⠸ 3 done (85/s) | 36ms
#> ⠼ 4 done (94/s) | 43ms
#> ⠴ 5 done (100/s) | 51ms
#> ⠦ 6 done (105/s) | 58ms
#> ⠧ 7 done (109/s) | 65ms
#> ⠇ 8 done (112/s) | 72ms
#> ⠏ 9 done (114/s) | 79ms
#> ⠋ 10 done (116/s) | 87ms
#> ⠙ 11 done (118/s) | 94ms
#> ⠹ 12 done (119/s) | 101ms
#> ⠸ 13 done (121/s) | 108ms
#> ⠼ 14 done (122/s) | 116ms
#> ⠴ 15 done (123/s) | 123ms
#> ⠦ 16 done (124/s) | 130ms
#> ⠧ 17 done (124/s) | 137ms
#> ⠇ 18 done (125/s) | 145ms
#> ⠏ 19 done (126/s) | 152ms
#> ⠋ 20 done (126/s) | 159ms
#> ⠙ 21 done (127/s) | 167ms
#> ⠹ 22 done (127/s) | 174ms
#> ⠸ 23 done (127/s) | 181ms
#> ⠼ 24 done (128/s) | 188ms
#> ⠴ 25 done (128/s) | 196ms
#> ⠦ 26 done (129/s) | 203ms
#> ⠧ 27 done (129/s) | 210ms
#> ⠇ 28 done (129/s) | 218ms
#> ⠏ 29 done (129/s) | 225ms
#> ⠋ 30 done (130/s) | 232ms
#> ⠙ 31 done (130/s) | 239ms
#> ⠹ 32 done (130/s) | 247ms
#> ⠸ 33 done (130/s) | 254ms
#> ⠼ 34 done (131/s) | 261ms
#> ⠴ 35 done (131/s) | 268ms
#> ⠦ 36 done (131/s) | 276ms
#> ⠧ 37 done (131/s) | 283ms
#> ⠇ 38 done (131/s) | 290ms
#> ⠏ 39 done (131/s) | 298ms
#> ⠋ 40 done (131/s) | 305ms
#> ⠙ 41 done (132/s) | 312ms
#> ⠹ 42 done (132/s) | 319ms
#> ⠸ 43 done (132/s) | 327ms
#> ⠼ 44 done (132/s) | 334ms
#> ⠴ 45 done (132/s) | 341ms
#> ⠦ 46 done (132/s) | 349ms
#> ⠧ 47 done (132/s) | 356ms
#> ⠇ 48 done (132/s) | 363ms
#> ⠏ 49 done (133/s) | 370ms
#> ⠋ 50 done (132/s) | 381ms
#> ⠙ 51 done (132/s) | 388ms
#> ⠹ 52 done (132/s) | 395ms
#> ⠸ 53 done (132/s) | 402ms
#> ⠼ 54 done (132/s) | 409ms
#> ⠴ 55 done (132/s) | 417ms
#> ⠦ 56 done (132/s) | 424ms
#> ⠧ 57 done (132/s) | 431ms
#> ⠇ 58 done (133/s) | 438ms
#> ⠏ 59 done (133/s) | 445ms
#> ⠋ 60 done (133/s) | 453ms
#> ⠙ 61 done (132/s) | 462ms
#> ⠹ 62 done (132/s) | 470ms
#> ⠸ 63 done (132/s) | 477ms
#> ⠼ 64 done (132/s) | 485ms
#> ⠴ 65 done (132/s) | 492ms
#> ⠦ 66 done (132/s) | 499ms
#> ⠧ 67 done (132/s) | 507ms
#> ⠇ 68 done (132/s) | 514ms
#> ⠏ 69 done (133/s) | 521ms
#> ⠋ 70 done (133/s) | 528ms
#> # A tibble: 1 × 6
#> expression min median `itr/sec` mem_alloc `gc/sec`
#> <bch:expr> <bch:> <bch:> <dbl> <bch:byt> <dbl>
#> 1 cli_progress_update(force… 7.12ms 7.25ms 137. 265KB 2.02
cli_progress_done()