From 87a00a8a10c7f60133fc3d0dd9a32fdab35d5dbd Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 31 Oct 2026 06:16:50 +0000 Subject: [PATCH 1/6] Add momentum and the momentum coefficient Moved from the lds gradient descent chapter, reworded to mds notation (f, step length, iterates); lds will link here. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_019xfc9ow6EykAkrucpc8Fyw --- .../_sec_optimization_gradient_descent.qmd | 16 ++++++ .../vector-calc/_def-momentum-coefficient.qmd | 14 +++++ _subfiles/vector-calc/_def-momentum.qmd | 19 +++++++ .../vector-calc/_exm-momentum-coefficient.qmd | 41 ++++++++++++++ _subfiles/vector-calc/_exm-momentum.qmd | 56 +++++++++++++++++++ 5 files changed, 146 insertions(+) create mode 100644 _subfiles/vector-calc/_def-momentum-coefficient.qmd create mode 100644 _subfiles/vector-calc/_def-momentum.qmd create mode 100644 _subfiles/vector-calc/_exm-momentum-coefficient.qmd create mode 100644 _subfiles/vector-calc/_exm-momentum.qmd diff --git a/_subfiles/_sec_optimization_gradient_descent.qmd b/_subfiles/_sec_optimization_gradient_descent.qmd index a4356bde9..6d6eacf57 100644 --- a/_subfiles/_sec_optimization_gradient_descent.qmd +++ b/_subfiles/_sec_optimization_gradient_descent.qmd @@ -153,3 +153,19 @@ {{< slidebreak >}} {{< include mds/_subfiles/vector-calc/_sol-best-step-count.qmd >}} + +{{< slidebreak >}} + +{{< include mds/_subfiles/vector-calc/_def-momentum-coefficient.qmd >}} + +{{< slidebreak >}} + +{{< include mds/_subfiles/vector-calc/_exm-momentum-coefficient.qmd >}} + +{{< slidebreak >}} + +{{< include mds/_subfiles/vector-calc/_def-momentum.qmd >}} + +{{< slidebreak >}} + +{{< include mds/_subfiles/vector-calc/_exm-momentum.qmd >}} diff --git a/_subfiles/vector-calc/_def-momentum-coefficient.qmd b/_subfiles/vector-calc/_def-momentum-coefficient.qmd new file mode 100644 index 000000000..2530dc42d --- /dev/null +++ b/_subfiles/vector-calc/_def-momentum-coefficient.qmd @@ -0,0 +1,14 @@ +:::{#def-momentum-coefficient} +#### Momentum coefficient + +Let $\vg^{(0)}, \vg^{(1)}, \ldots$ be vectors in $\reals^p$, +and let $\vv^{(t)}$ be a running velocity built from them. +The **momentum coefficient** $\beta \in [0, 1)$ is the weight on the previous velocity in + +$$ +\vv^{(t)} \eqdef \beta\,\vv^{(t-1)} + \vg^{(t)}, +$$ + +with $\vv^{(-1)} \eqdef \vzero$. + +::: diff --git a/_subfiles/vector-calc/_def-momentum.qmd b/_subfiles/vector-calc/_def-momentum.qmd new file mode 100644 index 000000000..7686bde00 --- /dev/null +++ b/_subfiles/vector-calc/_def-momentum.qmd @@ -0,0 +1,19 @@ +:::{#def-momentum} +#### Momentum (heavy-ball method) + +Let $f : \reals^p \to \reals$ have first partial derivatives, +let $\vx^{(0)}$ be a starting point, +let $\steplen > 0$ be a step length, +and let $\beta \in [0, 1)$ be a [momentum coefficient](#def-momentum-coefficient). +**Momentum** (the **heavy-ball method**) computes, for $t = 1, 2, \ldots$, + +$$ +\begin{aligned} +\vv^{(t)} &\eqdef \beta\,\vv^{(t-1)} + \deriv{\vx} f(\vx^{(t-1)}) \\ +\vx^{(t)} &\eqdef \vx^{(t-1)} - \steplen\,\vv^{(t)}, +\end{aligned} +$$ + +with $\vv^{(0)} \eqdef \vzero$. + +::: diff --git a/_subfiles/vector-calc/_exm-momentum-coefficient.qmd b/_subfiles/vector-calc/_exm-momentum-coefficient.qmd new file mode 100644 index 000000000..5771bbb31 --- /dev/null +++ b/_subfiles/vector-calc/_exm-momentum-coefficient.qmd @@ -0,0 +1,41 @@ +:::{#exm-momentum-coefficient} +#### How far back the velocity remembers + +Unrolling the recursion, the vector $\vg^{(t-k)}$ enters $\vv^{(t)}$ with weight $\beta^k$. +These weights sum to at most $1 / (1 - \beta)$, +so the velocity remembers roughly the last $1 / (1 - \beta)$ vectors. + +::: {.panel-tabset group="language"} + +### R + +```{r} +#| label: momentum-coefficient-r +#| code-fold: false +beta_mc <- c(0, 0.5, 0.9, 0.99) +k_mc <- 10 +weight_mc <- beta_mc^k_mc +memory_mc <- 1 / (1 - beta_mc) +weight_rounded_mc <- signif(weight_mc, 2) +rbind(beta = beta_mc, weight = weight_rounded_mc, memory = memory_mc) +``` + +### Python + +```{python} +#| label: momentum-coefficient-python +#| code-fold: false +beta_mc = [0, 0.5, 0.9, 0.99] +k_mc = 10 +for b_mc in beta_mc: + print(b_mc, float(f"{b_mc**k_mc:.2g}"), round(1 / (1 - b_mc))); +``` + +::: + +At $\beta = 0$ the velocity is the current vector alone, +and at $\beta = 0.99$ it averages over about `r round(memory_mc[4])` vectors. + + + +::: diff --git a/_subfiles/vector-calc/_exm-momentum.qmd b/_subfiles/vector-calc/_exm-momentum.qmd new file mode 100644 index 000000000..f9d8c2ae6 --- /dev/null +++ b/_subfiles/vector-calc/_exm-momentum.qmd @@ -0,0 +1,56 @@ +:::{#exm-momentum} +#### Momentum on a one-dimensional quadratic + +Let $f(x) = \tfrac{1}{2} x^2$, so $\deriv{x} f(x) = x$. +Start at $x^{(0)} = 1$ with $\steplen = 0.1$, +and take $20$ updates with $\beta = 0$, which is plain [gradient descent](#def-gradient-descent), and with $\beta = 0.9$: + +::: {.panel-tabset group="language"} + +### R + +```{r} +#| label: momentum-r +#| code-fold: false +x0_mo <- 1 +s_mo <- 0.1 +beta_mo <- c(plain = 0, momentum = 0.9) +n_mo <- 20 +x_mo <- rep(x0_mo, 2) +v_mo <- c(0, 0) +path_mo <- matrix(NA_real_, nrow = n_mo, ncol = 2) +for (k_mo in seq_len(n_mo)) { + v_mo <- beta_mo * v_mo + x_mo + x_mo <- x_mo - s_mo * v_mo + path_mo[k_mo, ] <- x_mo +} +last_mo <- path_mo[n_mo, ] +lowest_mo <- apply(path_mo, 2, min) +summary_mo <- rbind(last = last_mo, lowest = lowest_mo) +round(summary_mo, 4) +``` + +### Python + +```{python} +#| label: momentum-python +#| code-fold: false +s_mo, n_mo = 0.1, 20 +for beta_mo in (0.0, 0.9): + x_mo, v_mo, path_mo = 1.0, 0.0, [] + for _ in range(n_mo): + v_mo = beta_mo * v_mo + x_mo + x_mo = x_mo - s_mo * v_mo + path_mo.append(x_mo) + print(beta_mo, round(path_mo[-1], 4), round(min(path_mo), 4)); +``` + +::: + +After $20$ updates, plain descent has reached `r signif(last_mo[1], 3)` and momentum `r signif(last_mo[2], 3)`. +Momentum overshoots the minimizer $0$, down to `r signif(lowest_mo[2], 3)` at update `r which.min(path_mo[, 2])`, and then swings back. +On a cost this simple, with a small step length, the swing makes momentum slower than plain descent. + + + +::: From 9a4f4143d60e40f2980d43205d3effa581c8ce03 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 31 Oct 2026 06:17:19 +0000 Subject: [PATCH 2/6] Link first uses of defined terms in the momentum pages Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_019xfc9ow6EykAkrucpc8Fyw --- _subfiles/vector-calc/_def-momentum-coefficient.qmd | 2 +- _subfiles/vector-calc/_def-momentum.qmd | 6 +++--- _subfiles/vector-calc/_exm-momentum-coefficient.qmd | 6 +++--- _subfiles/vector-calc/_exm-momentum.qmd | 6 +++--- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/_subfiles/vector-calc/_def-momentum-coefficient.qmd b/_subfiles/vector-calc/_def-momentum-coefficient.qmd index 2530dc42d..7e09d9557 100644 --- a/_subfiles/vector-calc/_def-momentum-coefficient.qmd +++ b/_subfiles/vector-calc/_def-momentum-coefficient.qmd @@ -3,7 +3,7 @@ Let $\vg^{(0)}, \vg^{(1)}, \ldots$ be vectors in $\reals^p$, and let $\vv^{(t)}$ be a running velocity built from them. -The **momentum coefficient** $\beta \in [0, 1)$ is the weight on the previous velocity in +The **momentum coefficient** $\beta \in [0, 1)$ is the factor that multiplies the previous velocity in $$ \vv^{(t)} \eqdef \beta\,\vv^{(t-1)} + \vg^{(t)}, diff --git a/_subfiles/vector-calc/_def-momentum.qmd b/_subfiles/vector-calc/_def-momentum.qmd index 7686bde00..d418ae35c 100644 --- a/_subfiles/vector-calc/_def-momentum.qmd +++ b/_subfiles/vector-calc/_def-momentum.qmd @@ -1,11 +1,11 @@ :::{#def-momentum} #### Momentum (heavy-ball method) -Let $f : \reals^p \to \reals$ have first partial derivatives, +Let $f : \reals^p \to \reals$ have first [partial derivatives](vector-calculus.qmd#def-partial-derivative), let $\vx^{(0)}$ be a starting point, -let $\steplen > 0$ be a step length, +let $\steplen > 0$ be a [step length](#def-gradient-descent), and let $\beta \in [0, 1)$ be a [momentum coefficient](#def-momentum-coefficient). -**Momentum** (the **heavy-ball method**) computes, for $t = 1, 2, \ldots$, +**Momentum** (the **heavy-ball method**) is the sequence given, for $t = 1, 2, \ldots$, by $$ \begin{aligned} diff --git a/_subfiles/vector-calc/_exm-momentum-coefficient.qmd b/_subfiles/vector-calc/_exm-momentum-coefficient.qmd index 5771bbb31..35c19171b 100644 --- a/_subfiles/vector-calc/_exm-momentum-coefficient.qmd +++ b/_subfiles/vector-calc/_exm-momentum-coefficient.qmd @@ -1,8 +1,8 @@ :::{#exm-momentum-coefficient} #### How far back the velocity remembers -Unrolling the recursion, the vector $\vg^{(t-k)}$ enters $\vv^{(t)}$ with weight $\beta^k$. -These weights sum to at most $1 / (1 - \beta)$, +Unrolling the recursion, the vector $\vg^{(t-k)}$ enters $\vv^{(t)}$ with factor $\beta^k$. +These factors add up to at most $1 / (1 - \beta)$, so the velocity remembers roughly the last $1 / (1 - \beta)$ vectors. ::: {.panel-tabset group="language"} @@ -34,7 +34,7 @@ for b_mc in beta_mc: ::: At $\beta = 0$ the velocity is the current vector alone, -and at $\beta = 0.99$ it averages over about `r round(memory_mc[4])` vectors. +and at $\beta = 0.99$ it combines about `r round(memory_mc[4])` vectors. diff --git a/_subfiles/vector-calc/_exm-momentum.qmd b/_subfiles/vector-calc/_exm-momentum.qmd index f9d8c2ae6..ae264a057 100644 --- a/_subfiles/vector-calc/_exm-momentum.qmd +++ b/_subfiles/vector-calc/_exm-momentum.qmd @@ -47,9 +47,9 @@ for beta_mo in (0.0, 0.9): ::: -After $20$ updates, plain descent has reached `r signif(last_mo[1], 3)` and momentum `r signif(last_mo[2], 3)`. -Momentum overshoots the minimizer $0$, down to `r signif(lowest_mo[2], 3)` at update `r which.min(path_mo[, 2])`, and then swings back. -On a cost this simple, with a small step length, the swing makes momentum slower than plain descent. +After $20$ updates, plain descent has reached `r signif(last_mo[1], 3)` and [momentum](#def-momentum) `r signif(last_mo[2], 3)`. +Momentum overshoots the [minimizer](algebra.qmd#def-global-minimizer) $0$, down to `r signif(lowest_mo[2], 3)` at update `r which.min(path_mo[, 2])`, and then swings back. +On a cost this simple, with a small [step length](#def-gradient-descent), the swing makes momentum slower than plain descent. From cec614b952085b34c681e388d360307e01fa566f Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 31 Oct 2026 06:17:29 +0000 Subject: [PATCH 3/6] Reword the momentum update to avoid an unlinked term Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_019xfc9ow6EykAkrucpc8Fyw --- _subfiles/vector-calc/_def-momentum.qmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_subfiles/vector-calc/_def-momentum.qmd b/_subfiles/vector-calc/_def-momentum.qmd index d418ae35c..6e71c0b72 100644 --- a/_subfiles/vector-calc/_def-momentum.qmd +++ b/_subfiles/vector-calc/_def-momentum.qmd @@ -5,7 +5,7 @@ Let $f : \reals^p \to \reals$ have first [partial derivatives](vector-calculus.q let $\vx^{(0)}$ be a starting point, let $\steplen > 0$ be a [step length](#def-gradient-descent), and let $\beta \in [0, 1)$ be a [momentum coefficient](#def-momentum-coefficient). -**Momentum** (the **heavy-ball method**) is the sequence given, for $t = 1, 2, \ldots$, by +**Momentum** (the **heavy-ball method**) updates, for $t = 1, 2, \ldots$, by $$ \begin{aligned} From cc49119b15070e7f30fac68ae456998a3009acb6 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 31 Oct 2026 06:20:24 +0000 Subject: [PATCH 4/6] Define velocity in place, align the initial velocity, and qualify the example Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_019xfc9ow6EykAkrucpc8Fyw --- _subfiles/vector-calc/_def-momentum-coefficient.qmd | 6 +++--- _subfiles/vector-calc/_exm-momentum-coefficient.qmd | 8 ++++---- _subfiles/vector-calc/_exm-momentum.qmd | 5 +++-- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/_subfiles/vector-calc/_def-momentum-coefficient.qmd b/_subfiles/vector-calc/_def-momentum-coefficient.qmd index 7e09d9557..8ea28b392 100644 --- a/_subfiles/vector-calc/_def-momentum-coefficient.qmd +++ b/_subfiles/vector-calc/_def-momentum-coefficient.qmd @@ -1,14 +1,14 @@ :::{#def-momentum-coefficient} #### Momentum coefficient -Let $\vg^{(0)}, \vg^{(1)}, \ldots$ be vectors in $\reals^p$, -and let $\vv^{(t)}$ be a running velocity built from them. +Let $\vg^{(1)}, \vg^{(2)}, \ldots$ be vectors in $\reals^p$, +and let $\vv^{(t)}$ be a running **velocity** built from them. The **momentum coefficient** $\beta \in [0, 1)$ is the factor that multiplies the previous velocity in $$ \vv^{(t)} \eqdef \beta\,\vv^{(t-1)} + \vg^{(t)}, $$ -with $\vv^{(-1)} \eqdef \vzero$. +for $t = 1, 2, \ldots$, with $\vv^{(0)} \eqdef \vzero$. ::: diff --git a/_subfiles/vector-calc/_exm-momentum-coefficient.qmd b/_subfiles/vector-calc/_exm-momentum-coefficient.qmd index 35c19171b..19062ec89 100644 --- a/_subfiles/vector-calc/_exm-momentum-coefficient.qmd +++ b/_subfiles/vector-calc/_exm-momentum-coefficient.qmd @@ -1,9 +1,9 @@ :::{#exm-momentum-coefficient} #### How far back the velocity remembers -Unrolling the recursion, the vector $\vg^{(t-k)}$ enters $\vv^{(t)}$ with factor $\beta^k$. -These factors add up to at most $1 / (1 - \beta)$, -so the velocity remembers roughly the last $1 / (1 - \beta)$ vectors. +Unrolling the recursion, the vector $\vg^{(t-k)}$ enters $\vv^{(t)}$ with factor $\beta^k$, for $k < t$. +These factors add up to at most $\sum_{k=0}^{\infty} \beta^k = 1 / (1 - \beta)$. +As a rule of thumb, the [velocity](#def-momentum-coefficient) is therefore dominated by the last $1 / (1 - \beta)$ vectors. ::: {.panel-tabset group="language"} @@ -33,7 +33,7 @@ for b_mc in beta_mc: ::: -At $\beta = 0$ the velocity is the current vector alone, +At $\beta = 0$ the [velocity](#def-momentum-coefficient) is the current vector alone, and at $\beta = 0.99$ it combines about `r round(memory_mc[4])` vectors. diff --git a/_subfiles/vector-calc/_exm-momentum.qmd b/_subfiles/vector-calc/_exm-momentum.qmd index ae264a057..9d31e234d 100644 --- a/_subfiles/vector-calc/_exm-momentum.qmd +++ b/_subfiles/vector-calc/_exm-momentum.qmd @@ -1,7 +1,8 @@ :::{#exm-momentum} #### Momentum on a one-dimensional quadratic -Let $f(x) = \tfrac{1}{2} x^2$, so $\deriv{x} f(x) = x$. +Let $f(x) = \tfrac{1}{2} x^2$, +so $\deriv{x} f(x) = x$. Start at $x^{(0)} = 1$ with $\steplen = 0.1$, and take $20$ updates with $\beta = 0$, which is plain [gradient descent](#def-gradient-descent), and with $\beta = 0.9$: @@ -49,7 +50,7 @@ for beta_mo in (0.0, 0.9): After $20$ updates, plain descent has reached `r signif(last_mo[1], 3)` and [momentum](#def-momentum) `r signif(last_mo[2], 3)`. Momentum overshoots the [minimizer](algebra.qmd#def-global-minimizer) $0$, down to `r signif(lowest_mo[2], 3)` at update `r which.min(path_mo[, 2])`, and then swings back. -On a cost this simple, with a small [step length](#def-gradient-descent), the swing makes momentum slower than plain descent. +In this run, with a small [step length](#def-gradient-descent), the swing leaves momentum behind plain descent after $20$ updates. From b4dd5fc34c0a53acdbc420742e54f533f00bb63f Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 31 Oct 2026 06:21:18 +0000 Subject: [PATCH 5/6] Exempt the matrix rbind calls from the undesirable function lint Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_019xfc9ow6EykAkrucpc8Fyw --- _subfiles/vector-calc/_exm-momentum-coefficient.qmd | 2 +- _subfiles/vector-calc/_exm-momentum.qmd | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/_subfiles/vector-calc/_exm-momentum-coefficient.qmd b/_subfiles/vector-calc/_exm-momentum-coefficient.qmd index 19062ec89..ba377bd8d 100644 --- a/_subfiles/vector-calc/_exm-momentum-coefficient.qmd +++ b/_subfiles/vector-calc/_exm-momentum-coefficient.qmd @@ -17,7 +17,7 @@ k_mc <- 10 weight_mc <- beta_mc^k_mc memory_mc <- 1 / (1 - beta_mc) weight_rounded_mc <- signif(weight_mc, 2) -rbind(beta = beta_mc, weight = weight_rounded_mc, memory = memory_mc) +rbind(beta = beta_mc, weight = weight_rounded_mc, memory = memory_mc) # nolint: undesirable_function_linter. ``` ### Python diff --git a/_subfiles/vector-calc/_exm-momentum.qmd b/_subfiles/vector-calc/_exm-momentum.qmd index 9d31e234d..8f87b1697 100644 --- a/_subfiles/vector-calc/_exm-momentum.qmd +++ b/_subfiles/vector-calc/_exm-momentum.qmd @@ -27,7 +27,7 @@ for (k_mo in seq_len(n_mo)) { } last_mo <- path_mo[n_mo, ] lowest_mo <- apply(path_mo, 2, min) -summary_mo <- rbind(last = last_mo, lowest = lowest_mo) +summary_mo <- rbind(last = last_mo, lowest = lowest_mo) # nolint: undesirable_function_linter. round(summary_mo, 4) ``` From f03f65d66b555d2c9a80f4ef304aaf4613a41c5b Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 31 Oct 2026 06:25:19 +0000 Subject: [PATCH 6/6] Put the rbind lint exemption on its own lines to stay within 80 characters Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_019xfc9ow6EykAkrucpc8Fyw --- _subfiles/vector-calc/_exm-momentum-coefficient.qmd | 4 +++- _subfiles/vector-calc/_exm-momentum.qmd | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/_subfiles/vector-calc/_exm-momentum-coefficient.qmd b/_subfiles/vector-calc/_exm-momentum-coefficient.qmd index ba377bd8d..714c9f427 100644 --- a/_subfiles/vector-calc/_exm-momentum-coefficient.qmd +++ b/_subfiles/vector-calc/_exm-momentum-coefficient.qmd @@ -17,7 +17,9 @@ k_mc <- 10 weight_mc <- beta_mc^k_mc memory_mc <- 1 / (1 - beta_mc) weight_rounded_mc <- signif(weight_mc, 2) -rbind(beta = beta_mc, weight = weight_rounded_mc, memory = memory_mc) # nolint: undesirable_function_linter. +# nolint start: undesirable_function_linter. +rbind(beta = beta_mc, weight = weight_rounded_mc, memory = memory_mc) +# nolint end ``` ### Python diff --git a/_subfiles/vector-calc/_exm-momentum.qmd b/_subfiles/vector-calc/_exm-momentum.qmd index 8f87b1697..2e08109db 100644 --- a/_subfiles/vector-calc/_exm-momentum.qmd +++ b/_subfiles/vector-calc/_exm-momentum.qmd @@ -27,7 +27,9 @@ for (k_mo in seq_len(n_mo)) { } last_mo <- path_mo[n_mo, ] lowest_mo <- apply(path_mo, 2, min) -summary_mo <- rbind(last = last_mo, lowest = lowest_mo) # nolint: undesirable_function_linter. +# nolint start: undesirable_function_linter. +summary_mo <- rbind(last = last_mo, lowest = lowest_mo) +# nolint end round(summary_mo, 4) ```