update week 37

Author: mhjensen

doc/pub/week37/html/._week37-bs000.html

@@ -123,6 +123,28 @@
                2,
                None,
                'second-moment-of-the-gradient'),
+              ('Challenge: Choosing a Fixed Learning Rate',
+               2,
+               None,
+               'challenge-choosing-a-fixed-learning-rate'),
+              ('Motivation for Adaptive Step Sizes',
+               2,
+               None,
+               'motivation-for-adaptive-step-sizes'),
+              ('Derivation of the AdaGrad Algorithm',
+               2,
+               None,
+               'derivation-of-the-adagrad-algorithm'),
+              ('AdaGrad Update Rule Derivation',
+               2,
+               None,
+               'adagrad-update-rule-derivation'),
+              ('AdaGrad Properties', 2, None, 'adagrad-properties'),
+              ('RMSProp: Adaptive Learning Rates',
+               2,
+               None,
+               'rmsprop-adaptive-learning-rates'),
+              ('Adam Optimizer', 2, None, 'adam-optimizer'),
               ('RMS prop', 2, None, 'rms-prop'),
               ('"ADAM optimizer":"https://arxiv.org/abs/1412.6980"',
                2,
@@ -247,7 +269,7 @@
      <!-- navigation toc: --> <li><a href="._week37-bs012.html#program-example-for-gradient-descent-with-ridge-regression" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
      <!-- navigation toc: --> <li><a href="._week37-bs013.html#using-gradient-descent-methods-limitations" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
      <!-- navigation toc: --> <li><a href="._week37-bs014.html#improving-gradient-descent-with-momentum" style="font-size: 80%;">Improving gradient descent with momentum</a></li>
-     <!-- navigation toc: --> <li><a href="._week37-bs043.html#same-code-but-now-with-momentum-gradient-descent" style="font-size: 80%;">Same code but now with momentum gradient descent</a></li>
+     <!-- navigation toc: --> <li><a href="._week37-bs050.html#same-code-but-now-with-momentum-gradient-descent" style="font-size: 80%;">Same code but now with momentum gradient descent</a></li>
      <!-- navigation toc: --> <li><a href="._week37-bs016.html#overview-video-on-stochastic-gradient-descent-sgd" style="font-size: 80%;">Overview video on Stochastic Gradient Descent (SGD)</a></li>
      <!-- navigation toc: --> <li><a href="._week37-bs017.html#batches-and-mini-batches" style="font-size: 80%;">Batches and mini-batches</a></li>
      <!-- navigation toc: --> <li><a href="._week37-bs018.html#pros-and-cons" style="font-size: 80%;">Pros and cons</a></li>
@@ -268,26 +290,33 @@
      <!-- navigation toc: --> <li><a href="._week37-bs033.html#more-on-momentum-based-approaches" style="font-size: 80%;">More on momentum based approaches</a></li>
      <!-- navigation toc: --> <li><a href="._week37-bs034.html#momentum-parameter" style="font-size: 80%;">Momentum parameter</a></li>
      <!-- navigation toc: --> <li><a href="._week37-bs035.html#second-moment-of-the-gradient" style="font-size: 80%;">Second moment of the gradient</a></li>
-     <!-- navigation toc: --> <li><a href="._week37-bs036.html#rms-prop" style="font-size: 80%;">RMS prop</a></li>
-     <!-- navigation toc: --> <li><a href="._week37-bs037.html#adam-optimizer-https-arxiv-org-abs-1412-6980" style="font-size: 80%;">"ADAM optimizer":"https://arxiv.org/abs/1412.6980"</a></li>
-     <!-- navigation toc: --> <li><a href="._week37-bs038.html#algorithms-and-codes-for-adagrad-rmsprop-and-adam" style="font-size: 80%;">Algorithms and codes for Adagrad, RMSprop and Adam</a></li>
-     <!-- navigation toc: --> <li><a href="._week37-bs038.html#adagrad-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html" style="font-size: 80%;">AdaGrad algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"</a></li>
-     <!-- navigation toc: --> <li><a href="._week37-bs038.html#rmsprop-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html" style="font-size: 80%;">RMSProp algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"</a></li>
-     <!-- navigation toc: --> <li><a href="._week37-bs038.html#adam-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html" style="font-size: 80%;">ADAM algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"</a></li>
-     <!-- navigation toc: --> <li><a href="._week37-bs039.html#practical-tips" style="font-size: 80%;">Practical tips</a></li>
-     <!-- navigation toc: --> <li><a href="._week37-bs040.html#sneaking-in-automatic-differentiation-using-autograd" style="font-size: 80%;">Sneaking in automatic differentiation using Autograd</a></li>
-     <!-- navigation toc: --> <li><a href="._week37-bs043.html#same-code-but-now-with-momentum-gradient-descent" style="font-size: 80%;">Same code but now with momentum gradient descent</a></li>
-     <!-- navigation toc: --> <li><a href="._week37-bs042.html#including-stochastic-gradient-descent-with-autograd" style="font-size: 80%;">Including Stochastic Gradient Descent with Autograd</a></li>
-     <!-- navigation toc: --> <li><a href="._week37-bs043.html#same-code-but-now-with-momentum-gradient-descent" style="font-size: 80%;">Same code but now with momentum gradient descent</a></li>
-     <!-- navigation toc: --> <li><a href="._week37-bs044.html#but-none-of-these-can-compete-with-newton-s-method" style="font-size: 80%;">But none of these can compete with Newton's method</a></li>
-     <!-- navigation toc: --> <li><a href="._week37-bs045.html#similar-second-order-function-now-problem-but-now-with-adagrad" style="font-size: 80%;">Similar (second order function now) problem but now with AdaGrad</a></li>
-     <!-- navigation toc: --> <li><a href="._week37-bs046.html#rmsprop-for-adaptive-learning-rate-with-stochastic-gradient-descent" style="font-size: 80%;">RMSprop for adaptive learning rate with Stochastic Gradient Descent</a></li>
-     <!-- navigation toc: --> <li><a href="._week37-bs047.html#and-finally-adam-https-arxiv-org-pdf-1412-6980-pdf" style="font-size: 80%;">And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"</a></li>
-     <!-- navigation toc: --> <li><a href="._week37-bs048.html#material-for-the-lab-sessions" style="font-size: 80%;">Material for the lab sessions</a></li>
-     <!-- navigation toc: --> <li><a href="._week37-bs049.html#reminder-on-different-scaling-methods" style="font-size: 80%;">Reminder on different scaling methods</a></li>
-     <!-- navigation toc: --> <li><a href="._week37-bs050.html#functionality-in-scikit-learn" style="font-size: 80%;">Functionality in Scikit-Learn</a></li>
-     <!-- navigation toc: --> <li><a href="._week37-bs051.html#more-preprocessing" style="font-size: 80%;">More preprocessing</a></li>
-     <!-- navigation toc: --> <li><a href="._week37-bs052.html#frequently-used-scaling-functions" style="font-size: 80%;">Frequently used scaling functions</a></li>
+     <!-- navigation toc: --> <li><a href="._week37-bs036.html#challenge-choosing-a-fixed-learning-rate" style="font-size: 80%;">Challenge: Choosing a Fixed Learning Rate</a></li>
+     <!-- navigation toc: --> <li><a href="._week37-bs037.html#motivation-for-adaptive-step-sizes" style="font-size: 80%;">Motivation for Adaptive Step Sizes</a></li>
+     <!-- navigation toc: --> <li><a href="._week37-bs038.html#derivation-of-the-adagrad-algorithm" style="font-size: 80%;">Derivation of the AdaGrad Algorithm</a></li>
+     <!-- navigation toc: --> <li><a href="._week37-bs039.html#adagrad-update-rule-derivation" style="font-size: 80%;">AdaGrad Update Rule Derivation</a></li>
+     <!-- navigation toc: --> <li><a href="._week37-bs040.html#adagrad-properties" style="font-size: 80%;">AdaGrad Properties</a></li>
+     <!-- navigation toc: --> <li><a href="._week37-bs041.html#rmsprop-adaptive-learning-rates" style="font-size: 80%;">RMSProp: Adaptive Learning Rates</a></li>
+     <!-- navigation toc: --> <li><a href="._week37-bs042.html#adam-optimizer" style="font-size: 80%;">Adam Optimizer</a></li>
+     <!-- navigation toc: --> <li><a href="._week37-bs043.html#rms-prop" style="font-size: 80%;">RMS prop</a></li>
+     <!-- navigation toc: --> <li><a href="._week37-bs044.html#adam-optimizer-https-arxiv-org-abs-1412-6980" style="font-size: 80%;">"ADAM optimizer":"https://arxiv.org/abs/1412.6980"</a></li>
+     <!-- navigation toc: --> <li><a href="._week37-bs045.html#algorithms-and-codes-for-adagrad-rmsprop-and-adam" style="font-size: 80%;">Algorithms and codes for Adagrad, RMSprop and Adam</a></li>
+     <!-- navigation toc: --> <li><a href="._week37-bs045.html#adagrad-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html" style="font-size: 80%;">AdaGrad algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"</a></li>
+     <!-- navigation toc: --> <li><a href="._week37-bs045.html#rmsprop-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html" style="font-size: 80%;">RMSProp algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"</a></li>
+     <!-- navigation toc: --> <li><a href="._week37-bs045.html#adam-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html" style="font-size: 80%;">ADAM algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"</a></li>
+     <!-- navigation toc: --> <li><a href="._week37-bs046.html#practical-tips" style="font-size: 80%;">Practical tips</a></li>
+     <!-- navigation toc: --> <li><a href="._week37-bs047.html#sneaking-in-automatic-differentiation-using-autograd" style="font-size: 80%;">Sneaking in automatic differentiation using Autograd</a></li>
+     <!-- navigation toc: --> <li><a href="._week37-bs050.html#same-code-but-now-with-momentum-gradient-descent" style="font-size: 80%;">Same code but now with momentum gradient descent</a></li>
+     <!-- navigation toc: --> <li><a href="._week37-bs049.html#including-stochastic-gradient-descent-with-autograd" style="font-size: 80%;">Including Stochastic Gradient Descent with Autograd</a></li>
+     <!-- navigation toc: --> <li><a href="._week37-bs050.html#same-code-but-now-with-momentum-gradient-descent" style="font-size: 80%;">Same code but now with momentum gradient descent</a></li>
+     <!-- navigation toc: --> <li><a href="._week37-bs051.html#but-none-of-these-can-compete-with-newton-s-method" style="font-size: 80%;">But none of these can compete with Newton's method</a></li>
+     <!-- navigation toc: --> <li><a href="._week37-bs052.html#similar-second-order-function-now-problem-but-now-with-adagrad" style="font-size: 80%;">Similar (second order function now) problem but now with AdaGrad</a></li>
+     <!-- navigation toc: --> <li><a href="._week37-bs053.html#rmsprop-for-adaptive-learning-rate-with-stochastic-gradient-descent" style="font-size: 80%;">RMSprop for adaptive learning rate with Stochastic Gradient Descent</a></li>
+     <!-- navigation toc: --> <li><a href="._week37-bs054.html#and-finally-adam-https-arxiv-org-pdf-1412-6980-pdf" style="font-size: 80%;">And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"</a></li>
+     <!-- navigation toc: --> <li><a href="._week37-bs055.html#material-for-the-lab-sessions" style="font-size: 80%;">Material for the lab sessions</a></li>
+     <!-- navigation toc: --> <li><a href="._week37-bs056.html#reminder-on-different-scaling-methods" style="font-size: 80%;">Reminder on different scaling methods</a></li>
+     <!-- navigation toc: --> <li><a href="._week37-bs057.html#functionality-in-scikit-learn" style="font-size: 80%;">Functionality in Scikit-Learn</a></li>
+     <!-- navigation toc: --> <li><a href="._week37-bs058.html#more-preprocessing" style="font-size: 80%;">More preprocessing</a></li>
+     <!-- navigation toc: --> <li><a href="._week37-bs059.html#frequently-used-scaling-functions" style="font-size: 80%;">Frequently used scaling functions</a></li>
 
         </ul>
       </li>
@@ -341,7 +370,7 @@ <h4>September 8-12, 2025</h4>
   <li><a href="._week37-bs008.html">9</a></li>
   <li><a href="._week37-bs009.html">10</a></li>
   <li><a href="">...</a></li>
-  <li><a href="._week37-bs052.html">53</a></li>
+  <li><a href="._week37-bs059.html">60</a></li>
   <li><a href="._week37-bs001.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->