Skip to content

Commit 9a0d761

Browse files
mhjensenmhjensen
committed
update
1 parent f81a71e commit 9a0d761

21 files changed

+11040
-2323
lines changed

‎doc/pub/week10/ipynb/ipynb-week10-src.tar.gz‎

0 Bytes
Binary file not shown.

‎doc/pub/week10/ipynb/week10.ipynb‎

Lines changed: 552 additions & 1763 deletions
Large diffs are not rendered by default.

‎doc/pub/week10/pdf/week10.pdf‎

-33 KB
Binary file not shown.

‎doc/src/week10/Latexfiles/transformers.tex‎

Lines changed: 716 additions & 0 deletions
Large diffs are not rendered by default.

‎doc/src/week10/_minted/096E567314AD88FE1F78F83D00B63F71.highlight.minted‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
\begin{MintedVerbatim}[commandchars=\\\{\},codes={\catcode`\$=3\catcode`\^=7\catcode`\_=8\relax}]
2+
\PYG{k+kn}{import}\PYG{+w}{ }\PYG{n+nn}{sys}
3+
\PYG{k}{assert} \PYG{n}{sys}\PYG{o}{.}\PYG{n}{version\PYGZus{}info} \PYG{o}{\PYGZgt{}}\PYG{o}{=} \PYG{p}{(}\PYG{l+m+mi}{3}\PYG{p}{,} \PYG{l+m+mi}{5}\PYG{p}{)}
4+
5+
\PYG{c+c1}{\PYGZsh{} Is this notebook running on Colab or Kaggle?}
6+
\PYG{n}{IS\PYGZus{}COLAB} \PYG{o}{=} \PYG{l+s+s2}{\PYGZdq{}}\PYG{l+s+s2}{google.colab}\PYG{l+s+s2}{\PYGZdq{}} \PYG{o+ow}{in} \PYG{n}{sys}\PYG{o}{.}\PYG{n}{modules}
7+
\PYG{n}{IS\PYGZus{}KAGGLE} \PYG{o}{=} \PYG{l+s+s2}{\PYGZdq{}}\PYG{l+s+s2}{kaggle\PYGZus{}secrets}\PYG{l+s+s2}{\PYGZdq{}} \PYG{o+ow}{in} \PYG{n}{sys}\PYG{o}{.}\PYG{n}{modules}
8+
9+
\PYG{c+c1}{\PYGZsh{} Scikit\PYGZhy{}Learn \PYGZgt{}=0.20 is required}
10+
\PYG{k+kn}{import}\PYG{+w}{ }\PYG{n+nn}{sklearn}
11+
\PYG{k}{assert} \PYG{n}{sklearn}\PYG{o}{.}\PYG{n}{\PYGZus{}\PYGZus{}version\PYGZus{}\PYGZus{}} \PYG{o}{\PYGZgt{}}\PYG{o}{=} \PYG{l+s+s2}{\PYGZdq{}}\PYG{l+s+s2}{0.20}\PYG{l+s+s2}{\PYGZdq{}}
12+
13+
\PYG{c+c1}{\PYGZsh{} TensorFlow \PYGZgt{}= 2.0 is required}
14+
\PYG{k+kn}{import}\PYG{+w}{ }\PYG{n+nn}{tensorflow}\PYG{+w}{ }\PYG{k}{as}\PYG{+w}{ }\PYG{n+nn}{tf}
15+
\PYG{k+kn}{from}\PYG{+w}{ }\PYG{n+nn}{tensorflow}\PYG{+w}{ }\PYG{k+kn}{import} \PYG{n}{keras}
16+
\PYG{k}{assert} \PYG{n}{tf}\PYG{o}{.}\PYG{n}{\PYGZus{}\PYGZus{}version\PYGZus{}\PYGZus{}} \PYG{o}{\PYGZgt{}}\PYG{o}{=} \PYG{l+s+s2}{\PYGZdq{}}\PYG{l+s+s2}{2.0}\PYG{l+s+s2}{\PYGZdq{}}
17+
18+
\PYG{k}{if} \PYG{o+ow}{not} \PYG{n}{tf}\PYG{o}{.}\PYG{n}{config}\PYG{o}{.}\PYG{n}{list\PYGZus{}physical\PYGZus{}devices}\PYG{p}{(}\PYG{l+s+s1}{\PYGZsq{}}\PYG{l+s+s1}{GPU}\PYG{l+s+s1}{\PYGZsq{}}\PYG{p}{)}\PYG{p}{:}
19+
\PYG{n+nb}{print}\PYG{p}{(}\PYG{l+s+s2}{\PYGZdq{}}\PYG{l+s+s2}{No GPU was detected. LSTMs and CNNs can be very slow without a GPU.}\PYG{l+s+s2}{\PYGZdq{}}\PYG{p}{)}
20+
\PYG{k}{if} \PYG{n}{IS\PYGZus{}COLAB}\PYG{p}{:}
21+
\PYG{n+nb}{print}\PYG{p}{(}\PYG{l+s+s2}{\PYGZdq{}}\PYG{l+s+s2}{Go to Runtime \PYGZgt{} Change runtime and select a GPU hardware accelerator.}\PYG{l+s+s2}{\PYGZdq{}}\PYG{p}{)}
22+
\PYG{k}{if} \PYG{n}{IS\PYGZus{}KAGGLE}\PYG{p}{:}
23+
\PYG{n+nb}{print}\PYG{p}{(}\PYG{l+s+s2}{\PYGZdq{}}\PYG{l+s+s2}{Go to Settings \PYGZgt{} Accelerator and select GPU.}\PYG{l+s+s2}{\PYGZdq{}}\PYG{p}{)}
24+
25+
\PYG{c+c1}{\PYGZsh{} Common imports}
26+
\PYG{k+kn}{import}\PYG{+w}{ }\PYG{n+nn}{numpy}\PYG{+w}{ }\PYG{k}{as}\PYG{+w}{ }\PYG{n+nn}{np}
27+
\PYG{k+kn}{import}\PYG{+w}{ }\PYG{n+nn}{os}
28+
29+
\PYG{c+c1}{\PYGZsh{} to make this notebook\PYGZsq{}s output stable across runs}
30+
\PYG{n}{np}\PYG{o}{.}\PYG{n}{random}\PYG{o}{.}\PYG{n}{seed}\PYG{p}{(}\PYG{l+m+mi}{42}\PYG{p}{)}
31+
\PYG{n}{tf}\PYG{o}{.}\PYG{n}{random}\PYG{o}{.}\PYG{n}{set\PYGZus{}seed}\PYG{p}{(}\PYG{l+m+mi}{42}\PYG{p}{)}
32+
33+
\PYG{c+c1}{\PYGZsh{} To plot pretty figures}
34+
\PYG{k+kn}{import}\PYG{+w}{ }\PYG{n+nn}{matplotlib}\PYG{+w}{ }\PYG{k}{as}\PYG{+w}{ }\PYG{n+nn}{mpl}
35+
\PYG{k+kn}{import}\PYG{+w}{ }\PYG{n+nn}{matplotlib}\PYG{n+nn}{.}\PYG{n+nn}{pyplot}\PYG{+w}{ }\PYG{k}{as}\PYG{+w}{ }\PYG{n+nn}{plt}
36+
\PYG{n}{mpl}\PYG{o}{.}\PYG{n}{rc}\PYG{p}{(}\PYG{l+s+s1}{\PYGZsq{}}\PYG{l+s+s1}{axes}\PYG{l+s+s1}{\PYGZsq{}}\PYG{p}{,} \PYG{n}{labelsize}\PYG{o}{=}\PYG{l+m+mi}{14}\PYG{p}{)}
37+
\PYG{n}{mpl}\PYG{o}{.}\PYG{n}{rc}\PYG{p}{(}\PYG{l+s+s1}{\PYGZsq{}}\PYG{l+s+s1}{xtick}\PYG{l+s+s1}{\PYGZsq{}}\PYG{p}{,} \PYG{n}{labelsize}\PYG{o}{=}\PYG{l+m+mi}{12}\PYG{p}{)}
38+
\PYG{n}{mpl}\PYG{o}{.}\PYG{n}{rc}\PYG{p}{(}\PYG{l+s+s1}{\PYGZsq{}}\PYG{l+s+s1}{ytick}\PYG{l+s+s1}{\PYGZsq{}}\PYG{p}{,} \PYG{n}{labelsize}\PYG{o}{=}\PYG{l+m+mi}{12}\PYG{p}{)}
39+
40+
\PYG{c+c1}{\PYGZsh{} Where to save the figures}
41+
\PYG{n}{PROJECT\PYGZus{}ROOT\PYGZus{}DIR} \PYG{o}{=} \PYG{l+s+s2}{\PYGZdq{}}\PYG{l+s+s2}{.}\PYG{l+s+s2}{\PYGZdq{}}
42+
\PYG{n}{CHAPTER\PYGZus{}ID} \PYG{o}{=} \PYG{l+s+s2}{\PYGZdq{}}\PYG{l+s+s2}{autoencoders}\PYG{l+s+s2}{\PYGZdq{}}
43+
\PYG{n}{IMAGES\PYGZus{}PATH} \PYG{o}{=} \PYG{n}{os}\PYG{o}{.}\PYG{n}{path}\PYG{o}{.}\PYG{n}{join}\PYG{p}{(}\PYG{n}{PROJECT\PYGZus{}ROOT\PYGZus{}DIR}\PYG{p}{,} \PYG{l+s+s2}{\PYGZdq{}}\PYG{l+s+s2}{images}\PYG{l+s+s2}{\PYGZdq{}}\PYG{p}{,} \PYG{n}{CHAPTER\PYGZus{}ID}\PYG{p}{)}
44+
\PYG{n}{os}\PYG{o}{.}\PYG{n}{makedirs}\PYG{p}{(}\PYG{n}{IMAGES\PYGZus{}PATH}\PYG{p}{,} \PYG{n}{exist\PYGZus{}ok}\PYG{o}{=}\PYG{k+kc}{True}\PYG{p}{)}
45+
46+
\PYG{k}{def}\PYG{+w}{ }\PYG{n+nf}{save\PYGZus{}fig}\PYG{p}{(}\PYG{n}{fig\PYGZus{}id}\PYG{p}{,} \PYG{n}{tight\PYGZus{}layout}\PYG{o}{=}\PYG{k+kc}{True}\PYG{p}{,} \PYG{n}{fig\PYGZus{}extension}\PYG{o}{=}\PYG{l+s+s2}{\PYGZdq{}}\PYG{l+s+s2}{png}\PYG{l+s+s2}{\PYGZdq{}}\PYG{p}{,} \PYG{n}{resolution}\PYG{o}{=}\PYG{l+m+mi}{300}\PYG{p}{)}\PYG{p}{:}
47+
\PYG{n}{path} \PYG{o}{=} \PYG{n}{os}\PYG{o}{.}\PYG{n}{path}\PYG{o}{.}\PYG{n}{join}\PYG{p}{(}\PYG{n}{IMAGES\PYGZus{}PATH}\PYG{p}{,} \PYG{n}{fig\PYGZus{}id} \PYG{o}{+} \PYG{l+s+s2}{\PYGZdq{}}\PYG{l+s+s2}{.}\PYG{l+s+s2}{\PYGZdq{}} \PYG{o}{+} \PYG{n}{fig\PYGZus{}extension}\PYG{p}{)}
48+
\PYG{n+nb}{print}\PYG{p}{(}\PYG{l+s+s2}{\PYGZdq{}}\PYG{l+s+s2}{Saving figure}\PYG{l+s+s2}{\PYGZdq{}}\PYG{p}{,} \PYG{n}{fig\PYGZus{}id}\PYG{p}{)}
49+
\PYG{k}{if} \PYG{n}{tight\PYGZus{}layout}\PYG{p}{:}
50+
\PYG{n}{plt}\PYG{o}{.}\PYG{n}{tight\PYGZus{}layout}\PYG{p}{(}\PYG{p}{)}
51+
\PYG{n}{plt}\PYG{o}{.}\PYG{n}{savefig}\PYG{p}{(}\PYG{n}{path}\PYG{p}{,} \PYG{n+nb}{format}\PYG{o}{=}\PYG{n}{fig\PYGZus{}extension}\PYG{p}{,} \PYG{n}{dpi}\PYG{o}{=}\PYG{n}{resolution}\PYG{p}{)}
52+
53+
\PYG{k}{def}\PYG{+w}{ }\PYG{n+nf}{plot\PYGZus{}image}\PYG{p}{(}\PYG{n}{image}\PYG{p}{)}\PYG{p}{:}
54+
\PYG{n}{plt}\PYG{o}{.}\PYG{n}{imshow}\PYG{p}{(}\PYG{n}{image}\PYG{p}{,} \PYG{n}{cmap}\PYG{o}{=}\PYG{l+s+s2}{\PYGZdq{}}\PYG{l+s+s2}{binary}\PYG{l+s+s2}{\PYGZdq{}}\PYG{p}{)}
55+
\PYG{n}{plt}\PYG{o}{.}\PYG{n}{axis}\PYG{p}{(}\PYG{l+s+s2}{\PYGZdq{}}\PYG{l+s+s2}{off}\PYG{l+s+s2}{\PYGZdq{}}\PYG{p}{)}
56+
57+
58+
\PYG{n}{np}\PYG{o}{.}\PYG{n}{random}\PYG{o}{.}\PYG{n}{seed}\PYG{p}{(}\PYG{l+m+mi}{4}\PYG{p}{)}
59+
60+
\PYG{k}{def}\PYG{+w}{ }\PYG{n+nf}{generate\PYGZus{}3d\PYGZus{}data}\PYG{p}{(}\PYG{n}{m}\PYG{p}{,} \PYG{n}{w1}\PYG{o}{=}\PYG{l+m+mf}{0.1}\PYG{p}{,} \PYG{n}{w2}\PYG{o}{=}\PYG{l+m+mf}{0.3}\PYG{p}{,} \PYG{n}{noise}\PYG{o}{=}\PYG{l+m+mf}{0.1}\PYG{p}{)}\PYG{p}{:}
61+
\PYG{n}{angles} \PYG{o}{=} \PYG{n}{np}\PYG{o}{.}\PYG{n}{random}\PYG{o}{.}\PYG{n}{rand}\PYG{p}{(}\PYG{n}{m}\PYG{p}{)} \PYG{o}{*} \PYG{l+m+mi}{3} \PYG{o}{*} \PYG{n}{np}\PYG{o}{.}\PYG{n}{pi} \PYG{o}{/} \PYG{l+m+mi}{2} \PYG{o}{\PYGZhy{}} \PYG{l+m+mf}{0.5}
62+
\PYG{n}{data} \PYG{o}{=} \PYG{n}{np}\PYG{o}{.}\PYG{n}{empty}\PYG{p}{(}\PYG{p}{(}\PYG{n}{m}\PYG{p}{,} \PYG{l+m+mi}{3}\PYG{p}{)}\PYG{p}{)}
63+
\PYG{n}{data}\PYG{p}{[}\PYG{p}{:}\PYG{p}{,} \PYG{l+m+mi}{0}\PYG{p}{]} \PYG{o}{=} \PYG{n}{np}\PYG{o}{.}\PYG{n}{cos}\PYG{p}{(}\PYG{n}{angles}\PYG{p}{)} \PYG{o}{+} \PYG{n}{np}\PYG{o}{.}\PYG{n}{sin}\PYG{p}{(}\PYG{n}{angles}\PYG{p}{)}\PYG{o}{/}\PYG{l+m+mi}{2} \PYG{o}{+} \PYG{n}{noise} \PYG{o}{*} \PYG{n}{np}\PYG{o}{.}\PYG{n}{random}\PYG{o}{.}\PYG{n}{randn}\PYG{p}{(}\PYG{n}{m}\PYG{p}{)} \PYG{o}{/} \PYG{l+m+mi}{2}
64+
\PYG{n}{data}\PYG{p}{[}\PYG{p}{:}\PYG{p}{,} \PYG{l+m+mi}{1}\PYG{p}{]} \PYG{o}{=} \PYG{n}{np}\PYG{o}{.}\PYG{n}{sin}\PYG{p}{(}\PYG{n}{angles}\PYG{p}{)} \PYG{o}{*} \PYG{l+m+mf}{0.7} \PYG{o}{+} \PYG{n}{noise} \PYG{o}{*} \PYG{n}{np}\PYG{o}{.}\PYG{n}{random}\PYG{o}{.}\PYG{n}{randn}\PYG{p}{(}\PYG{n}{m}\PYG{p}{)} \PYG{o}{/} \PYG{l+m+mi}{2}
65+
\PYG{n}{data}\PYG{p}{[}\PYG{p}{:}\PYG{p}{,} \PYG{l+m+mi}{2}\PYG{p}{]} \PYG{o}{=} \PYG{n}{data}\PYG{p}{[}\PYG{p}{:}\PYG{p}{,} \PYG{l+m+mi}{0}\PYG{p}{]} \PYG{o}{*} \PYG{n}{w1} \PYG{o}{+} \PYG{n}{data}\PYG{p}{[}\PYG{p}{:}\PYG{p}{,} \PYG{l+m+mi}{1}\PYG{p}{]} \PYG{o}{*} \PYG{n}{w2} \PYG{o}{+} \PYG{n}{noise} \PYG{o}{*} \PYG{n}{np}\PYG{o}{.}\PYG{n}{random}\PYG{o}{.}\PYG{n}{randn}\PYG{p}{(}\PYG{n}{m}\PYG{p}{)}
66+
\PYG{k}{return} \PYG{n}{data}
67+
68+
\PYG{n}{X\PYGZus{}train} \PYG{o}{=} \PYG{n}{generate\PYGZus{}3d\PYGZus{}data}\PYG{p}{(}\PYG{l+m+mi}{60}\PYG{p}{)}
69+
\PYG{n}{X\PYGZus{}train} \PYG{o}{=} \PYG{n}{X\PYGZus{}train} \PYG{o}{\PYGZhy{}} \PYG{n}{X\PYGZus{}train}\PYG{o}{.}\PYG{n}{mean}\PYG{p}{(}\PYG{n}{axis}\PYG{o}{=}\PYG{l+m+mi}{0}\PYG{p}{,} \PYG{n}{keepdims}\PYG{o}{=}\PYG{l+m+mi}{0}\PYG{p}{)}
70+
71+
72+
\PYG{n}{np}\PYG{o}{.}\PYG{n}{random}\PYG{o}{.}\PYG{n}{seed}\PYG{p}{(}\PYG{l+m+mi}{42}\PYG{p}{)}
73+
\PYG{n}{tf}\PYG{o}{.}\PYG{n}{random}\PYG{o}{.}\PYG{n}{set\PYGZus{}seed}\PYG{p}{(}\PYG{l+m+mi}{42}\PYG{p}{)}
74+
75+
\PYG{n}{encoder} \PYG{o}{=} \PYG{n}{keras}\PYG{o}{.}\PYG{n}{models}\PYG{o}{.}\PYG{n}{Sequential}\PYG{p}{(}\PYG{p}{[}\PYG{n}{keras}\PYG{o}{.}\PYG{n}{layers}\PYG{o}{.}\PYG{n}{Dense}\PYG{p}{(}\PYG{l+m+mi}{2}\PYG{p}{,} \PYG{n}{input\PYGZus{}shape}\PYG{o}{=}\PYG{p}{[}\PYG{l+m+mi}{3}\PYG{p}{]}\PYG{p}{)}\PYG{p}{]}\PYG{p}{)}
76+
\PYG{n}{decoder} \PYG{o}{=} \PYG{n}{keras}\PYG{o}{.}\PYG{n}{models}\PYG{o}{.}\PYG{n}{Sequential}\PYG{p}{(}\PYG{p}{[}\PYG{n}{keras}\PYG{o}{.}\PYG{n}{layers}\PYG{o}{.}\PYG{n}{Dense}\PYG{p}{(}\PYG{l+m+mi}{3}\PYG{p}{,} \PYG{n}{input\PYGZus{}shape}\PYG{o}{=}\PYG{p}{[}\PYG{l+m+mi}{2}\PYG{p}{]}\PYG{p}{)}\PYG{p}{]}\PYG{p}{)}
77+
\PYG{n}{autoencoder} \PYG{o}{=} \PYG{n}{keras}\PYG{o}{.}\PYG{n}{models}\PYG{o}{.}\PYG{n}{Sequential}\PYG{p}{(}\PYG{p}{[}\PYG{n}{encoder}\PYG{p}{,} \PYG{n}{decoder}\PYG{p}{]}\PYG{p}{)}
78+
79+
\PYG{n}{autoencoder}\PYG{o}{.}\PYG{n}{compile}\PYG{p}{(}\PYG{n}{loss}\PYG{o}{=}\PYG{l+s+s2}{\PYGZdq{}}\PYG{l+s+s2}{mse}\PYG{l+s+s2}{\PYGZdq{}}\PYG{p}{,} \PYG{n}{optimizer}\PYG{o}{=}\PYG{n}{keras}\PYG{o}{.}\PYG{n}{optimizers}\PYG{o}{.}\PYG{n}{SGD}\PYG{p}{(}\PYG{n}{learning\PYGZus{}rate}\PYG{o}{=}\PYG{l+m+mf}{1.5}\PYG{p}{)}\PYG{p}{)}
80+
81+
\PYG{n}{codings} \PYG{o}{=} \PYG{n}{encoder}\PYG{o}{.}\PYG{n}{predict}\PYG{p}{(}\PYG{n}{X\PYGZus{}train}\PYG{p}{)}
82+
\PYG{n}{fig} \PYG{o}{=} \PYG{n}{plt}\PYG{o}{.}\PYG{n}{figure}\PYG{p}{(}\PYG{n}{figsize}\PYG{o}{=}\PYG{p}{(}\PYG{l+m+mi}{4}\PYG{p}{,}\PYG{l+m+mi}{3}\PYG{p}{)}\PYG{p}{)}
83+
\PYG{n}{plt}\PYG{o}{.}\PYG{n}{plot}\PYG{p}{(}\PYG{n}{codings}\PYG{p}{[}\PYG{p}{:}\PYG{p}{,}\PYG{l+m+mi}{0}\PYG{p}{]}\PYG{p}{,} \PYG{n}{codings}\PYG{p}{[}\PYG{p}{:}\PYG{p}{,} \PYG{l+m+mi}{1}\PYG{p}{]}\PYG{p}{,} \PYG{l+s+s2}{\PYGZdq{}}\PYG{l+s+s2}{b.}\PYG{l+s+s2}{\PYGZdq{}}\PYG{p}{)}
84+
\PYG{n}{plt}\PYG{o}{.}\PYG{n}{xlabel}\PYG{p}{(}\PYG{l+s+s2}{\PYGZdq{}}\PYG{l+s+s2}{\PYGZdl{}z\PYGZus{}1\PYGZdl{}}\PYG{l+s+s2}{\PYGZdq{}}\PYG{p}{,} \PYG{n}{fontsize}\PYG{o}{=}\PYG{l+m+mi}{18}\PYG{p}{)}
85+
\PYG{n}{plt}\PYG{o}{.}\PYG{n}{ylabel}\PYG{p}{(}\PYG{l+s+s2}{\PYGZdq{}}\PYG{l+s+s2}{\PYGZdl{}z\PYGZus{}2\PYGZdl{}}\PYG{l+s+s2}{\PYGZdq{}}\PYG{p}{,} \PYG{n}{fontsize}\PYG{o}{=}\PYG{l+m+mi}{18}\PYG{p}{,} \PYG{n}{rotation}\PYG{o}{=}\PYG{l+m+mi}{0}\PYG{p}{)}
86+
\PYG{n}{plt}\PYG{o}{.}\PYG{n}{grid}\PYG{p}{(}\PYG{k+kc}{True}\PYG{p}{)}
87+
\PYG{n}{save\PYGZus{}fig}\PYG{p}{(}\PYG{l+s+s2}{\PYGZdq{}}\PYG{l+s+s2}{linear\PYGZus{}autoencoder\PYGZus{}pca\PYGZus{}plot}\PYG{l+s+s2}{\PYGZdq{}}\PYG{p}{)}
88+
\PYG{n}{plt}\PYG{o}{.}\PYG{n}{show}\PYG{p}{(}\PYG{p}{)}
89+
90+
\end{MintedVerbatim}

0 commit comments

Comments
 (0)