#!/usr/bin/env python3 """Generate the neural-network illustrations for neural-networks.qmd. The figures re-draw the manim scenes from 3blue1brown's neural network series (https://github.com/3b1b/videos, _2017/nn/part{1,2,3}.py). Geometry and colours follow `NetworkMobject.CONFIG` and the manim palette so the slides look like the videos they accompany. python src/nn-figures.py Writes img/nn-*.svg. Animations are plain CSS keyframes embedded in each file, so the SVGs animate on their own when included with `![](img/...)`. """ import math import random import re from pathlib import Path OUT = Path(__file__).resolve().parent.parent / "img" # The lecture palette, from `colors_lecture` in src/config.qmd. C_BLUE = "#336699" # WU blue C_RED = "#e52320" C_PURPLE = "#9C6B91" C_GREY = "#d3d3d3" # The manim palette (manimlib/constants.py) is built for a black canvas. These # slides are light ($body-bg: #f6f6f6) so each manim colour is mapped to the # lecture colour that plays the same *role* in the videos. Tints and shades are # blends of the four above, so nothing outside the palette is introduced. # # manim here role # BLUE #58C4DD C_BLUE neuron outline + fill, positive weight # GREEN #83C167 C_BLUE activation (manim's green channel) # YELLOW #FFFF00 C_PURPLE highlight / annotation # RED #FC6255 C_RED negative weight, error, slope # MAROON_B #C55F73 C_PURPLE bias # TEAL #5CD0B3 C_BLUE shade second path # GREY_B #BBBBBB black secondary text # WHITE #FFFFFF black primary text INK = "#000000" BLUE = C_BLUE BLUE_D = "#7094B8" # C_BLUE + 30% white, contour lines BLUE_E = "#214263" # C_BLUE + 35% black, second descent path GREEN = C_BLUE # activation fill; opacity carries the value YELLOW = C_PURPLE YELLOW_D = "#75506D" # C_PURPLE + 25% black RED = C_RED RED_B = "#E94441" # C_RED + 15% white MAROON_B = C_PURPLE TEAL = BLUE_E # All *text* is black; greys are only for structure. GREY_B/GREY keep their # names so the per-figure calls still read like the manim scenes. GREY_B = INK GREY = INK AXIS = "#747474" # C_GREY + 45% black, axis lines and ticks GRID = C_GREY # pixel-grid hairlines EDGE = "#BABABA" # C_GREY + 12% black, the 672 connection lines W, H = 960, 540 # NetworkMobject.CONFIG, scaled to pixels NEURON_R = 9 NEURON_STROKE = 2 EDGE_STROKE = 0.6 # style.scss uses "Century Gothic", "Questrial", "Gothic A1". Questrial and # Gothic A1 are Google webfonts, and an SVG referenced by cannot load # external fonts -- so those two only apply if the viewer has them installed. # Century Gothic is a system font on Windows; Futura is the closest thing # macOS ships. Anything else falls back to a generic sans. FONT_SANS = "'Century Gothic','Questrial','Gothic A1',Futura,sans-serif" # Century Gothic has no glyphs for the math we need (partial, nabla, sigma), # which would silently fall back per glyph. Formulas keep a math font, which # also matches how MathJax renders the formulas on the slides themselves. FONT_MATH = "'Cambria Math','Latin Modern Math',Georgia,serif" # -------------------------------------------------------------------------- # svg helpers # -------------------------------------------------------------------------- def svg(body, w=W, h=H, style="", defs=""): return ( f'\n' f"\n{defs}\n" f"{body}\n\n" ) def write(name, content): path = OUT / name path.write_text(content, encoding="utf-8") print(f"{path.relative_to(OUT.parent)} {len(content):>8,} bytes") TOKEN = re.compile(r"(\^\{[^}]*\}|_\{[^}]*\}|\^\S|_\S)") def _spans(s): """Turn 'a^{(L)}_j' into tspans. Superscripts/subscripts only.""" out = [] for part in TOKEN.split(s): if not part: continue if part[0] in "^_": inner = part[2:-1] if part[1] == "{" else part[1:] dy = "-0.42em" if part[0] == "^" else "0.28em" back = "0.42em" if part[0] == "^" else "-0.28em" out.append( f'{inner}' f'' ) else: out.append(part) return "".join(out) # The figures are scaled down on the slides, so everything in them is set a # notch larger than the raw numbers below suggest. FONT_SCALE = 1.18 def text(x, y, s, size=16, fill=INK, anchor="middle", weight=None, math=True, serif=False): """Prose and labels, in the deck's sans stack. `^{}` / `_{}` become tspans; HTML entities pass through. Pass `serif=True` (or use `mtext`) for formulas, which need a math font. """ w = f' font-weight="{weight}"' if weight else "" size = round(size * FONT_SCALE, 1) return ( f'' f"{_spans(s) if math else s}" ) def mtext(x, y, s, size=16, fill=INK, anchor="middle", weight=None, math=True): """A formula or a single mathematical symbol.""" return text(x, y, s, size, fill, anchor, weight, math, serif=True) MINUS = "−" NABLA = "∇" SIGMA = "σ" ARROW = "→" TIMES = "×" def line(x1, y1, x2, y2, color=AXIS, width=1.5, extra=""): return (f'') def arrow(x1, y1, x2, y2, color=AXIS, width=2.0, head=11): """A line plus a filled head.""" d = math.hypot(x2 - x1, y2 - y1) or 1 ux, uy = (x2 - x1) / d, (y2 - y1) / d bx, by = x2 - ux * head, y2 - uy * head return ( line(x1, y1, bx, by, color, width) + f'' ) def polyline(pts, color, width=2.2, dash=None, cls=None): attrs = f'fill="none" stroke="{color}" stroke-width="{width}"' if dash: attrs += f' stroke-dasharray="{dash}"' if cls: attrs += f' class="{cls}"' return ('') def _hex(c): return tuple(int(c[i:i + 2], 16) for i in (1, 3, 5)) def mix(c0, c1, t): a, b = _hex(c0), _hex(c1) return "rgb(%d,%d,%d)" % tuple(round(a[i] + (b[i] - a[i]) * t) for i in range(3)) # -------------------------------------------------------------------------- # plot frame # -------------------------------------------------------------------------- class Plot: def __init__(self, x0, y0, w, h, xr, yr, clip_id): self.x0, self.y0, self.w, self.h = x0, y0, w, h self.xr, self.yr = xr, yr self.clip_id = clip_id def px(self, x, y): (xa, xb), (ya, yb) = self.xr, self.yr return (self.x0 + (x - xa) / (xb - xa) * self.w, self.y0 + self.h - (y - ya) / (yb - ya) * self.h) def clip(self): return (f'') def origin(self): """Where to draw the axes, clamped to the box. When 0 is outside the range (a cost that never reaches 0, say) the axis sits on the edge instead of floating outside and colliding with captions.""" ox, oy = self.px(0, 0) return (min(max(ox, self.x0), self.x0 + self.w), min(max(oy, self.y0), self.y0 + self.h)) def axes(self, xticks=(), yticks=(), color=AXIS): out = [] ox, oy = self.origin() out.append(line(self.x0, oy, self.x0 + self.w, oy, color, 1.2)) out.append(line(ox, self.y0, ox, self.y0 + self.h, color, 1.2)) for t in xticks: tx, _ = self.px(t, 0) out.append(line(tx, oy - 4, tx, oy + 4, color, 1.2)) out.append(text(tx, oy + 20, str(t), 12, GREY)) for t in yticks: _, ty = self.px(0, t) out.append(line(ox - 4, ty, ox + 4, ty, color, 1.2)) out.append(text(ox - 12, ty + 5, str(t), 12, GREY, "end")) return "".join(out) def curve(self, f, xa, xb, color, width=3.2, n=320, cls=None): pts = [self.px(xa + (xb - xa) * i / n, f(xa + (xb - xa) * i / n)) for i in range(n + 1)] return (f'' + polyline(pts, color, width, cls=cls) + "") # -------------------------------------------------------------------------- # fig 1: the handwritten 3 as 784 numbers (part1.ExampleThrees) # -------------------------------------------------------------------------- def arc_points(cx, cy, rx, ry, a0, a1, n=90): return [(cx + rx * math.cos(a0 + (a1 - a0) * i / (n - 1)), cy + ry * math.sin(a0 + (a1 - a0) * i / (n - 1))) for i in range(n)] def seg_points(x0, y0, x1, y1, n=45): return [(x0 + (x1 - x0) * i / (n - 1), y0 + (y1 - y0) * i / (n - 1)) for i in range(n)] def stroke_bitmap(point_sets, seed=0, sigma2=2.6): """Rasterise pen strokes into a 28x28 greyscale grid, MNIST style. Intensity is the distance from each pixel centre to the nearest point on any stroke, with a little jitter so it reads as handwriting. This is the one renderer behind every digit-like figure in the deck. """ rng = random.Random(seed) pts = [(x + rng.gauss(0, 0.12), y + rng.gauss(0, 0.12)) for ps in point_sets for x, y in ps] grid = [[0.0] * 28 for _ in range(28)] for r in range(28): for c in range(28): d2 = min((c - px) ** 2 + (r - py) ** 2 for px, py in pts) grid[r][c] = math.exp(-d2 / sigma2) return grid def three_bitmap(seed=3): """A 28x28 greyscale '3': two stacked arcs.""" return stroke_bitmap([arc_points(14.0, 10.0, 5.2, 4.8, -1.9, 1.5), arc_points(14.0, 18.0, 5.2, 4.8, -1.6, 1.9)], seed=seed) def pixel_grid(grid, x0, y0, cell, paint=None, stroke=None): """Draw a 2d array of numbers as squares. `paint(v) -> (fill, opacity)`. Opacity rather than a blended fill, so a zero cell is fully transparent and the slide background shows through. The default paints ink on paper: 0 blank, 1 solid. """ if paint is None: def paint(v): return INK, max(0.0, min(1.0, v)) out = [] for r, row in enumerate(grid): for c, v in enumerate(row): fill, op = paint(v) # A blank cell is invisible anyway, so drop it -- most of a digit # is background, and emitting all 784 rects per tile made the # multi-tile figures roughly a megabyte. Grids that draw hairlines # keep every cell, or the lattice would come out full of holes. if stroke is None and op < 0.03: continue out.append( f'" ) return "".join(out) def fig_three_pixels(): grid = three_bitmap() cell, x0 = 13, 120 side = 28 * cell y0 = (H - side) / 2 + 10 body = [ pixel_grid(grid, x0, y0, cell, stroke=GRID), f'', text(x0 + side / 2, y0 - 20, f"28 {TIMES} 28 = 784 pixels", 20, GREY_B), ] r, c = 13, 16 px_, py_ = x0 + c * cell + cell / 2, y0 + r * cell + cell / 2 tx = x0 + side + 130 body += [ line(px_, py_, tx - 40, py_, YELLOW, 1.5), f'', text(tx, py_ + 8, f"{grid[r][c]:.2f}", 30, YELLOW_D, "start"), # MNIST stores "how much stroke is in this pixel", conventionally shown # white-on-black; on a light slide the same number reads as ink text(tx, py_ + 40, "0.00 = blank  ·  1.00 = full stroke", 14, GREY, "start"), text(tx, py_ - 62, "784 numbers in,", 18, INK, "start"), text(tx, py_ - 34, "one digit out.", 18, INK, "start"), ] write("nn-three-pixels.svg", svg("\n".join(body))) # -------------------------------------------------------------------------- # fig 2/3: the 784-16-16-10 network (part1.MNistNetworkMobject) # -------------------------------------------------------------------------- # shifted right of centre to leave room for the "784 pixels" brace label LAYER_X = [180, 408, 628, 808] # fig_forward shifts everything right to make room for the input image # the output layer sits well left of the edge to leave room for the # "highest activation" callout beside it LAYER_X_FWD = [250, 410, 570, 715] SHOWN = [16, 16, 16, 10] def layer_positions(n, cy=H / 2, buff=6): step = 2 * NEURON_R + buff return [cy - (n - 1) * step / 2 + i * step for i in range(n)] def network_geometry(cy=H / 2 + 6, xs=None): xs = xs or LAYER_X return [[(xs[li], y) for y in layer_positions(n, cy)] for li, n in enumerate(SHOWN)] def input_activations(grid, n=16): """Collapse 784 pixels onto the 16 neurons actually drawn. This is `NetworkMobject.activate_layer`'s `arr_to_num` from part1.py: the cube root of the fraction of pixels in each chunk that are lit. """ flat = [v for row in grid for v in row] size = len(flat) // n out = [] for i in range(n): chunk = flat[i * size:(i + 1) * size] if i < n - 1 else flat[i * size:] out.append((sum(1 for v in chunk if v > 0.1) / len(chunk)) ** (1 / 3)) return out def edges(layers, cls_of=None, color=EDGE, width=EDGE_STROKE, opacity=0.75): """Lines between consecutive layers. `cls_of(layer, i, j) -> str` names a CSS class per connection, which is how the forward-propagation figure gives each edge a flash matched to how active the two neurons it joins are. """ out = [] for li in range(len(layers) - 1): for i, a in enumerate(layers[li]): for j, b in enumerate(layers[li + 1]): dx, dy = b[0] - a[0], b[1] - a[1] d = math.hypot(dx, dy) ux, uy = dx / d, dy / d attrs = (f'stroke="{color}" stroke-width="{width}" ' f'stroke-opacity="{opacity}"') if cls_of: attrs += f' class="{cls_of(li, i, j)}"' out.append( f'' ) return "".join(out) def neuron(x, y, op=0.0, r=NEURON_R, extra=""): return (f'') def brace(x, y0, y1, label, size=18): """A left-facing curly brace with a label (manim's Brace).""" mid = (y0 + y1) / 2 d = -14 path = ( f'' ) return path + text(x + d - 12, mid + 6, label, size, GREY_B, "end") def output_labels(layers): return "".join(text(x + 26, y + 6, str(n), 15, INK) for n, (x, y) in enumerate(layers[-1])) def fig_network(): layers = network_geometry() ys = [y for _, y in layers[0]] body = [ edges(layers), "".join(neuron(x, y) for layer in layers for x, y in layer), output_labels(layers), brace(LAYER_X[0] - NEURON_R - 6, ys[0] - NEURON_R, ys[-1] + NEURON_R, "784 pixels"), text(LAYER_X[0], H - 44, "input", 15, GREY), text((LAYER_X[1] + LAYER_X[2]) / 2, H - 44, "2 hidden layers", 15, GREY), text(LAYER_X[3], H - 44, "output", 15, GREY), text(W / 2, 36, f"784 {ARROW} 16 {ARROW} 16 {ARROW} 10", 22, GREY_B), text(W / 2, H - 16, "12,544 + 256 + 160 weights  +  42 biases  =  " "13,002 knobs to tune", 15, YELLOW_D), ] write("nn-network.svg", svg("\n".join(body))) def fig_forward(): """Activations sweeping left to right (get_edge_propogation_animations).""" # A shorter canvas than the other figures: the drawing only spans ~420px # vertically, and trimming the slack lets the slide render it larger for # the same max-height. fig_h = 500 cy = fig_h / 2 layers = network_geometry(cy=cy, xs=LAYER_X_FWD) grid = three_bitmap() rng = random.Random(11) acts = [input_activations(grid), [rng.random() ** 2 for _ in range(16)], [rng.random() ** 2 for _ in range(16)], [0.05] * 10] acts[3][3] = 0.95 # the network says "3" # One strictly left-to-right chain, one element per beat. The fan from the # image is simply the first bundle of connections, so connections and # layers alternate: # # image fan -> layer 0 -> edges 0 -> layer 1 -> edges 1 -> layer 2 -> # edges 2 -> layer 3 -> ring # # Every element gets its own keyframes with the timing baked in as # percentages and *no* animation-delay. Two reasons: a delayed animation # sits in its base state until it starts (which made the ring flicker), # and percentages let every flash last the same wall-clock time. Under the # old scheme the flash was a fixed share of the cycle, so lengthening the # cycle stretched each flash until neighbouring ones overlapped -- which is # what made the sweep look muddled and non-uniform. # Everything below is wall-clock seconds; BEAT must stay longer than a # flash or consecutive bundles would light at once. cycle = 16.0 FIRST, BEAT = 0.6, 1.35 # when the chain starts, and the gap per beat FLASH_RISE, FLASH_FALL = 0.35, 1.10 # a connection's flash, in seconds FILL = 0.55 # how long a layer takes to light up RING_GAP, RING_RISE = 0.35, 0.35 # the pause before the answer, and its fade HOLD, FADE = 0.90, 0.97 # everything stays lit, then clears together # bundles are the fan out of the image ("ein") plus the three between layers bundles = ["ein", "e0", "e1", "e2"] chain = ["ein", "n0", "e0", "n1", "e1", "n2", "e2", "n3"] at = {name: FIRST + i * BEAT for i, name in enumerate(chain)} ring_at = at["n3"] + FILL + RING_GAP # A connection carries a signal only to the extent that both ends are # active, so each edge flashes at one of three strengths set by the product # of the two activations it joins. Weak links stay faint, and the ones into # the winning output neuron are the ones that shout. STRENGTHS = [("l", "#C9AEC2", 0.9), ("m", C_PURPLE, 1.9), ("h", "#6B4363", 3.0)] def strength(a, b, peak): """Bucket a connection by activation product, relative to its bundle.""" frac = (a * b / peak) if peak else 0.0 return STRENGTHS[min(len(STRENGTHS) - 1, int(frac * len(STRENGTHS)))][0] peaks = [max(a * b for a in acts[i] for b in acts[i + 1]) for i in range(3)] peak_in = max(acts[0]) def pct(seconds): return 100.0 * seconds / cycle def flash_frames(name, t_key, lit, lit_w): """A connection lighting up and settling back.""" t = at[t_key] return ( f"@keyframes flash-{name} {{\n" f" 0%,{pct(t):.2f}% {{ stroke:{EDGE}; stroke-width:{EDGE_STROKE}; " f"stroke-opacity:.75; }}\n" f" {pct(t + FLASH_RISE):.2f}% {{ stroke:{lit}; " f"stroke-width:{lit_w}; stroke-opacity:1; }}\n" f" {pct(t + FLASH_FALL):.2f}%,100% {{ stroke:{EDGE}; " f"stroke-width:{EDGE_STROKE}; stroke-opacity:.75; }}\n" f"}}" ) def fill_frames(name): """A layer of neurons filling to its activation, then holding.""" t = at[name] return ( f"@keyframes lit-{name} {{\n" f" 0%,{pct(t):.2f}% {{ fill-opacity:0; }}\n" f" {pct(t + FILL):.2f}%,{HOLD * 100:.0f}% " f"{{ fill-opacity:var(--a); }}\n" f" {FADE * 100:.0f}%,100% {{ fill-opacity:0; }}\n" f"}}" ) blocks, sel = [], [] for b in bundles: for tag, lit, lit_w in STRENGTHS: blocks.append(flash_frames(f"{b}{tag}", b, lit, lit_w)) sel.append(f" .{b}{tag} {{ animation: flash-{b}{tag} " f"{cycle}s linear infinite; }}") for i in range(4): blocks.append(fill_frames(f"n{i}")) sel.append(f" .n{i} {{ animation: lit-n{i} " f"{cycle}s linear infinite; }}") selectors = "\n".join(sel) style = ( selectors + "\n" # stroke-opacity/fill-opacity rather than plain opacity: the ring's # children inherit both from the group, and unlike `opacity` these are # not composited, so the reveal shows up in a static render too. + f" .winner {{ animation: reveal {cycle}s linear infinite; }}\n" + f"@keyframes reveal {{\n" f" 0%,{pct(ring_at):.2f}% {{ stroke-opacity:0; fill-opacity:0; }}\n" f" {pct(ring_at + RING_RISE):.2f}%,{HOLD * 100:.0f}% " f"{{ stroke-opacity:1; fill-opacity:1; }}\n" f" {FADE * 100:.0f}%,100% {{ stroke-opacity:0; fill-opacity:0; }}\n" f"}}\n" + "\n".join(blocks) + "\n" ) # the image the network is being shown, on the left, feeding the input layer cell = 5 side = 28 * cell ix, iy = 28, cy - side / 2 body = [ pixel_grid(grid, ix, iy, cell), f'', text(ix + side / 2, iy + side + 26, f"28 {TIMES} 28 = 784 inputs", 15, GREY), ] # the first bundle in the chain: the image fans out to every input neuron, # each line leaving the edge of the image beside the pixels it carries fan = [] for j, (x, y) in enumerate(layers[0]): sy = iy + side * (j + 0.5) / len(layers[0]) dx, dy = x - (ix + side), y - sy d = math.hypot(dx, dy) ux, uy = dx / d, dy / d tag = strength(acts[0][j], 1.0, peak_in) fan.append( f'' ) body.append("".join(fan)) # every element's timing lives in its own keyframes, so no delays here body.append(edges(layers, cls_of=lambda li, i, j: ( f"e{li}{strength(acts[li][i], acts[li + 1][j], peaks[li])}"))) out = [] for li, layer in enumerate(layers): for ni, (x, y) in enumerate(layer): out.append(neuron(x, y, 0.0, extra=( f'class="n{li}" style="--a:{acts[li][ni]:.2f}"'))) body.append("".join(out)) body.append(output_labels(layers)) # the ring and its label share the output layer's delay, so the answer is # only called once the last layer has lit up bx, by = layers[3][3] body.append( f'' f'' + text(bx + 46, by + 6, "← highest activation", 13, YELLOW_D, "start") + "" ) write("nn-forward.svg", svg("\n".join(body), style=style, h=fig_h)) # -------------------------------------------------------------------------- # fig 4: one neuron (part1.IntroduceWeights / IncludeBias) # -------------------------------------------------------------------------- def fig_neuron(): # The formula that used to head this figure now lives on the slide, so the # canvas is short and the drawing sits right under it. fig_h = 440 cx, cy, R = 660, 250, 42 rows = [("a_1", "w_1", 90), ("a_2", "w_2", 155), ("a_3", "w_3", 220), ("⋮", None, 280), ("a_n", "w_n", 345)] body = [] for k, (lab, wl, y) in enumerate(rows): x = 190 if wl is None: body.append(mtext(x, y + 8, lab, 20, GREY_B)) continue body.append(neuron(x, y, 0.28 + 0.13 * k, r=17)) body.append(mtext(x, y + 6, lab, 16)) dx, dy = cx - x, cy - y d = math.hypot(dx, dy) ux, uy = dx / d, dy / d body.append(line(x + ux * 17, y + uy * 17, cx - ux * (R + 2), cy - uy * (R + 2), BLUE_D, 1.8)) body.append(mtext(x + dx * 0.42, y + dy * 0.42 - 9, wl, 16, BLUE)) # the bias enters the same neuron, but not from another neuron body += [ mtext(468, 374, "b", 22, MAROON_B), arrow(484, 368, cx - 26, cy + R - 4, MAROON_B, 1.8), neuron(cx, cy, 0.55, r=R), mtext(cx, cy + 9, "a", 24), text(190, 46, "previous layer", 14, GREY), text(cx, 150, "this neuron", 14, GREY), ] notes = [ (BLUE, "weights — which pattern this neuron looks for"), (MAROON_B, "bias — how high the bar is before it fires"), (YELLOW_D, f"{SIGMA} — squishes any number into (0, 1)"), ] for i, (col, s) in enumerate(notes): body.append(text(560, 350 + i * 30, s, 15, col, "start")) write("nn-neuron.svg", svg("\n".join(body), h=fig_h)) # -------------------------------------------------------------------------- # fig 5: activation functions (part1.IntroduceSigmoid / IntroduceReLU) # -------------------------------------------------------------------------- def fig_activations(): sig = lambda x: 1 / (1 + math.exp(-x)) relu = lambda x: max(0.0, x) p1 = Plot(95, 120, 340, 290, (-5, 5), (-0.45, 1.45), "c1") p2 = Plot(555, 120, 340, 290, (-5, 5), (-0.72, 2.32), "c2") body = [p1.axes([-4, -2, 2, 4], [1]), p2.axes([-4, -2, 2, 4], [1, 2])] # 3b1b splits the sigmoid into pinned-low / transition / pinned-high. His # third colour maps onto C_BLUE here, which would collide with the # transition, so both saturated tails share the red: red = no gradient. body += [p1.curve(sig, -5, -2, RED, 4), p1.curve(sig, -2, 2, BLUE, 4), p1.curve(sig, 2, 5, RED, 4)] body += [p2.curve(relu, -5, 0, RED, 4), p2.curve(relu, 0, 5, GREEN, 4)] for p, name, formula, note in ( (p1, "Sigmoid", f"{SIGMA}(x) = 1 / (1 + e^{{−x}})", "saturates — gradients vanish, slow to train"), (p2, "ReLU", "ReLU(x) = max(0, x)", "what deep networks actually use"), ): cx = p.x0 + p.w / 2 body += [text(cx, 82, name, 24, INK), text(cx, 462, note, 15, GREY)] defs = p1.clip() + p2.clip() write("nn-activations.svg", svg("\n".join(body), defs=defs)) # -------------------------------------------------------------------------- # fig 6: weights drawn as an image (part1.organize_weights_as_grid, # part2.InterpretFirstWeightMatrixRows) # -------------------------------------------------------------------------- def weight_paint(v): """Blue positive, red negative, transparent at zero (3b1b's convention, inverted for a light background: the page stands in for manim's black).""" return (BLUE if v >= 0 else RED), min(1.0, abs(v)) def fig_weight_images(): rng = random.Random(7) # the wish: a loop detector, positive on a ring, negative just outside it hope = [[math.exp(-((math.hypot(c - 13.5, r - 13.5) - 4.6) ** 2) / 2.0) - 0.8 * math.exp(-((math.hypot(c - 13.5, r - 13.5) - 8.6) ** 2) / 3.0) for c in range(28)] for r in range(28)] # the reality: mostly noise with a faint large-scale structure real = [[max(-1.0, min(1.0, rng.gauss(0, 0.42) + 0.35 * math.sin(c / 3.1) * math.cos(r / 4.3))) for c in range(28)] for r in range(28)] cell = 8.5 side = 28 * cell body = [text(W / 2, 60, "the 784 weights feeding one hidden neuron, " "laid out in the shape of the image", 17, GREY)] for i, (grid, title, sub) in enumerate(( (hope, "what we hope it learns", "a clean loop detector"), (real, "what it actually learns", "loosely structured noise"), )): x0, y0 = 155 + i * 415, 140 body += [ pixel_grid(grid, x0, y0, cell, paint=weight_paint), f'', text(x0 + side / 2, y0 - 24, title, 20, INK), text(x0 + side / 2, y0 + side + 32, sub, 15, GREY_B), ] # legend lx = W / 2 - 105 for j, (col, lab) in enumerate(((BLUE, "positive weight"), (RED, "negative weight"))): x = lx + j * 190 body += [f'', text(x + 24, H - 45, lab, 15, GREY_B, "start")] write("nn-weight-images.svg", svg("\n".join(body))) # -------------------------------------------------------------------------- # fig 7: cost of one weight (part2.SingleVariableCostFunction) # -------------------------------------------------------------------------- def fig_cost_1d(): f = lambda x: 0.16 * x ** 4 / 4 - 0.55 * x ** 2 / 2 + 0.22 * x + 1.55 df = lambda x: 0.16 * x ** 3 - 0.55 * x + 0.22 p = Plot(120, 100, 720, 320, (-3.3, 3.3), (0.55, 2.75), "cp") body = [p.axes([-3, -2, -1, 1, 2, 3], [1, 2]), p.curve(f, -3.3, 3.3, BLUE, 3.2)] # gradient-descent steps from a starting guess x = -3.0 steps = [x] for _ in range(15): x -= 0.9 * df(x) steps.append(x) keys = [] for i, sx in enumerate(steps): cxp, cyp = p.px(sx, f(sx)) keys.append(f"{100 * i / (len(steps) - 1):.2f}% " f"{{ transform: translate({cxp:.1f}px,{cyp:.1f}px); }}") style = (".ball { animation: roll 5s steps(1,end) infinite; }\n" "@keyframes roll { " + " ".join(keys) + " }") body.append(f'') # the slope at the start, clipped to the plot x0 = -3.0 m, dx = df(x0), 0.55 a = p.px(x0 - dx, f(x0) - m * dx) b = p.px(x0 + dx, f(x0) + m * dx) sx, sy = p.px(x0, f(x0)) body += [f'' + line(a[0], a[1], b[0], b[1], RED, 2.4) + "", mtext(sx + 14, sy - 30, "dC/dw", 17, RED, "start"), text(W / 2, 56, "cost as a function of one weight", 21, INK), mtext(160, 130, "C(w)", 19, BLUE, "start"), mtext(846, p.origin()[1] + 6, "w", 18, GREY, "start"), text(W / 2, H - 24, "shallow slope, small step", 15, GREY_B)] write("nn-cost-1d.svg", svg("\n".join(body), style=style, defs=p.clip())) # -------------------------------------------------------------------------- # fig 8: cost of two weights (part2.TwoVariableInputSpace, LocalVsGlobal) # -------------------------------------------------------------------------- def _contour(f, p, level, nx=90, ny=90): """Marching squares: line segments of {f = level} inside the plot box.""" (xa, xb), (ya, yb) = p.xr, p.yr xs = [xa + (xb - xa) * i / nx for i in range(nx + 1)] ys = [ya + (yb - ya) * j / ny for j in range(ny + 1)] vals = [[f(x, y) for y in ys] for x in xs] segs = [] def crossing(x1, y1, v1, x2, y2, v2): t = (level - v1) / (v2 - v1) return p.px(x1 + (x2 - x1) * t, y1 + (y2 - y1) * t) for i in range(nx): for j in range(ny): corners = [(xs[i], ys[j], vals[i][j]), (xs[i + 1], ys[j], vals[i + 1][j]), (xs[i + 1], ys[j + 1], vals[i + 1][j + 1]), (xs[i], ys[j + 1], vals[i][j + 1])] hits = [] for k in range(4): x1, y1, v1 = corners[k] x2, y2, v2 = corners[(k + 1) % 4] if (v1 - level) * (v2 - level) < 0: hits.append(crossing(x1, y1, v1, x2, y2, v2)) for k in range(0, len(hits) - 1, 2): segs.append((hits[k], hits[k + 1])) return segs def fig_cost_2d(): # a global bowl with two local minima, so gradients never vanish at the edge def C(x, y): return (0.30 * (x * x + y * y) - 1.5 * math.exp(-((x - 1.3) ** 2 + (y - 0.7) ** 2) / 0.9) - 1.1 * math.exp(-((x + 1.4) ** 2 + (y + 1.0) ** 2) / 0.8)) def grad(x, y, e=1e-4): return ((C(x + e, y) - C(x - e, y)) / (2 * e), (C(x, y + e) - C(x, y - e)) / (2 * e)) p = Plot(120, 105, 380, 380, (-3, 3), (-3, 3), "cs") body = [f''] for k, lv in enumerate([-1.4, -1.0, -0.6, -0.2, 0.3, 0.9, 1.6, 2.4]): col = mix(BLUE_E, BLUE, 1 - k / 7) for (a, b) in _contour(C, p, lv): body.append(line(a[0], a[1], b[0], b[1], col, 1.1)) for start, color in (((2.6, 2.5), YELLOW), ((-2.6, -2.5), TEAL)): x, y = start pts = [p.px(x, y)] for _ in range(60): gx, gy = grad(x, y) x, y = x - 0.16 * gx, y - 0.16 * gy pts.append(p.px(x, y)) body += [polyline(pts, color, 2.2, dash="5 4"), f'', f''] body += [text(p.x0 + p.w / 2, p.y0 - 24, "two weights, two local minima", 19, INK), mtext(p.x0 + p.w / 2, p.y0 + p.h + 30, "w_1", 17, GREY), mtext(p.x0 - 26, p.y0 + p.h / 2, "w_2", 17, GREY)] tx = 580 body.append(text(tx, 150, "In the real network:", 20, INK, "start")) for i, s in enumerate(("13,002 weights and biases, not 2 —", "so this “surface” lives in 13,002", "dimensions. Same idea, no picture.")): body.append(text(tx, 186 + i * 27, s, 16, GREY_B, "start")) for i, s in enumerate((f"{MINUS}{NABLA}C says which way to nudge", "every parameter at once, and", "which ones matter most.")): body.append(text(tx, 300 + i * 27, s, 16, YELLOW_D, "start")) for i, s in enumerate(("Different starting weights end in a", "different valley. Nothing guarantees", "you find the global minimum.")): body.append(text(tx, 414 + i * 25, s, 15, GREY, "start")) write("nn-cost-2d.svg", svg("\n".join(body))) # -------------------------------------------------------------------------- # fig 9: the gradient vector (part2.ShowFullCostFunctionGradient) # -------------------------------------------------------------------------- def fig_gradient_vector(): nums = [0.18, 0.45, -0.51, 0.40, -0.32, 0.82] # part2.show_gradient names = ["w_1", "w_2", "w_3", "w_4", "w_5", "b_{16}"] x, y0, step = 520, 158, 44 rows = 7 top, bot = y0 - 32, y0 + step * (rows - 1) + 16 body = [ mtext(300, H / 2 + 4, f"{MINUS}{NABLA}C(W) =", 30, INK, "end"), f'', f'', ] for i in range(rows): y = y0 + i * step if i == 3: body.append(mtext(x + 55, y + 8, "⋮", 24, INK)) continue j = i if i < 3 else i - 1 v = nums[j] body += [text(x + 55, y + 9, f"{v:+.2f}".replace("-", MINUS), 25, BLUE if v > 0 else RED_B), mtext(x + 180, y + 9, names[j], 17, GREY, "start")] body += [text(W / 2, 66, "one number per weight and bias — " "13,002 of them", 19, GREY_B), text(W / 2, H - 44, "sign: nudge up or down   ·   " "size: how much this parameter matters", 17, YELLOW_D)] write("nn-gradient-vector.svg", svg("\n".join(body))) # -------------------------------------------------------------------------- # fig 10: what one example wants (part3.WalkThroughTwoExample) # -------------------------------------------------------------------------- def fig_nudges(): acts = [0.20, 0.75, 0.15, 0.55, 0.10, 0.30, 0.05, 0.60, 0.25, 0.40] want = [0, 0, 1, 0, 0, 0, 0, 0, 0, 0] # this example is a 2 xs = [140 + i * 76 for i in range(10)] base = 300 body = [text(W / 2, 56, "one training example: this image is a 2", 21, INK), text(W / 2, 88, "each output has an opinion about where it should go", 15, GREY)] for i, x in enumerate(xs): err = want[i] - acts[i] body += [neuron(x, base, acts[i], r=19), text(x, base + 7, str(i), 16, INK), text(x, base + 46, f"{acts[i]:.2f}", 14, GREY_B)] # vertical nudge arrow, length proportional to the error L = 20 + abs(err) * 82 color = GREEN if err > 0 else RED y_from, y_to = base - 26, base - 26 - L if err < 0: y_from, y_to = base + 68, base + 68 + L body.append(arrow(x, y_from, x, y_to, color, 1.8 + 2.2 * abs(err), 9 + 4 * abs(err))) if err > 0: body.append(text(x, y_to - 14, f"+{err:.2f}", 14, color)) body += [text(W / 2, H - 38, "the size of each nudge is proportional to how wrong " "that output is", 16, GREY_B), text(W / 2, H - 12, "every one of the 60,000 examples casts a vote for every " "weight — backprop averages them", 16, YELLOW_D)] write("nn-nudges.svg", svg("\n".join(body))) # -------------------------------------------------------------------------- # fig 11: the computational graph (part3.break_into_computational_graph) # -------------------------------------------------------------------------- def fig_comp_graph(): body = [text(W / 2, 58, "how a single weight reaches the cost", 22, INK)] chain = [(180, 265, "w^{(L)}", BLUE, 36), # manim gives z its own colour; GREEN maps onto C_BLUE here, so # the darker blue keeps z distinguishable from w (410, 265, "z^{(L)}", BLUE_E, 36), (640, 265, "a^{(L)}", INK, 36), (860, 265, "C_0", RED, 36)] side = [(180, 130, "a^{(L−1)}", INK, 28), (180, 400, "b^{(L)}", MAROON_B, 28), (640, 400, "y", YELLOW, 28)] radius = {(x, y): r for x, y, _, _, r in chain + side} for x1, y1, x2, y2 in ((180, 265, 410, 265), (410, 265, 640, 265), (640, 265, 860, 265), (180, 130, 410, 265), (180, 400, 410, 265), (640, 400, 860, 265)): d = math.hypot(x2 - x1, y2 - y1) ux, uy = (x2 - x1) / d, (y2 - y1) / d r1, r2 = radius[(x1, y1)], radius[(x2, y2)] body.append(arrow(x1 + ux * (r1 + 5), y1 + uy * (r1 + 5), x2 - ux * (r2 + 6), y2 - uy * (r2 + 6), AXIS, 2)) for x, y, lab, col, r in chain + side: body += [f'', mtext(x, y + 9, lab, 21, col)] body += [mtext(295, 237, f"{TIMES} a^{{(L−1)}}, + b", 15, GREY), mtext(525, 237, SIGMA, 20, GREY), mtext(750, 237, "(a − y)²", 15, GREY, math=False), text(W / 2, 495, "one factor per arrow", 15, GREY)] write("nn-comp-graph.svg", svg("\n".join(body))) # -------------------------------------------------------------------------- # fig 12: mini-batches (part3.OrganizeDataIntoMiniBatches) # -------------------------------------------------------------------------- def fig_minibatch(): rng = random.Random(5) body = [text(W / 2, 58, "gradient descent vs. stochastic gradient descent", 21, INK)] for i, (title, sub, jitter, color) in enumerate(( ("all 60,000 examples per step", "a careful, expensive walk downhill", 0.0, TEAL), ("mini-batches of 100", "a drunk stumble — much faster", 24.0, YELLOW), )): cx, cy = 245 + i * 460, 290 for r in (155, 105, 58): body.append(f'') body.append(f'') x, y = cx - 150, cy - 145 pts = [(x, y)] for _ in range(28): dx, dy = cx - x, cy - y d = math.hypot(dx, dy) or 1 x += dx / d * 13 + rng.gauss(0, jitter) y += dy / d * 13 + rng.gauss(0, jitter) pts.append((x, y)) body += [polyline(pts, color, 2.2), f'', text(cx, 486, title, 18, INK), text(cx, 512, sub, 15, color)] write("nn-minibatch.svg", svg("\n".join(body))) # -------------------------------------------------------------------------- # fig 13: what we hope each layer does (part1.BreakUpMacroPatterns, # BreakUpMicroPatterns, SecondLayerIsLittleEdgeLayer) # -------------------------------------------------------------------------- # The decomposition is the one encoded in part1.py. `BreakUpMacroPatterns` # loads handwritten_{nine,eight,four} alongside the components it builds them # from, and `BreakUpMicroPatterns` then splits a loop into five little edges # and a long line into three. Those raster assets are not in the repo, so the # strokes below are drawn in the same 28-unit space as the MNIST figures. TAU = 2 * math.pi COMPONENTS = { "upper_loop": arc_points(14, 9.5, 5, 5, 0, TAU), "lower_loop": arc_points(14, 19.5, 5, 5, 0, TAU), "right_line": seg_points(19, 4, 19, 24), "upper_left_line": seg_points(9, 4, 9, 14), "horizontal_line": seg_points(7, 15, 21, 15), } # part1.show_nine / show_eight / show_four DIGIT_PARTS = { "9": ("upper_loop", "right_line"), "8": ("upper_loop", "lower_loop"), "4": ("upper_left_line", "horizontal_line", "right_line"), } # The little oriented pieces a loop or a line breaks down into. Two of them are # stubs of a long vertical stroke and three are quarter-arcs, so the figure can # show them being assembled into `right_line` and `upper_loop` respectively. _TOP = -math.pi / 2 # 12 o'clock, where a loop is easiest to read from def _thirds(cx, cy, r): """A circle cut into three arcs, each drawn where it sits in the whole.""" return [arc_points(cx, cy, r, r, _TOP + k * TAU / 3, _TOP + (k + 1) * TAU / 3) for k in range(3)] # Each piece sits exactly where it lies inside the stroke it builds, at the # same coordinates as the component in COMPONENTS -- stack the pieces of a # group and you get the component back. Column order matches the order of the # components in `fig_layers`, which is what keeps the arrows from crossing. EDGES = ( [seg_points(19, 4, 19, 11), seg_points(19, 12, 19, 24)] # right_line + _thirds(14, 9.5, 5) # upper_loop + _thirds(14, 19.5, 5) # lower_loop + [seg_points(9, 4, 9, 9), seg_points(9, 10, 9, 14)] # upper_left_line + [seg_points(7, 15, 14, 15), seg_points(14, 15, 21, 15)] # horizontal_line ) # which edges build which component, by index into EDGES EDGE_GROUPS = [(0, 1), (2, 3, 4), (5, 6, 7), (8, 9), (10, 11)] def pixel_tile(x, y, size, point_sets, seed=0, frame=True): """A tile holding a 28x28 rasterisation of `point_sets`. Same renderer as the MNIST digit figures, so an edge, a loop and a digit all look like the data the network actually sees. """ out = [] if frame: out.append(f'') out.append(pixel_grid(stroke_bitmap(point_sets, seed=seed), x, y, size / 28.0)) return "".join(out) def fig_layers(): """Each layer as a single row, composing into the row below it. Laid out in horizontal bands rather than columns: with a dozen edge detectors to show, a vertical column would squeeze the tiles down to about 24px on the slide, where a 28x28 rasterisation is unreadable. Rows put the count along the axis that has room, and keep every tile the same size. """ tile, gap = 68, 8 x0, pitch = 218, 130 row_w = len(EDGES) * (tile + gap) - gap w, h = x0 + row_w + 36, 450 ys = [56, 56 + pitch, 56 + 2 * pitch] body = [] def row(y, items, seed0=0): """Tiles in a line, centred under the widest row. Returns centre xs.""" left = x0 + (row_w - (len(items) * (tile + gap) - gap)) / 2 xs = [] for i, point_sets in enumerate(items): x = left + i * (tile + gap) body.append(pixel_tile(x, y, tile, point_sets, seed=seed0 + i)) xs.append(x + tile / 2) return xs xs1 = row(ys[0], [[p] for p in EDGES], seed0=10) # `right_line` and `upper_loop` lead, so each sits under the edges that # build it and the composition arrows never have to cross. order = ["right_line", "upper_loop", "lower_loop", "upper_left_line", "horizontal_line"] xs2 = row(ys[1], [[COMPONENTS[n]] for n in order], seed0=20) xs3 = row(ys[2], [[COMPONENTS[p] for p in parts] for parts in DIGIT_PARTS.values()], seed0=30) for i, digit in enumerate(DIGIT_PARTS): body.append(text(xs3[i], ys[2] + tile + 30, digit, 22, BLUE)) def compose(xa, ya, xb, yb): return arrow(xa, ya + tile + 5, xb, yb - 7, AXIS, 1.3, 8) # edges assemble into a stroke, strokes assemble into a digit for target, group in enumerate(EDGE_GROUPS): for i in group: body.append(compose(xs1[i], ys[0], xs2[target], ys[1])) for j in (0, 1): body.append(compose(xs2[j], ys[1], xs3[0], ys[2])) for y, head, sub in ((ys[0], "little edges", "layer 1"), (ys[1], "loops and strokes", "layer 2"), (ys[2], "digits", "layer 3")): body.append(text(x0 - 26, y + tile / 2 - 3, head, 16, INK, "end")) body.append(text(x0 - 26, y + tile / 2 + 21, sub, 13, GREY, "end")) write("nn-layers.svg", svg("\n".join(body), w=w, h=h)) # -------------------------------------------------------------------------- # fig 14: a sheet of MNIST samples (part2.MNistDescription) # -------------------------------------------------------------------------- # Schematic pen strokes for each digit, in the same 28-unit space. Two written # forms per digit, so a row of samples shows the variation the network has to # cope with rather than ten identical glyphs. DIGIT_STROKES = { "0": [[arc_points(14, 14, 5.5, 8, 0, TAU)], [arc_points(14, 14, 4.5, 8.5, 0, TAU)]], "1": [[seg_points(14, 6, 14, 23), seg_points(10.5, 9, 14, 6)], [seg_points(15, 6, 13, 23), seg_points(10, 9.5, 15, 6)]], "2": [[arc_points(14, 10, 5, 4.5, math.pi, 5.78), seg_points(18.4, 7.9, 9, 22), seg_points(9, 22, 19.5, 22)], [arc_points(14, 10.5, 4.5, 4.5, math.pi, 5.9), seg_points(18.2, 8.6, 8.5, 21.5), seg_points(8.5, 21.5, 19, 21.5)]], "3": [[arc_points(14, 10, 5.2, 4.8, -1.9, 1.5), arc_points(14, 18, 5.2, 4.8, -1.6, 1.9)], [arc_points(13.5, 10.5, 4.6, 4.6, -1.8, 1.6), arc_points(13.5, 18.5, 5.4, 4.6, -1.5, 1.9)]], "4": [[seg_points(9.5, 5, 8, 16), seg_points(8, 16, 20, 16), seg_points(17, 6, 17, 23)], [seg_points(10, 5.5, 7.5, 15.5), seg_points(7.5, 15.5, 20.5, 15.5), seg_points(16.5, 7, 16, 23)]], "5": [[seg_points(19, 6, 10, 6), seg_points(10, 6, 10, 13), seg_points(10, 13, 16, 13), arc_points(14, 17.5, 5, 5, -0.9, 2.2)], [seg_points(18.5, 6.5, 9.5, 6), seg_points(9.5, 6, 9.5, 13.5), seg_points(9.5, 13.5, 15.5, 13), arc_points(13.5, 18, 5, 4.8, -1.0, 2.3)]], "6": [[arc_points(14, 18, 5, 5, 0, TAU), arc_points(17, 12, 7, 8, 2.4, 4.6)], [arc_points(13.5, 18.5, 4.6, 4.6, 0, TAU), arc_points(16.5, 12.5, 6.6, 8, 2.5, 4.6)]], "7": [[seg_points(8, 6, 20, 6), seg_points(20, 6, 11, 23)], [seg_points(8.5, 6.5, 20, 6), seg_points(20, 6, 12, 23), seg_points(11, 15, 17, 14.5)]], "8": [[arc_points(14, 9.5, 4.6, 4.6, 0, TAU), arc_points(14, 18.5, 5.2, 5, 0, TAU)], [arc_points(13.5, 9.5, 4.2, 4.4, 0, TAU), arc_points(14, 18.5, 5, 5.2, 0, TAU)]], "9": [[arc_points(14, 10, 5, 5, 0, TAU), seg_points(19, 9, 18, 23)], [arc_points(13.5, 10, 4.6, 4.8, 0, TAU), seg_points(18, 9.5, 17, 23)]], } def fig_mnist_samples(): """Two rows of ten labelled digits, as the network is shown them.""" tile, gap = 74, 12 cols = len(DIGIT_STROKES) w = cols * (tile + gap) - gap + 80 h = 2 * (tile + gap) + 112 x0 = (w - (cols * (tile + gap) - gap)) / 2 body = [text(w / 2, 34, "MNIST: 60,000 labelled training images, " "10,000 for testing", 19, INK)] for col, (digit, variants) in enumerate(DIGIT_STROKES.items()): x = x0 + col * (tile + gap) for row, strokes in enumerate(variants): y = 58 + row * (tile + gap) body.append(pixel_tile(x, y, tile, strokes, seed=col * 7 + row * 101)) body.append(text(x + tile / 2, h - 44, digit, 20, BLUE)) body.append(text(w / 2, h - 14, "same label, different handwriting — " "this is the variation the network has to absorb", 15, YELLOW_D)) write("nn-mnist-samples.svg", svg("\n".join(body), w=w, h=h)) if __name__ == "__main__": for fn in (fig_three_pixels, fig_network, fig_forward, fig_neuron, fig_activations, fig_weight_images, fig_cost_1d, fig_cost_2d, fig_gradient_vector, fig_nudges, fig_comp_graph, fig_minibatch, fig_layers, fig_mnist_samples): fn()