Activation-functions.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Activation Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "def plot(title, yTickMult=1):\n",
    "    fig, ax = plt.subplots(figsize=(10,6))\n",
    "    ax.axvline(x=0, color='black', linewidth=1)\n",
    "    ax.axhline(y=0, color='black', linewidth=1)\n",
    "    ax.yaxis.set_major_locator(plt.MultipleLocator(yTickMult))\n",
    "    ax.set_title(title, fontsize=18)\n",
    "    return (fig, ax)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sigmoid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sigmoid(z):\n",
    "  return 1.0 / (1 + np.exp(-z))\n",
    "\n",
    "def dsigmoid(z):\n",
    "  return sigmoid(z)*(1-sigmoid(z))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = np.linspace(-8,8,100)\n",
    "y = sigmoid(x)\n",
    "\n",
    "fig, ax = plot('Sigmoid Function', 0.5)\n",
    "ax.plot(x,y);\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = np.linspace(-2.5,2.5,100)\n",
    "y = sigmoid(x)\n",
    "\n",
    "fig, ax = plot('Sigmoid Function - Echte Steigung', 0.5)\n",
    "\n",
    "ax.axis('equal')\n",
    "ax.set_ylim([0,1])\n",
    "ax.plot(x,y);\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = np.linspace(-8,8,100)\n",
    "y = sigmoid(x)\n",
    "\n",
    "fig, ax = plt.subplots(figsize=(10,6))\n",
    "ax.axvline(x=0, color='black', linewidth=1)\n",
    "ax.set_yscale('logit')\n",
    "ax.set_title('Sigmoid Function - Logit-Skala', fontsize=18)\n",
    "ax.plot(x,y);\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = np.linspace(-8,8,100)\n",
    "y = dsigmoid(x)\n",
    "\n",
    "fig, ax = plot('Ableitung der Sigmoid Function', 0.5)\n",
    "ax.plot(x, sigmoid(x), label = 'Sigmoid')\n",
    "ax.plot(x,y, label = 'Ableitung');\n",
    "ax.legend();"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ReLu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = np.linspace(-8,8,100)\n",
    "y = np.maximum(0,x)\n",
    "\n",
    "fig, ax = plot('Rectified Linear Unit', 2)\n",
    "ax.plot(x,y);\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Leaky ReLU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = np.linspace(-8,8,100)\n",
    "alpha = 0.1\n",
    "y = np.maximum(0,x) - np.maximum(0, -alpha*x)\n",
    "\n",
    "fig, ax = plot('Leaky Rectified Linear Unit', 2)\n",
    "ax.plot(x,y);\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tanh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = np.linspace(-8,8,100)\n",
    "y = np.tanh(x)\n",
    "\n",
    "fig, ax = plot('Tangens Hyperbolicus', 0.5)\n",
    "ax.plot(x,y);\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ArcTan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = np.linspace(-8,8,100)\n",
    "y = np.arctan(x)\n",
    "\n",
    "fig, ax = plot('Arcustangens', 0.5)\n",
    "ax.plot(x,y);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Swish"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def swish(z, beta=1):\n",
    "  return z / (1 + np.exp(-z*beta))\n",
    "\n",
    "def dswish(z, beta):\n",
    "  return beta * swish(z, beta) + sigmoid(z)*(1 - beta * swish(z, beta))\n",
    "\n",
    "betas = [0.1, 0.25, 0.5, 1]\n",
    "\n",
    "x = np.linspace(-8,8,100)\n",
    "ys = [swish(x, beta) for beta in betas]\n",
    "\n",
    "fig, ax = plot(r'Swish für verschiedene $\\beta$')\n",
    "\n",
    "for idx, beta in enumerate(betas):\n",
    "  ax.plot(x,ys[idx], label=r'$\\beta$ = ' + f'{beta}')\n",
    "\n",
    "ax.legend();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plot(r'Swish für $\\beta$ = 1')\n",
    "ax.plot(x,swish(x));"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = np.linspace(-8,8,100)\n",
    "ys = [dswish(x, beta) for beta in betas]\n",
    "\n",
    "fig, ax = plt.subplots(figsize=(10,6))\n",
    "ax.axvline(x=0, color='black', linewidth=1)\n",
    "ax.axhline(y=0, color='black', linewidth=1)\n",
    "ax.yaxis.set_major_locator(plt.MultipleLocator(0.5))\n",
    "ax.set_title(r'Ableitung von Swish für verschiedene $\\beta$', fontsize=18)\n",
    "\n",
    "for idx, beta in enumerate(betas):\n",
    "  ax.plot(x,ys[idx], label=r'$\\beta$ = ' + f'{beta}')\n",
    "\n",
    "ax.legend();"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sigmoid, Tanh, ArcTan im Vergleich"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = np.linspace(-8,8,100)\n",
    "yarctan = np.arctan(x)\n",
    "ytanh = np.tanh(x)\n",
    "ysig = sigmoid(x)\n",
    "#ylrelu = np.maximum(0,x) - np.maximum(0, -alpha*x)\n",
    "\n",
    "fig, ax = plt.subplots(figsize=(15,8))\n",
    "ax.axvline(x=0, color='black', linewidth=1)\n",
    "ax.axhline(y=0, color='black', linewidth=1)\n",
    "ax.yaxis.set_major_locator(plt.MultipleLocator(0.5))\n",
    "ax.set_title('Aktivierungen im Vergleich', fontsize=18)\n",
    "ax.plot(x,ysig, label=r'Sigmoid - $\\sigma$');\n",
    "ax.plot(x,yarctan, label='Arcustangens - arctan');\n",
    "ax.plot(x,ytanh, label='Tangens Hyperbolicus - tanh');\n",
    "#ax.plot(x,ylrelu, label='Leaky ReLU');\n",
    "ax.legend();"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ReLU und Swish im Vergleich"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = np.linspace(-8,8,100)\n",
    "ysw = x*sigmoid(x)\n",
    "yrelu = np.maximum(0,x)\n",
    "\n",
    "fig, ax = plt.subplots(figsize=(15,8))\n",
    "ax.axvline(x=0, color='black', linewidth=1)\n",
    "ax.axhline(y=0, color='black', linewidth=1)\n",
    "ax.yaxis.set_major_locator(plt.MultipleLocator(0.5))\n",
    "ax.set_title('Aktivierungen im Vergleich', fontsize=18)\n",
    "ax.plot(x,yrelu, label=r'ReLU');\n",
    "ax.plot(x,ysw, label='Swish');\n",
    "ax.legend();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}