{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<div style=\"\n",
    "    border: 2px solid #4CAF50; \n",
    "    padding: 15px; \n",
    "    background-color: #f4f4f4; \n",
    "    border-radius: 10px; \n",
    "    align-items: center;\">\n",
    "\n",
    "<h1 style=\"margin: 0; color: #4CAF50;\">Statistik im Data Science Bereich 2 (Streumaße)</h1>\n",
    "<h2 style=\"margin: 5px 0; color: #555;\">DSAI</h2>\n",
    "<h3 style=\"margin: 5px 0; color: #555;\">Jakob Eggl</h3>\n",
    "\n",
    "<div style=\"flex-shrink: 0;\">\n",
    "    <img src=\"https://www.htl-grieskirchen.at/wp/wp-content/uploads/2022/11/logo_bildschirm-1024x503.png\" alt=\"Logo\" style=\"width: 250px; height: auto;\"/>\n",
    "</div>\n",
    "<p1> © 2024/25 Jakob Eggl. Nutzung oder Verbreitung nur mit ausdrücklicher Genehmigung des Autors.</p1>\n",
    "</div>\n",
    "<div style=\"flex: 1;\">\n",
    "</div>   "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Wir kommen nun zu den Streuungsmaßen. Sie geben uns Einsicht in die Verteilung der Daten."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "\n",
    "path = os.path.join(\"..\", \"..\", \"_data\", \"titanic.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(path, decimal=\".\", sep=\",\", header=0, index_col=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pclass</th>\n",
       "      <th>survived</th>\n",
       "      <th>name</th>\n",
       "      <th>sex</th>\n",
       "      <th>age</th>\n",
       "      <th>sibsp</th>\n",
       "      <th>parch</th>\n",
       "      <th>ticket</th>\n",
       "      <th>fare</th>\n",
       "      <th>cabin</th>\n",
       "      <th>embarked</th>\n",
       "      <th>boat</th>\n",
       "      <th>body</th>\n",
       "      <th>home.dest</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Allen, Miss. Elisabeth Walton</td>\n",
       "      <td>female</td>\n",
       "      <td>29.0000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>24160</td>\n",
       "      <td>211.3375</td>\n",
       "      <td>B5</td>\n",
       "      <td>S</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>St Louis, MO</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Allison, Master. Hudson Trevor</td>\n",
       "      <td>male</td>\n",
       "      <td>0.9167</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>113781</td>\n",
       "      <td>151.5500</td>\n",
       "      <td>C22 C26</td>\n",
       "      <td>S</td>\n",
       "      <td>11</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Montreal, PQ / Chesterville, ON</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Allison, Miss. Helen Loraine</td>\n",
       "      <td>female</td>\n",
       "      <td>2.0000</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>113781</td>\n",
       "      <td>151.5500</td>\n",
       "      <td>C22 C26</td>\n",
       "      <td>S</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Montreal, PQ / Chesterville, ON</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Allison, Mr. Hudson Joshua Creighton</td>\n",
       "      <td>male</td>\n",
       "      <td>30.0000</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>113781</td>\n",
       "      <td>151.5500</td>\n",
       "      <td>C22 C26</td>\n",
       "      <td>S</td>\n",
       "      <td>NaN</td>\n",
       "      <td>135.0</td>\n",
       "      <td>Montreal, PQ / Chesterville, ON</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Allison, Mrs. Hudson J C (Bessie Waldo Daniels)</td>\n",
       "      <td>female</td>\n",
       "      <td>25.0000</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>113781</td>\n",
       "      <td>151.5500</td>\n",
       "      <td>C22 C26</td>\n",
       "      <td>S</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Montreal, PQ / Chesterville, ON</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1304</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>Zabour, Miss. Hileni</td>\n",
       "      <td>female</td>\n",
       "      <td>14.5000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2665</td>\n",
       "      <td>14.4542</td>\n",
       "      <td>NaN</td>\n",
       "      <td>C</td>\n",
       "      <td>NaN</td>\n",
       "      <td>328.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1305</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>Zabour, Miss. Thamine</td>\n",
       "      <td>female</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2665</td>\n",
       "      <td>14.4542</td>\n",
       "      <td>NaN</td>\n",
       "      <td>C</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1306</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>Zakarian, Mr. Mapriededer</td>\n",
       "      <td>male</td>\n",
       "      <td>26.5000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2656</td>\n",
       "      <td>7.2250</td>\n",
       "      <td>NaN</td>\n",
       "      <td>C</td>\n",
       "      <td>NaN</td>\n",
       "      <td>304.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1307</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>Zakarian, Mr. Ortin</td>\n",
       "      <td>male</td>\n",
       "      <td>27.0000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2670</td>\n",
       "      <td>7.2250</td>\n",
       "      <td>NaN</td>\n",
       "      <td>C</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1308</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>Zimmerman, Mr. Leo</td>\n",
       "      <td>male</td>\n",
       "      <td>29.0000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>315082</td>\n",
       "      <td>7.8750</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1309 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      pclass  survived                                             name  \\\n",
       "0          1         1                    Allen, Miss. Elisabeth Walton   \n",
       "1          1         1                   Allison, Master. Hudson Trevor   \n",
       "2          1         0                     Allison, Miss. Helen Loraine   \n",
       "3          1         0             Allison, Mr. Hudson Joshua Creighton   \n",
       "4          1         0  Allison, Mrs. Hudson J C (Bessie Waldo Daniels)   \n",
       "...      ...       ...                                              ...   \n",
       "1304       3         0                             Zabour, Miss. Hileni   \n",
       "1305       3         0                            Zabour, Miss. Thamine   \n",
       "1306       3         0                        Zakarian, Mr. Mapriededer   \n",
       "1307       3         0                              Zakarian, Mr. Ortin   \n",
       "1308       3         0                               Zimmerman, Mr. Leo   \n",
       "\n",
       "         sex      age  sibsp  parch  ticket      fare    cabin embarked boat  \\\n",
       "0     female  29.0000      0      0   24160  211.3375       B5        S    2   \n",
       "1       male   0.9167      1      2  113781  151.5500  C22 C26        S   11   \n",
       "2     female   2.0000      1      2  113781  151.5500  C22 C26        S  NaN   \n",
       "3       male  30.0000      1      2  113781  151.5500  C22 C26        S  NaN   \n",
       "4     female  25.0000      1      2  113781  151.5500  C22 C26        S  NaN   \n",
       "...      ...      ...    ...    ...     ...       ...      ...      ...  ...   \n",
       "1304  female  14.5000      1      0    2665   14.4542      NaN        C  NaN   \n",
       "1305  female      NaN      1      0    2665   14.4542      NaN        C  NaN   \n",
       "1306    male  26.5000      0      0    2656    7.2250      NaN        C  NaN   \n",
       "1307    male  27.0000      0      0    2670    7.2250      NaN        C  NaN   \n",
       "1308    male  29.0000      0      0  315082    7.8750      NaN        S  NaN   \n",
       "\n",
       "       body                        home.dest  \n",
       "0       NaN                     St Louis, MO  \n",
       "1       NaN  Montreal, PQ / Chesterville, ON  \n",
       "2       NaN  Montreal, PQ / Chesterville, ON  \n",
       "3     135.0  Montreal, PQ / Chesterville, ON  \n",
       "4       NaN  Montreal, PQ / Chesterville, ON  \n",
       "...     ...                              ...  \n",
       "1304  328.0                              NaN  \n",
       "1305    NaN                              NaN  \n",
       "1306  304.0                              NaN  \n",
       "1307    NaN                              NaN  \n",
       "1308    NaN                              NaN  \n",
       "\n",
       "[1309 rows x 14 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Spannweite"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Die Spannweite (auch \"Range\" genannt) ist ein einfaches, aber nützliches Maß der Streuung in der Statistik, das die Differenz zwischen dem größten und dem kleinsten Wert in einer Datenmenge angibt. Sie bietet einen groben Überblick über die Variabilität der Daten, indem sie die extreme Ausbreitung der Werte darstellt.\n",
    "Definition der Spannweite\n",
    "\n",
    "Die Spannweite berechnet sich durch die folgende Formel:\n",
    "\n",
    "Spannweite=Maximalwert−Minimalwert\n",
    "\n",
    "Die Spannweite gibt also an, wie weit die Daten auseinanderliegen, jedoch ohne dabei die Streuung oder Verteilung der Werte zwischen diesen Extrempunkten zu berücksichtigen.\n",
    "\n",
    "Eigenschaften der Spannweite:\n",
    "* Empfindlich gegenüber Ausreißern: Da die Spannweite nur die Extremwerte betrachtet, kann sie stark von Ausreißern beeinflusst werden.\n",
    "* Einfache Berechnung: Die Spannweite ist leicht zu berechnen und schnell verständlich.\n",
    "* Begrenzte Aussagekraft: Die Spannweite allein bietet keinen detaillierten Einblick in die Verteilung der Daten, da sie keine Informationen über die Verteilung innerhalb der Werte liefert (im Gegensatz zu anderen Maßzahlen wie dem Interquartilsabstand).\n",
    "\n",
    "#### Beispiel 1\n",
    "\n",
    "Nehmen wir an, wir haben die folgenden Daten:\n",
    "$$2, 4, 6, 8, 10, 12, 14, 16, 18, 20.$$\n",
    "Dann ist:\n",
    "* Maximalwert: 20\n",
    "* Minimalwert: 2\n",
    "\n",
    "Die Spannweite beträgt somit:\n",
    "Spannweite=20−2=18\n",
    "\n",
    "Das bedeutet, dass die Werte in dieser Datenreihe sich über eine Spannweite von 18 erstrecken.\n",
    "\n",
    "#### Beispiel 2\n",
    "\n",
    "Wenn wir derselben Datenreihe einen Ausreißer hinzufügen, z.B. 100:\n",
    "\n",
    "$$2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 100.$$\n",
    "Dann haben wir:\n",
    "* Maximalwert: 100\n",
    "* Minimalwert: 2\n",
    "\n",
    "Nun beträgt die Spannweite:\n",
    "* Spannweite=100−2=98\n",
    "\n",
    "In Kombination des realtiv niedrigen Mittelwertes ($\\bar{x}=19.091$) kann man hier somit vermuten, dass ein Ausreißer dabei ist.\n",
    "\n",
    "Im Vergleich: Bei Beispiel 1 ist der Mittelwert $\\bar{x}=11.0$, was in etwa die halbe Spannweite wiederspiegelt. Also sind die Daten kompakter, als in Beispiel 2."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Die Anwendung in pandas ist sehr einfach und intuitiv\n",
    "\n",
    "range_of_df = df.max(numeric_only=True, skipna=True) - df.min(numeric_only=True, skipna=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "pclass        2.0000\n",
       "survived      1.0000\n",
       "age          79.8333\n",
       "sibsp         8.0000\n",
       "parch         9.0000\n",
       "fare        512.3292\n",
       "body        327.0000\n",
       "dtype: float64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "range_of_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Interquartilsabstand"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Der Interquartilsabstand (IQR), der die Differenz zwischen dem dritten und ersten Quartil darstellt, ist ein Maß für die Streuung der mittleren 50% der Daten."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Der Interquartilsabstand (IQR) ist ein wichtiges Maß für die Streuung einer Datenmenge in der Statistik. Im Gegensatz zur Spannweite, die die gesamte Verteilung betrachtet, fokussiert der IQR nur auf die mittleren 50 % der Daten, was ihn robuster gegenüber Ausreißern macht. Er gibt den Bereich an, in dem sich der Großteil der Daten ohne Extremwerte befindet.\n",
    "Definition des Interquartilsabstands (IQR)\n",
    "\n",
    "Der IQR ist die Differenz zwischen dem dritten Quartil (Q3) und dem ersten Quartil (Q1):\n",
    "* IQR=Q3−Q1\n",
    "\n",
    "Der Interquartilsabstand umfasst also die mittleren 50 % der Daten. Datenpunkte außerhalb dieses Bereichs können als potenzielle Ausreißer betrachtet werden.\n",
    "\n",
    "Eigenschaften des IQR:\n",
    "* Robustheit gegenüber Ausreißern: Da der IQR nur die mittleren 50 % der Daten betrachtet, ist er unempfindlich gegenüber extrem hohen oder niedrigen Werten, die bei der Spannweite das Ergebnis stark beeinflussen könnten.\n",
    "* Nützlich für die Identifikation von Ausreißern: Datenpunkte, die außerhalb des Bereichs Q1−1.5×IQR und Q3+1.5×IQR liegen, werden oft als Ausreißer identifiziert (siehe später Boxplot).\n",
    "\n",
    "#### Beispiel 1\n",
    "\n",
    "Betrachten wir die folgende Datenreihe:\n",
    "$$12, 13, 1, 3, 10, 4, 7, 8, 9, 18, 14, 15.$$\n",
    "\n",
    "Berechnung der Quartile liefert:\n",
    "* Q1 (erstes Quartil): 5.5 (25 % der Daten sind kleiner oder gleich 5.5)\n",
    "* Q3 (drittes Quartil): 13.5 (75 % der Daten sind kleiner oder gleich 13.5)\n",
    "\n",
    "Der Interquartilsabstand (IQR) beträgt:\n",
    "IQR=13.5−5.5=8\n",
    "\n",
    "Das bedeutet, dass sich die mittleren 50 % der Daten zwischen 5.5 und 13.5 befinden. Werte außerhalb dieses Bereichs könnten Ausreißer sein.\n",
    "Beispiel mit Ausreißern:\n",
    "\n",
    "Wenn wir dieser Datenreihe einen Ausreißer hinzufügen, z.B. 30:\n",
    "$$12, 13, 1, 3, 10, 4, 7, 8, 9, 18, 14, 15, 30.$$\n",
    "* Q1 ändert sich zu 5.5 und Q3 bei 14.5, IQR=14.5-5.5=9.\n",
    "* IQR ändert sich nur wenig, obwohl der Ausreißer hinzugefügt wurde. Dies zeigt, dass der IQR robust gegenüber Ausreißern ist."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Der IQR kann wie folgt berechnet werden\n",
    "\n",
    "Q1 = df.quantile(0.25, axis=0, numeric_only=True, interpolation='midpoint')\n",
    "Q3 = df.quantile(0.75, axis=0, numeric_only=True, interpolation='midpoint')\n",
    "\n",
    "IQR = Q3 - Q1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Varianz und Standardabweichung"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Die Standardabweichung und die Varianz sind zwei der wichtigsten Maßzahlen in der Statistik, die die Streuung oder Variabilität von Daten um ihren Mittelwert beschreiben. Beide Maßzahlen werden häufig verwendet, um zu verstehen, wie weit die einzelnen Datenpunkte von ihrem Durchschnitt entfernt sind.\n"
   ]
  },
  {
   "attachments": {
    "mlr-nonconstvar-1.png": {
     "image/png": "[encoded data, 27980 characters]"
    }
   },
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![mlr-nonconstvar-1.png](attachment:mlr-nonconstvar-1.png)\n",
    "\n",
    "(from https://www.bookdown.org/rwnahhas/RMPH/RMPH_files/figure-html/mlr-nonconstvar-1.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Definition der Varianz\n",
    "\n",
    "Die Varianz misst die durchschnittliche quadratische Abweichung der Datenpunkte vom Mittelwert. Sie wird mit der folgenden Formel berechnet:\n",
    "$$\\text{Var}(X)=\\frac{\\sum_{i=1}^{n}(x_i-\\bar{x})^2}{n}$$\n",
    "Dabei ist:\n",
    "* $X$: Das Dataset, i.e. $X=(x_1, x_2, \\ldots, x_n)$\n",
    "* $n$: Anzahl der Datenpunkte\n",
    "* $x_i$: Die einzelnen Datenpunkte\n",
    "* $\\bar{x}$: Der Mittelwert\n",
    "\n",
    "Die Varianz gibt an, wie stark die Werte in einer Datenmenge um den Mittelwert schwanken. Ein kleiner Wert bedeutet, dass die Datenpunkte nahe beieinander und um den Mittelwert konzentriert sind. Ein großer Wert zeigt, dass die Daten über einen größeren Bereich verteilt sind.\n",
    "\n",
    "Bemerkung: Die obige Formel gilt nur, wenn die Daten die Grundgesamtheit respräsentieren. Ansonsten (für Stichproben) wird die Formel\n",
    "$$\n",
    "\\text{Var}(X)=\\frac{\\sum_{i=1}^{n}(x_i-\\bar{x})^2}{n-1}\n",
    "$$\n",
    "verwendet.\n",
    "**Wichtig:** Wir verwenden (außer anders spezifiziert) für die folgenden Beispiele immer die Formel für die Grundgesamtheit.\n",
    "\n",
    "##### Steinerscher Verschiebungssatz\n",
    "Es gibt auch eine Vereinfachung für die Varianzformel. Dafür sei $\\mu=1/n \\sum_{i=1}^{n} x_i$.\n",
    "\\begin{align*}\n",
    "\t\\text{Var}(X) &= \\frac{1}{n} \\sum_{i=1}^{n}(x_i-\\mu)^2\\\\\n",
    "\t&= \\frac{1}{n}\\sum_{i=1}^{n}\\left( x_i^2 - 2 x_i \\mu + \\mu^2\\right)\\\\\n",
    "\t&= \\frac{1}{n}\\sum_{i=1}^{n} x_i^2 - \n",
    "\t2\\mu\\underbrace{\\frac{1}{n}\\sum_{i=1}^{n}x_i}_{\\mu} + \n",
    "\t\\underbrace{\\frac{1}{n}\\sum_{i=1}^{n}\\mu^2}_{\\mu^2}\\\\\n",
    "\t&= \\frac{1}{n}\\sum_{i=1}^{n}x_i^2 - \\mu^2\\\\\n",
    "\t&= -\\mu^2 + \\frac{1}{n} \\sum_{i=1}^{n}x_i^2.\n",
    "\\end{align*}\n",
    "\n",
    "#### Definition der Standardabweichung\n",
    "\n",
    "Die Standardabweichung ist die Quadratwurzel aus der Varianz und gibt die durchschnittliche Abweichung der Datenpunkte vom Mittelwert an. Sie wird mit der folgenden Formel berechnet:\n",
    "\n",
    "​$$\\sigma(X)=\\sqrt[2]{\\text{Var}(X)} = \\sqrt[2]{\\frac{\\sum_{i=1}^{n}(x_i-\\bar{x})^2}{n}}$$\n",
    "\n",
    "Die **Standardabweichung** wird häufig verwendet, da sie in **derselben Einheit** wie die Originaldaten ausgedrückt wird, während die **Varianz** in **quadrierten Einheiten** vorliegt. Dadurch ist die Standardabweichung leichter interpretierbar.\n",
    "\n",
    "#### Eigenschaften von Varianz und Standardabweichung\n",
    "Empfindlichkeit gegenüber Ausreißern: Beide Maßzahlen sind empfindlich gegenüber Ausreißern, da sie auf den quadratischen Abweichungen basieren, wodurch große Abweichungen stärker gewichtet werden.\n",
    "\n",
    "#### Beispiel 1\n",
    "\n",
    "Betrachten wir eine Datenreihe:\n",
    "$$2, 4, 4, 4, 5, 5, 7, 9.$$\n",
    "\n",
    "Um die Varianz bzw. die Standardabweichung zu bestimmen berechnen wir zuerst den Mittelwert:\n",
    "$$\\bar{x} = \\frac{1}{n}\\sum_{i=1}^{n}x_i = \\frac{2+4+4+4+5+5+7+9}{8} = \\frac{40}{8} = 5.$$\n",
    "\n",
    "Nun berechnen wir die Varianz:\n",
    "\n",
    "$$\\text{Var}(X) = \\frac{1}{n}\\sum_{i=1}^{n}(x_i-\\bar{x})^2 = \\frac{1}{8}\\left((2-5)^2+(4-5)^2+(4-5)^2+(4-5)^2+(5-5)^2+(5-5)^2+(7-5)^2+(9-5)^2\\right) = \\frac{1}{8}\\left(9+1+1+1+0+0+4+16\\right)=4\\, \\text{EH}^2$$\n",
    "\n",
    "Die dazugehörige Standardabweichung ist dann die Wurzel, i.e.\n",
    "$$\\sigma(X)=\\sqrt{4} = 2\\,\\text{EH}.$$\n",
    "\n",
    "In diesem Beispiel beträgt die Varianz 4, was bedeutet, dass die durchschnittliche quadratische Abweichung der Datenpunkte vom Mittelwert 4 beträgt. Die Standardabweichung von 2 gibt an, dass die Datenpunkte im Durchschnitt um 2 Einheiten vom Mittelwert abweichen.\n",
    "\n",
    "Wie würde die Rechnung mit dem Steinerschen Verschiebungssatz aussehen?\n",
    "\n",
    "Lösung?\n",
    "\n",
    "### Beispiel 2\n",
    "\n",
    "Berechne die Standardabweichung, für folgendes Dataset. Es repräsentiert das Ergebnis einer Umfrage, bei der die befragten Personen angaben, wie viele Urlaubstage sie dieses Jahr bereits verbraucht haben.\n",
    "| Tage | Anzahl |\n",
    "|------|--------|\n",
    "| 21   | 5      |\n",
    "| 14   | 10     |\n",
    "| 25   | 40     |\n",
    "| 31   | 49     |\n",
    "| 35   | 30     |\n",
    "| 21   | 16     |\n",
    "| 29   | 50     |\n",
    "\n",
    "Lösung?\n",
    "\n",
    "#### Beispiel 3\n",
    "\n",
    "Betrachten wir die Datenreihe $X$ mit den Parametern $a,b\\in\\mathbb{R}$. Wir wissen, dass der Mittelwert $\\bar{x}=77.2$ ist und die Varianz $1183.16$ beträgt. Berechne die Werte $a$ und $b$.\n",
    "$$X=(32, 111, 138, 28, 97, 89, 101, 51, a, b)$$\n",
    "\n",
    "Lösung (Verwendung von Computerprogramm erlaubt)?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Nun berechnen wir die Varianz\n",
    "\n",
    "variance = df.var(numeric_only=True, skipna=True, axis=0) # Achtung, diese Funktion berechnet die Stichprobenvarianz, kann mit ddof geändert werden"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "pclass         0.701969\n",
       "survived       0.236250\n",
       "age          207.748974\n",
       "sibsp          1.085052\n",
       "parch          0.749195\n",
       "fare        2678.959738\n",
       "body        9544.688567\n",
       "dtype: float64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "variance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Genauso kann die Standardabweichung berechnet werden\n",
    "\n",
    "std = df.std(numeric_only=True, skipna=True, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "pclass       0.837836\n",
       "survived     0.486055\n",
       "age         14.413500\n",
       "sibsp        1.041658\n",
       "parch        0.865560\n",
       "fare        51.758668\n",
       "body        97.696922\n",
       "dtype: float64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "std"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Normalverteilung"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Nun beschäftigen wir uns noch mit der Normalverteilung. Sie ist ein essentielles Tool in der Statistik und modelliert sehr viele (zufällige) Sachverhalte. Einige Beispiele aus dem Alltag:\n",
    "* Körpergröße\n",
    "* Gewicht\n",
    "* IQ-Werte\n",
    "* Notenverteilung\n",
    "* Laufzeit von Prozessen\n",
    "\n",
    "Außerdem ist sie auch in Gebrauch beim Normalisieren von Daten (siehe Normalisierung)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Die Normalverteilung, auch als **Gaußsche Verteilung** oder **Glockenkurve** bekannt, ist eine der wichtigsten Verteilungen in der Statistik. Sie beschreibt, wie sich die Werte einer Zufallsvariablen um einen Mittelwert herum verteilen, wobei die meisten Werte nahe am Mittelwert liegen und die Wahrscheinlichkeit für extremere Werte nach beiden Seiten abnimmt."
   ]
  },
  {
   "attachments": {
    "images.png": {
     "image/png": "[encoded data, 6912 characters]"
    }
   },
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![images.png](attachment:images.png)\n",
    "\n",
    "(from https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcQ1iWPhC9cNM1uGHf6YzjJzWonw0HR3EYKMMg&s)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Eigenschaften der Normalverteilung\n",
    "\n",
    "Die Normalverteilung hat folgende Eigenschaften:\n",
    "* Symmetrisch: Die Normalverteilung ist symmetrisch um den Mittelwert.\n",
    "* Mittelwert, Median und Modus sind identisch.\n",
    "* Glockenförmige Kurve: Die Verteilung hat die Form einer Glocke, wobei die Breite und Höhe der Glocke von der Standardabweichung abhängen. Je größer die Standardabweichung, desto steiler und schmaler die Kurve.\n",
    "* Asymptotisch: Die Enden der Verteilung nähern sich der x-Achse an, berühren sie aber nie. \n",
    "* $68-95-99.7$ Regel: Diese Regel beschreibt, dass:\n",
    "    * $68 \\%$ der Werte innerhalb einer Standardabweichung ($1 \\sigma$) vom Mittelwert liegen.\n",
    "    * $95 \\%$ der Werte innerhalb von zwei Standardabweichungen ($2 \\sigma$) liegen.\n",
    "    * $99.7 \\%$ der Werte innerhalb von drei Standardabweichungen ($3 \\sigma$) liegen.\n",
    "\n",
    "Definition der Funktion:\n",
    "\n",
    "$$p(x) = \\frac{1}{\\sqrt{2\\pi}\\sigma}\\exp\\left(-\\frac{(x-\\mu)^2}{2\\sigma^2}\\right)$$\n",
    "\n",
    "wobei:\n",
    "* $\\mu$ ist der Mittelwert (Erwartungswert).\n",
    "* $\\sigma$ ist die Standardabweichung.\n",
    "\n",
    "Eine spezielle Form der Normalverteilung ist die **Standardnormalverteilung**. Sie hat $\\mu=0$ und $\\sigma=1$.\n",
    "\n",
    "#### Beispiel:\n",
    "\n",
    "Angenommen, wir betrachten die Körpergröße einer Bevölkerung, die normalverteilt ist:\n",
    "* Mittelwert $\\mu=175\\,\\textrm{cm}$\n",
    "* Standardabweichung $\\sigma=10\\, \\textrm{cm}$\n",
    "\n",
    "Berechne mit Hilfe bekannten Prozentwerte die Körpergröße Intervalle für eine Abweichung von $\\pm 1, \\pm 2, \\pm 3$ Sigma.\n",
    "\n",
    "Nach der $68-95-99.7$-Regel:\n",
    "* $68 \\%$ der Menschen haben eine Körpergröße zwischen 165 cm und 185 cm ($\\pm 1 \\sigma$)\n",
    "* $95 \\%$ der Menschen haben eine Körpergröße zwischen 155 cm und 195 cm ($\\pm 2 \\sigma$).\n",
    "* $99.7 \\%$ der Menschen haben eine Körpergröße zwischen 145 cm und 205 cm ($\\pm 3 \\sigma$).\n",
    "\n",
    "Diese Verteilung der Körpergrößen ist ein typisches Beispiel für eine Normalverteilung, bei der die meisten Menschen Größen nahe am Mittelwert haben, während sehr große oder sehr kleine Größen seltener vorkommen."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.stats import norm\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 38868 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Nun werden wir mit Normalverteilungen rechnen bzw. uns die Graphen ansehen\n",
    "\n",
    "# Verschiedene Werte können hier getestet werden, zBsp.: IQ-Verteilung: mu=100, sigma=15\n",
    "mu = 0\n",
    "sigma = 1\n",
    "\n",
    "z = np.linspace(-4, 4, 1000)\n",
    "x = z * sigma + mu\n",
    "\n",
    "y = norm.pdf(x, mu, sigma)\n",
    "\n",
    "# Nun können wir uns die Verteilung ansehen\n",
    "\n",
    "plt.plot(x, y, label=f'Normalverteilung (μ={mu}, σ={sigma})')\n",
    "plt.title('Normalverteilung')\n",
    "plt.xlabel('x')\n",
    "plt.ylabel('f(x)')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 88436 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Genauso können mehrere Normalverteilungen auf einem Graphen dargestellt werden\n",
    "params = [\n",
    "    {'mu': 0, 'sigma': 0.5},\n",
    "    {'mu': 0, 'sigma': 1},\n",
    "    {'mu': 0, 'sigma': 1.5},\n",
    "    {'mu': 1, 'sigma': 1},\n",
    "    {'mu': -1, 'sigma': 1}\n",
    "]\n",
    "\n",
    "z = np.linspace(-4,4, 1000)\n",
    "\n",
    "max_mu = max([param['mu'] for param in params])\n",
    "max_sigma = max([param['sigma'] for param in params])\n",
    "\n",
    "x = z * max_sigma + max_mu\n",
    "\n",
    "# Erstellen des Plots\n",
    "plt.figure(figsize=(10, 6))\n",
    "\n",
    "# Für jede Verteilung die PDF berechnen und plotten\n",
    "for param in params:\n",
    "    mu = param['mu']\n",
    "    sigma = param['sigma']\n",
    "    y = norm.pdf(x, mu, sigma)\n",
    "    plt.plot(x, y, label=f'μ={mu}, σ={sigma}')\n",
    "\n",
    "plt.title('Vergleich von 5 verschiedenen Normalverteilungen')\n",
    "plt.xlabel('x')\n",
    "plt.ylabel('f(x)')\n",
    "plt.legend()\n",
    "\n",
    "plt.grid(True)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Wiederholung:\n",
    "\n",
    "Was passiert bei der $z$-Normalisierung?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Pandas Describe Methode"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Viele statistische Größen können auch berechnet werden, indem die Werte in den Dataframes aggregiert werden.\n",
    "\n",
    "Außerdem gibt es die Methode `describe()`, welche wichtige statistische Maße direkt liefert."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pclass</th>\n",
       "      <th>survived</th>\n",
       "      <th>age</th>\n",
       "      <th>sibsp</th>\n",
       "      <th>parch</th>\n",
       "      <th>fare</th>\n",
       "      <th>body</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1309.000000</td>\n",
       "      <td>1309.000000</td>\n",
       "      <td>1046.000000</td>\n",
       "      <td>1309.000000</td>\n",
       "      <td>1309.000000</td>\n",
       "      <td>1308.000000</td>\n",
       "      <td>121.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>2.294882</td>\n",
       "      <td>0.381971</td>\n",
       "      <td>29.881135</td>\n",
       "      <td>0.498854</td>\n",
       "      <td>0.385027</td>\n",
       "      <td>33.295479</td>\n",
       "      <td>160.809917</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.837836</td>\n",
       "      <td>0.486055</td>\n",
       "      <td>14.413500</td>\n",
       "      <td>1.041658</td>\n",
       "      <td>0.865560</td>\n",
       "      <td>51.758668</td>\n",
       "      <td>97.696922</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.166700</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>2.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>21.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.895800</td>\n",
       "      <td>72.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>3.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>28.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>14.454200</td>\n",
       "      <td>155.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>3.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>39.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>31.275000</td>\n",
       "      <td>256.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>3.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>80.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>512.329200</td>\n",
       "      <td>328.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            pclass     survived          age        sibsp        parch  \\\n",
       "count  1309.000000  1309.000000  1046.000000  1309.000000  1309.000000   \n",
       "mean      2.294882     0.381971    29.881135     0.498854     0.385027   \n",
       "std       0.837836     0.486055    14.413500     1.041658     0.865560   \n",
       "min       1.000000     0.000000     0.166700     0.000000     0.000000   \n",
       "25%       2.000000     0.000000    21.000000     0.000000     0.000000   \n",
       "50%       3.000000     0.000000    28.000000     0.000000     0.000000   \n",
       "75%       3.000000     1.000000    39.000000     1.000000     0.000000   \n",
       "max       3.000000     1.000000    80.000000     8.000000     9.000000   \n",
       "\n",
       "              fare        body  \n",
       "count  1308.000000  121.000000  \n",
       "mean     33.295479  160.809917  \n",
       "std      51.758668   97.696922  \n",
       "min       0.000000    1.000000  \n",
       "25%       7.895800   72.000000  \n",
       "50%      14.454200  155.000000  \n",
       "75%      31.275000  256.000000  \n",
       "max     512.329200  328.000000  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Deskriptive Statistiken für alle (numerischen) Spalten\n",
    "df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pclass</th>\n",
       "      <th>survived</th>\n",
       "      <th>name</th>\n",
       "      <th>sex</th>\n",
       "      <th>age</th>\n",
       "      <th>sibsp</th>\n",
       "      <th>parch</th>\n",
       "      <th>ticket</th>\n",
       "      <th>fare</th>\n",
       "      <th>cabin</th>\n",
       "      <th>embarked</th>\n",
       "      <th>boat</th>\n",
       "      <th>body</th>\n",
       "      <th>home.dest</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1309.000000</td>\n",
       "      <td>1309.000000</td>\n",
       "      <td>1309</td>\n",
       "      <td>1309</td>\n",
       "      <td>1046.000000</td>\n",
       "      <td>1309.000000</td>\n",
       "      <td>1309.000000</td>\n",
       "      <td>1309</td>\n",
       "      <td>1308.000000</td>\n",
       "      <td>295</td>\n",
       "      <td>1307</td>\n",
       "      <td>486</td>\n",
       "      <td>121.000000</td>\n",
       "      <td>745</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unique</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1307</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>929</td>\n",
       "      <td>NaN</td>\n",
       "      <td>186</td>\n",
       "      <td>3</td>\n",
       "      <td>27</td>\n",
       "      <td>NaN</td>\n",
       "      <td>369</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>top</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Kelly, Mr. James</td>\n",
       "      <td>male</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>CA. 2343</td>\n",
       "      <td>NaN</td>\n",
       "      <td>C23 C25 C27</td>\n",
       "      <td>S</td>\n",
       "      <td>13</td>\n",
       "      <td>NaN</td>\n",
       "      <td>New York, NY</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>freq</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "      <td>843</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>11</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6</td>\n",
       "      <td>914</td>\n",
       "      <td>39</td>\n",
       "      <td>NaN</td>\n",
       "      <td>64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>2.294882</td>\n",
       "      <td>0.381971</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>29.881135</td>\n",
       "      <td>0.498854</td>\n",
       "      <td>0.385027</td>\n",
       "      <td>NaN</td>\n",
       "      <td>33.295479</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>160.809917</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.837836</td>\n",
       "      <td>0.486055</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>14.413500</td>\n",
       "      <td>1.041658</td>\n",
       "      <td>0.865560</td>\n",
       "      <td>NaN</td>\n",
       "      <td>51.758668</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>97.696922</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.166700</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>2.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>21.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7.895800</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>72.000000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>3.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>28.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>14.454200</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>155.000000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>3.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>39.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>31.275000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>256.000000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>3.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>80.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>512.329200</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>328.000000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             pclass     survived              name   sex          age  \\\n",
       "count   1309.000000  1309.000000              1309  1309  1046.000000   \n",
       "unique          NaN          NaN              1307     2          NaN   \n",
       "top             NaN          NaN  Kelly, Mr. James  male          NaN   \n",
       "freq            NaN          NaN                 2   843          NaN   \n",
       "mean       2.294882     0.381971               NaN   NaN    29.881135   \n",
       "std        0.837836     0.486055               NaN   NaN    14.413500   \n",
       "min        1.000000     0.000000               NaN   NaN     0.166700   \n",
       "25%        2.000000     0.000000               NaN   NaN    21.000000   \n",
       "50%        3.000000     0.000000               NaN   NaN    28.000000   \n",
       "75%        3.000000     1.000000               NaN   NaN    39.000000   \n",
       "max        3.000000     1.000000               NaN   NaN    80.000000   \n",
       "\n",
       "              sibsp        parch    ticket         fare        cabin embarked  \\\n",
       "count   1309.000000  1309.000000      1309  1308.000000          295     1307   \n",
       "unique          NaN          NaN       929          NaN          186        3   \n",
       "top             NaN          NaN  CA. 2343          NaN  C23 C25 C27        S   \n",
       "freq            NaN          NaN        11          NaN            6      914   \n",
       "mean       0.498854     0.385027       NaN    33.295479          NaN      NaN   \n",
       "std        1.041658     0.865560       NaN    51.758668          NaN      NaN   \n",
       "min        0.000000     0.000000       NaN     0.000000          NaN      NaN   \n",
       "25%        0.000000     0.000000       NaN     7.895800          NaN      NaN   \n",
       "50%        0.000000     0.000000       NaN    14.454200          NaN      NaN   \n",
       "75%        1.000000     0.000000       NaN    31.275000          NaN      NaN   \n",
       "max        8.000000     9.000000       NaN   512.329200          NaN      NaN   \n",
       "\n",
       "       boat        body     home.dest  \n",
       "count   486  121.000000           745  \n",
       "unique   27         NaN           369  \n",
       "top      13         NaN  New York, NY  \n",
       "freq     39         NaN            64  \n",
       "mean    NaN  160.809917           NaN  \n",
       "std     NaN   97.696922           NaN  \n",
       "min     NaN    1.000000           NaN  \n",
       "25%     NaN   72.000000           NaN  \n",
       "50%     NaN  155.000000           NaN  \n",
       "75%     NaN  256.000000           NaN  \n",
       "max     NaN  328.000000           NaN  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Sollen alle Spalten verwendet werden (i.e. auch die nicht numerischen), so verwenden wir folgendes.\n",
    "df.describe(include='all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Wir können natürlich auch die deskriptiven Statistiken mit der Aggregationsfunktion berechnen\n",
    "summary_statistics = df.select_dtypes(include='number').agg(['mean', 'std', 'min', 'max', 'median'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pclass</th>\n",
       "      <th>survived</th>\n",
       "      <th>age</th>\n",
       "      <th>sibsp</th>\n",
       "      <th>parch</th>\n",
       "      <th>fare</th>\n",
       "      <th>body</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>2.294882</td>\n",
       "      <td>0.381971</td>\n",
       "      <td>29.881135</td>\n",
       "      <td>0.498854</td>\n",
       "      <td>0.385027</td>\n",
       "      <td>33.295479</td>\n",
       "      <td>160.809917</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.837836</td>\n",
       "      <td>0.486055</td>\n",
       "      <td>14.413500</td>\n",
       "      <td>1.041658</td>\n",
       "      <td>0.865560</td>\n",
       "      <td>51.758668</td>\n",
       "      <td>97.696922</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.166700</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>3.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>80.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>512.329200</td>\n",
       "      <td>328.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>median</th>\n",
       "      <td>3.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>28.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>14.454200</td>\n",
       "      <td>155.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          pclass  survived        age     sibsp     parch        fare  \\\n",
       "mean    2.294882  0.381971  29.881135  0.498854  0.385027   33.295479   \n",
       "std     0.837836  0.486055  14.413500  1.041658  0.865560   51.758668   \n",
       "min     1.000000  0.000000   0.166700  0.000000  0.000000    0.000000   \n",
       "max     3.000000  1.000000  80.000000  8.000000  9.000000  512.329200   \n",
       "median  3.000000  0.000000  28.000000  0.000000  0.000000   14.454200   \n",
       "\n",
       "              body  \n",
       "mean    160.809917  \n",
       "std      97.696922  \n",
       "min       1.000000  \n",
       "max     328.000000  \n",
       "median  155.000000  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "summary_statistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Des weiteren können viele Statistiken auch mit dem stats Modul von scipy erstellt werden\n",
    "\n",
    "from scipy import stats\n",
    "\n",
    "# Berechnen der Summary Statistics für das Dataframe df mit scipy stats\n",
    "summary_statistics_scipy = df.select_dtypes(include='number').apply(stats.describe, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pclass</th>\n",
       "      <th>survived</th>\n",
       "      <th>age</th>\n",
       "      <th>sibsp</th>\n",
       "      <th>parch</th>\n",
       "      <th>fare</th>\n",
       "      <th>body</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1309</td>\n",
       "      <td>1309</td>\n",
       "      <td>1309</td>\n",
       "      <td>1309</td>\n",
       "      <td>1309</td>\n",
       "      <td>1309</td>\n",
       "      <td>1309</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>(1, 3)</td>\n",
       "      <td>(0, 1)</td>\n",
       "      <td>(nan, nan)</td>\n",
       "      <td>(0, 8)</td>\n",
       "      <td>(0, 9)</td>\n",
       "      <td>(nan, nan)</td>\n",
       "      <td>(nan, nan)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2.294882</td>\n",
       "      <td>0.381971</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.498854</td>\n",
       "      <td>0.385027</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.701969</td>\n",
       "      <td>0.23625</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.085052</td>\n",
       "      <td>0.749195</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-0.597961</td>\n",
       "      <td>0.485847</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.839814</td>\n",
       "      <td>3.664872</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>-1.314641</td>\n",
       "      <td>-1.763953</td>\n",
       "      <td>NaN</td>\n",
       "      <td>19.962194</td>\n",
       "      <td>21.454306</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     pclass  survived         age      sibsp      parch        fare  \\\n",
       "0      1309      1309        1309       1309       1309        1309   \n",
       "1    (1, 3)    (0, 1)  (nan, nan)     (0, 8)     (0, 9)  (nan, nan)   \n",
       "2  2.294882  0.381971         NaN   0.498854   0.385027         NaN   \n",
       "3  0.701969   0.23625         NaN   1.085052   0.749195         NaN   \n",
       "4 -0.597961  0.485847         NaN   3.839814   3.664872         NaN   \n",
       "5 -1.314641 -1.763953         NaN  19.962194  21.454306         NaN   \n",
       "\n",
       "         body  \n",
       "0        1309  \n",
       "1  (nan, nan)  \n",
       "2         NaN  \n",
       "3         NaN  \n",
       "4         NaN  \n",
       "5         NaN  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "summary_statistics_scipy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Wir sehen, dass hier der Output etwas anders ist. Die genaue Output Beschreibung findet man in der Dokumentation."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Anscombe's Quartett"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Ist Statistik wirklich ausreichend, um die Daten gut zu beschreiben?\n",
    "\n",
    "**Nein**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 40552 characters]",
      "text/plain": [
       "<Figure size 600x600 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# from https://matplotlib.org/stable/gallery/specialty_plots/anscombe.html\n",
    "\n",
    "x = [10, 8, 13, 9, 11, 14, 6, 4, 12, 7, 5]\n",
    "y1 = [8.04, 6.95, 7.58, 8.81, 8.33, 9.96, 7.24, 4.26, 10.84, 4.82, 5.68]\n",
    "y2 = [9.14, 8.14, 8.74, 8.77, 9.26, 8.10, 6.13, 3.10, 9.13, 7.26, 4.74]\n",
    "y3 = [7.46, 6.77, 12.74, 7.11, 7.81, 8.84, 6.08, 5.39, 8.15, 6.42, 5.73]\n",
    "x4 = [8, 8, 8, 8, 8, 8, 8, 19, 8, 8, 8]\n",
    "y4 = [6.58, 5.76, 7.71, 8.84, 8.47, 7.04, 5.25, 12.50, 5.56, 7.91, 6.89]\n",
    "\n",
    "datasets = {\n",
    "    'I': (x, y1),\n",
    "    'II': (x, y2),\n",
    "    'III': (x, y3),\n",
    "    'IV': (x4, y4)\n",
    "}\n",
    "\n",
    "fig, axs = plt.subplots(2, 2, sharex=True, sharey=True, figsize=(6, 6),\n",
    "                        gridspec_kw={'wspace': 0.08, 'hspace': 0.08})\n",
    "axs[0, 0].set(xlim=(0, 20), ylim=(2, 14))\n",
    "axs[0, 0].set(xticks=(0, 10, 20), yticks=(4, 8, 12))\n",
    "\n",
    "for ax, (label, (x, y)) in zip(axs.flat, datasets.items()):\n",
    "    ax.text(0.1, 0.9, label, fontsize=20, transform=ax.transAxes, va='top')\n",
    "    ax.tick_params(direction='in', top=True, right=True)\n",
    "    ax.plot(x, y, 'o')\n",
    "\n",
    "    # linear regression\n",
    "    p1, p0 = np.polyfit(x, y, deg=1)  # slope, intercept\n",
    "    ax.axline(xy1=(0, p0), slope=p1, color='r', lw=2)\n",
    "\n",
    "    # add text box for the statistics\n",
    "    stats = (f'$\\\\mu$ = {np.mean(y):.2f}\\n'\n",
    "             f'$\\\\sigma$ = {np.std(y):.2f}\\n')\n",
    "            #  f'$r$ = {np.corrcoef(x, y)[0][1]:.2f}')\n",
    "    bbox = dict(boxstyle='round', fc='blanchedalmond', ec='orange', alpha=0.5)\n",
    "    ax.text(0.95, 0.07, stats, fontsize=9, bbox=bbox,\n",
    "            transform=ax.transAxes, horizontalalignment='right')\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Wie wir sehen können, geben uns die Grafiken viel mehr Einblick in die gegebenen Daten, weswegen wir nun Visualisierungen besprechen werden."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "dsai",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.20"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
