Lecture15_Logistic_Regression_Stat.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "pd.options.display.max_columns=None\n",
    "pd.options.display.max_rows=None"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## &#50508;&#44256; &#44032;&#50556; &#54624; &#44163; &#46308;\n",
    "\n",
    "### 1. Evidence of Weight\n",
    "### 2. Information Value\n",
    "### 3. VIF\n",
    "### 4. C &#53685;&#44228;&#47049;\n",
    "### 5. AIC\n",
    "### 6. ROC, AUC"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## &#45936;&#51060;&#53552; &#49483;&#51012; &#47196;&#46377;&#54633;&#45768;&#45796;."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = 'https://archive.ics.uci.edu/ml/machine-learning-databases/statlog/german/german.data'\n",
    "dataset = pd.read_csv(path, delimiter=' ', header=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1000, 21)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>14</th>\n",
       "      <th>15</th>\n",
       "      <th>16</th>\n",
       "      <th>17</th>\n",
       "      <th>18</th>\n",
       "      <th>19</th>\n",
       "      <th>20</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A11</td>\n",
       "      <td>6</td>\n",
       "      <td>A34</td>\n",
       "      <td>A43</td>\n",
       "      <td>1169</td>\n",
       "      <td>A65</td>\n",
       "      <td>A75</td>\n",
       "      <td>4</td>\n",
       "      <td>A93</td>\n",
       "      <td>A101</td>\n",
       "      <td>4</td>\n",
       "      <td>A121</td>\n",
       "      <td>67</td>\n",
       "      <td>A143</td>\n",
       "      <td>A152</td>\n",
       "      <td>2</td>\n",
       "      <td>A173</td>\n",
       "      <td>1</td>\n",
       "      <td>A192</td>\n",
       "      <td>A201</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A12</td>\n",
       "      <td>48</td>\n",
       "      <td>A32</td>\n",
       "      <td>A43</td>\n",
       "      <td>5951</td>\n",
       "      <td>A61</td>\n",
       "      <td>A73</td>\n",
       "      <td>2</td>\n",
       "      <td>A92</td>\n",
       "      <td>A101</td>\n",
       "      <td>2</td>\n",
       "      <td>A121</td>\n",
       "      <td>22</td>\n",
       "      <td>A143</td>\n",
       "      <td>A152</td>\n",
       "      <td>1</td>\n",
       "      <td>A173</td>\n",
       "      <td>1</td>\n",
       "      <td>A191</td>\n",
       "      <td>A201</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A14</td>\n",
       "      <td>12</td>\n",
       "      <td>A34</td>\n",
       "      <td>A46</td>\n",
       "      <td>2096</td>\n",
       "      <td>A61</td>\n",
       "      <td>A74</td>\n",
       "      <td>2</td>\n",
       "      <td>A93</td>\n",
       "      <td>A101</td>\n",
       "      <td>3</td>\n",
       "      <td>A121</td>\n",
       "      <td>49</td>\n",
       "      <td>A143</td>\n",
       "      <td>A152</td>\n",
       "      <td>1</td>\n",
       "      <td>A172</td>\n",
       "      <td>2</td>\n",
       "      <td>A191</td>\n",
       "      <td>A201</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>A11</td>\n",
       "      <td>42</td>\n",
       "      <td>A32</td>\n",
       "      <td>A42</td>\n",
       "      <td>7882</td>\n",
       "      <td>A61</td>\n",
       "      <td>A74</td>\n",
       "      <td>2</td>\n",
       "      <td>A93</td>\n",
       "      <td>A103</td>\n",
       "      <td>4</td>\n",
       "      <td>A122</td>\n",
       "      <td>45</td>\n",
       "      <td>A143</td>\n",
       "      <td>A153</td>\n",
       "      <td>1</td>\n",
       "      <td>A173</td>\n",
       "      <td>2</td>\n",
       "      <td>A191</td>\n",
       "      <td>A201</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>A11</td>\n",
       "      <td>24</td>\n",
       "      <td>A33</td>\n",
       "      <td>A40</td>\n",
       "      <td>4870</td>\n",
       "      <td>A61</td>\n",
       "      <td>A73</td>\n",
       "      <td>3</td>\n",
       "      <td>A93</td>\n",
       "      <td>A101</td>\n",
       "      <td>4</td>\n",
       "      <td>A124</td>\n",
       "      <td>53</td>\n",
       "      <td>A143</td>\n",
       "      <td>A153</td>\n",
       "      <td>2</td>\n",
       "      <td>A173</td>\n",
       "      <td>2</td>\n",
       "      <td>A191</td>\n",
       "      <td>A201</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    0   1    2    3     4    5    6   7    8     9   10    11  12    13    14  \\\n",
       "0  A11   6  A34  A43  1169  A65  A75   4  A93  A101   4  A121  67  A143  A152   \n",
       "1  A12  48  A32  A43  5951  A61  A73   2  A92  A101   2  A121  22  A143  A152   \n",
       "2  A14  12  A34  A46  2096  A61  A74   2  A93  A101   3  A121  49  A143  A152   \n",
       "3  A11  42  A32  A42  7882  A61  A74   2  A93  A103   4  A122  45  A143  A153   \n",
       "4  A11  24  A33  A40  4870  A61  A73   3  A93  A101   4  A124  53  A143  A153   \n",
       "\n",
       "   15    16  17    18    19  20  \n",
       "0   2  A173   1  A192  A201   1  \n",
       "1   1  A173   1  A191  A201   2  \n",
       "2   1  A172   2  A191  A201   1  \n",
       "3   1  A173   2  A191  A201   1  \n",
       "4   2  A173   2  A191  A201   2  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "COL = [\n",
    "    'Status_of_existing_checking_account', \n",
    "    'Duration_in_month',\n",
    "    'Credit_history',\n",
    "    'Purpose', \n",
    "    'Credit_amount', \n",
    "    'Savings_account_bonds', \n",
    "    'Present_employment_since', \n",
    "    'Installment_rate_in_percentage_of_disposable_income',\n",
    "    'Personal_status_and_sex',\n",
    "    'Other_debtors_guarantors',\n",
    "    'Present_residence_since', \n",
    "    'Property', \n",
    "    'Age_in_years', \n",
    "    'Other_installment_plans', \n",
    "    'Housing',\n",
    "    'Number_of_existing_credits_at_this_bank',\n",
    "    'Job',\n",
    "    'Number_of_people_being_liable_to_provide_maintenance_for',\n",
    "    'Telephone',\n",
    "    'foreign_worker',\n",
    "    'Target'\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset.columns = COL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Status_of_existing_checking_account</th>\n",
       "      <th>Duration_in_month</th>\n",
       "      <th>Credit_history</th>\n",
       "      <th>Purpose</th>\n",
       "      <th>Credit_amount</th>\n",
       "      <th>Savings_account_bonds</th>\n",
       "      <th>Present_employment_since</th>\n",
       "      <th>Installment_rate_in_percentage_of_disposable_income</th>\n",
       "      <th>Personal_status_and_sex</th>\n",
       "      <th>Other_debtors_guarantors</th>\n",
       "      <th>Present_residence_since</th>\n",
       "      <th>Property</th>\n",
       "      <th>Age_in_years</th>\n",
       "      <th>Other_installment_plans</th>\n",
       "      <th>Housing</th>\n",
       "      <th>Number_of_existing_credits_at_this_bank</th>\n",
       "      <th>Job</th>\n",
       "      <th>Number_of_people_being_liable_to_provide_maintenance_for</th>\n",
       "      <th>Telephone</th>\n",
       "      <th>foreign_worker</th>\n",
       "      <th>Target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A11</td>\n",
       "      <td>6</td>\n",
       "      <td>A34</td>\n",
       "      <td>A43</td>\n",
       "      <td>1169</td>\n",
       "      <td>A65</td>\n",
       "      <td>A75</td>\n",
       "      <td>4</td>\n",
       "      <td>A93</td>\n",
       "      <td>A101</td>\n",
       "      <td>4</td>\n",
       "      <td>A121</td>\n",
       "      <td>67</td>\n",
       "      <td>A143</td>\n",
       "      <td>A152</td>\n",
       "      <td>2</td>\n",
       "      <td>A173</td>\n",
       "      <td>1</td>\n",
       "      <td>A192</td>\n",
       "      <td>A201</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A12</td>\n",
       "      <td>48</td>\n",
       "      <td>A32</td>\n",
       "      <td>A43</td>\n",
       "      <td>5951</td>\n",
       "      <td>A61</td>\n",
       "      <td>A73</td>\n",
       "      <td>2</td>\n",
       "      <td>A92</td>\n",
       "      <td>A101</td>\n",
       "      <td>2</td>\n",
       "      <td>A121</td>\n",
       "      <td>22</td>\n",
       "      <td>A143</td>\n",
       "      <td>A152</td>\n",
       "      <td>1</td>\n",
       "      <td>A173</td>\n",
       "      <td>1</td>\n",
       "      <td>A191</td>\n",
       "      <td>A201</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A14</td>\n",
       "      <td>12</td>\n",
       "      <td>A34</td>\n",
       "      <td>A46</td>\n",
       "      <td>2096</td>\n",
       "      <td>A61</td>\n",
       "      <td>A74</td>\n",
       "      <td>2</td>\n",
       "      <td>A93</td>\n",
       "      <td>A101</td>\n",
       "      <td>3</td>\n",
       "      <td>A121</td>\n",
       "      <td>49</td>\n",
       "      <td>A143</td>\n",
       "      <td>A152</td>\n",
       "      <td>1</td>\n",
       "      <td>A172</td>\n",
       "      <td>2</td>\n",
       "      <td>A191</td>\n",
       "      <td>A201</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>A11</td>\n",
       "      <td>42</td>\n",
       "      <td>A32</td>\n",
       "      <td>A42</td>\n",
       "      <td>7882</td>\n",
       "      <td>A61</td>\n",
       "      <td>A74</td>\n",
       "      <td>2</td>\n",
       "      <td>A93</td>\n",
       "      <td>A103</td>\n",
       "      <td>4</td>\n",
       "      <td>A122</td>\n",
       "      <td>45</td>\n",
       "      <td>A143</td>\n",
       "      <td>A153</td>\n",
       "      <td>1</td>\n",
       "      <td>A173</td>\n",
       "      <td>2</td>\n",
       "      <td>A191</td>\n",
       "      <td>A201</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>A11</td>\n",
       "      <td>24</td>\n",
       "      <td>A33</td>\n",
       "      <td>A40</td>\n",
       "      <td>4870</td>\n",
       "      <td>A61</td>\n",
       "      <td>A73</td>\n",
       "      <td>3</td>\n",
       "      <td>A93</td>\n",
       "      <td>A101</td>\n",
       "      <td>4</td>\n",
       "      <td>A124</td>\n",
       "      <td>53</td>\n",
       "      <td>A143</td>\n",
       "      <td>A153</td>\n",
       "      <td>2</td>\n",
       "      <td>A173</td>\n",
       "      <td>2</td>\n",
       "      <td>A191</td>\n",
       "      <td>A201</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Status_of_existing_checking_account  Duration_in_month Credit_history  \\\n",
       "0                                 A11                  6            A34   \n",
       "1                                 A12                 48            A32   \n",
       "2                                 A14                 12            A34   \n",
       "3                                 A11                 42            A32   \n",
       "4                                 A11                 24            A33   \n",
       "\n",
       "  Purpose  Credit_amount Savings_account_bonds Present_employment_since  \\\n",
       "0     A43           1169                   A65                      A75   \n",
       "1     A43           5951                   A61                      A73   \n",
       "2     A46           2096                   A61                      A74   \n",
       "3     A42           7882                   A61                      A74   \n",
       "4     A40           4870                   A61                      A73   \n",
       "\n",
       "   Installment_rate_in_percentage_of_disposable_income  \\\n",
       "0                                                  4     \n",
       "1                                                  2     \n",
       "2                                                  2     \n",
       "3                                                  2     \n",
       "4                                                  3     \n",
       "\n",
       "  Personal_status_and_sex Other_debtors_guarantors  Present_residence_since  \\\n",
       "0                     A93                     A101                        4   \n",
       "1                     A92                     A101                        2   \n",
       "2                     A93                     A101                        3   \n",
       "3                     A93                     A103                        4   \n",
       "4                     A93                     A101                        4   \n",
       "\n",
       "  Property  Age_in_years Other_installment_plans Housing  \\\n",
       "0     A121            67                    A143    A152   \n",
       "1     A121            22                    A143    A152   \n",
       "2     A121            49                    A143    A152   \n",
       "3     A122            45                    A143    A153   \n",
       "4     A124            53                    A143    A153   \n",
       "\n",
       "   Number_of_existing_credits_at_this_bank   Job  \\\n",
       "0                                        2  A173   \n",
       "1                                        1  A173   \n",
       "2                                        1  A172   \n",
       "3                                        1  A173   \n",
       "4                                        2  A173   \n",
       "\n",
       "   Number_of_people_being_liable_to_provide_maintenance_for Telephone  \\\n",
       "0                                                  1             A192   \n",
       "1                                                  1             A191   \n",
       "2                                                  2             A191   \n",
       "3                                                  2             A191   \n",
       "4                                                  2             A191   \n",
       "\n",
       "  foreign_worker  Target  \n",
       "0           A201       1  \n",
       "1           A201       2  \n",
       "2           A201       1  \n",
       "3           A201       1  \n",
       "4           A201       2  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# &#52509; 20&#44060;&#51032; &#49444;&#47749; &#48320;&#49688;&#50752; 1&#44060;&#51032; &#51333;&#49549; &#48320;&#49688;\n",
    "dataset.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Label\n",
    " - 1 : Good (&#44058;&#51012; &#49688; &#51080;&#45796; / &#49888;&#50857;&#46020;&#44032; &#51339;&#45796;) $\\to$ 0&#51004;&#47196; &#48320;&#44221;\n",
    " - 2 : Bad (&#44058;&#51012; &#49688; &#50630;&#45796; / &#49888;&#50857;&#46020;&#44032; &#45208;&#49240;&#45796;) $\\to$ 1&#47196; &#48320;&#44221;"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Ham(0) / Spam(1)\n",
    "- &#51221;&#49345;&#49324;&#46988;(0) / &#50516;&#54872;&#51088;(1)\n",
    "- &#51060;&#48292;&#53944;&#44032; &#48156;&#49373;&#54616;&#51648; &#50506;&#51020;(0) / &#51060;&#48292;&#53944;&#44032; &#48156;&#49373;&#54632;(1)\n",
    "- &#44480;&#47924;&#44032;&#49444;(0) / &#45824;&#47549;&#44032;&#49444;(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Label&#51012; 1/2 -> 0/1&#47196; &#48320;&#44221;\n",
    "dataset['Target'] = dataset['Target'] - 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Information Value (&#51221;&#48372; &#44032;&#52824;)\n",
    "&#47784;&#45944;&#50640;&#49436; &#48320;&#49688;&#51032; &#49324;&#50857;&#50976;&#47924;&#47484; &#54032;&#45800;&#54616;&#45716; feature selection&#50640;&#49436; &#50976;&#50857;&#54620; &#48169;&#48277;&#51077;&#45768;&#45796;. &#51452;&#47196; &#47784;&#45944;&#51012; &#54617;&#49845;&#54616;&#44592;&#51204; &#52395; &#45800;&#44228;&#50640;&#49436; &#48320;&#49688;&#46308;&#51012; &#51228;&#44144;&#54616;&#45716; &#45936; &#49324;&#50857;&#54633;&#45768;&#45796;. &#52572;&#51333; &#47784;&#45944;&#50640;&#49436;&#45716; &#45824;&#47029; 10&#44060; &#45236;&#50808;&#51032; &#48320;&#49688;&#47484; &#49324;&#50857;&#54616;&#46020;&#47197; &#54633;&#45768;&#45796;. IV&#50752; WOE &#49888;&#50857;&#52292;&#47924;&#45733;&#47141;&#51060; &#44032;&#45733;&#54620;(good) &#44256;&#44061;&#44284; &#48520;&#44032;&#45733;&#54620;(bad) &#44256;&#44061;&#51012; &#50696;&#52769;&#54616;&#45716; &#47196;&#51648;&#49828;&#54001; &#54924;&#44480; &#47784;&#45944;&#47553;&#44284; &#48128;&#51217;&#54620; &#44288;&#44228;&#44032; &#51080;&#49845;&#45768;&#45796;. &#49888;&#50857; &#51221;&#48372; &#44288;&#47144;&#48516;&#50556;&#50640;&#49436;&#45716; good customer&#45716; &#48512;&#52292;&#47484; &#44058;&#51012; &#49688; &#51080;&#45716; &#44256;&#44061;, bad customer&#45716; &#48512;&#52292;&#47484; &#44058;&#51012; &#49688; &#50630;&#45716; &#44256;&#44061;&#51012; &#46907;&#54633;&#45768;&#45796;. &#51068;&#48152;&#51201;&#51004;&#47196; &#51060;&#50556;&#44592;&#54624; &#46412;&#45716; good customer&#45716; non-events&#47484; &#51032;&#48120;&#54616;&#44256; bad customer&#45716; events&#47484; &#51032;&#48120;&#54633;&#45768;&#45796;.\n",
    "\n",
    "> &#49888;&#50857; &#44288;&#47144; &#48516;&#50556;\n",
    "\n",
    "$\n",
    "\\begin{align}\n",
    "&\\text{WOE} = ln{\\frac{\\text{distribution of good}}{\\text{distribution of bad}}} \\\\\n",
    "&\\text{IV} = \\sum{(\\text{WOE} \\times (\\text{distribution of good} - \\text{distribution of bad}))}\\\\\n",
    "\\end{align}\n",
    "$\n",
    "\n",
    ">&#51068;&#48152;&#51201;\n",
    "\n",
    "$\n",
    "\\begin{align}\n",
    "&\\text{WOE} = ln{\\frac{\\text{distribution of non-events}}{\\text{distribution of events}}} \\\\\n",
    "&\\text{IV} = \\sum{(\\text{WOE} \\times (\\text{distribution of non-events} - \\text{distribution of events}))}\\\\\n",
    "\\end{align}\n",
    "$\n",
    "\n",
    "\n",
    "- &#44396;&#44036;&#51012; &#45208;&#45580; &#46412;, `pd.qcut`&#51012; &#49324;&#50857;&#54616;&#50688;&#49845;&#45768;&#45796;. bin&#51032; &#44079;&#49688;&#47564;&#53372; &#44050;&#51032; &#44396;&#44036;&#51012; &#45208;&#45572;&#44256; &#44050;&#51012; &#54644;&#45817; &#44396;&#44036;&#50640; mapping &#49884;&#53020;&#51469;&#45768;&#45796;.\n",
    "- Information value&#47484; &#52769;&#51221;&#54624; &#46412;, EOW&#44032; inf, -inf&#44032; &#45208;&#50732; &#49688; &#51080;&#49845;&#45768;&#45796;. &#48516;&#54252;&#44050;&#51060; 0&#51060; &#46112; &#49688; &#51080;&#44592; &#46412;&#47928;&#51077;&#45768;&#45796;. &#51060; &#46412;&#45716; EOW&#47484; 0&#51004;&#47196; &#48320;&#44221;&#54633;&#45768;&#45796;.\n",
    "\n",
    "\n",
    "|<center>Information Value</center>|<center>&#50696;&#52769;&#47141;</center>|\n",
    "|:-----------:|:------------------------------:|\n",
    "|<center>0 to 0.02</center>|<center>&#47924;&#51032;&#48120;</center>|\n",
    "|<center>0.02 to 0.1</center>|<center>&#45230;&#51008; &#50696;&#52769;</center>|\n",
    "|<center>0.1 to 0.3</center>|<center>&#51473;&#44036; &#50696;&#52769;</center>|\n",
    "|<center>0.3 to 0.5</center>|<center>&#44053;&#54620; &#50696;&#52769;</center>|\n",
    "|<center>0.5 to 1</center>|<center>&#45320;&#47924; &#44053;&#54620; &#50696;&#52769;(&#51032;&#49900;&#46104;&#45716; &#49688;&#52824;)</center>|"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "&#50696;&#51228;)\n",
    "\n",
    "<img src=\"./images/information_value.png\" style=\"height:400px width:400px\">"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_bin = 10  # &#51204;&#52404; &#45936;&#51060;&#53552;&#51032; &#50696;&#52769;&#47141;&#50640; &#54644;&#47484; &#44032;&#54616;&#51648; &#50506;&#45716; &#54620;&#50640;&#49436; &#44396;&#44036;&#51012; &#53580;&#49828;&#53944;&#54616;&#50688;&#49845;&#45768;&#45796;.\n",
    "def calc_iv(df, col, label, max_bin = max_bin):\n",
    "    \"\"\"IV helper function\"\"\"\n",
    "    bin_df = df[[col, label]].copy()\n",
    "    # Categorical column\n",
    "    if bin_df[col].dtype == 'object':\n",
    "        bin_df = bin_df.groupby(col)[label].agg(['count', 'sum'])\n",
    "    # Numerical column\n",
    "    else:\n",
    "        bin_df.loc[:, 'bins'] = pd.qcut(bin_df[col].rank(method='first'), max_bin)\n",
    "#         bin_df.loc[:, 'bins'] = pd.cut(bin_df[col], max_bin)\n",
    "        bin_df = bin_df.groupby('bins')[label].agg(['count', 'sum'])\n",
    "    \n",
    "    bin_df.columns = ['total', 'abuse']\n",
    "    bin_df['normal'] = bin_df['total'] - bin_df['abuse']\n",
    "    bin_df['normal_dist'] = bin_df['normal'] / sum(bin_df['normal'])\n",
    "    bin_df['abuse_dist'] = bin_df['abuse'] / sum(bin_df['abuse'])\n",
    "    bin_df['woe'] = np.log(bin_df['normal_dist'] / bin_df['abuse_dist'])\n",
    "    bin_df['iv'] = bin_df['woe'] * (bin_df['normal_dist'] - bin_df['abuse_dist'])\n",
    "    \n",
    "    bin_df.replace([np.inf, -np.inf], 0, inplace=True)\n",
    "    bin_df = bin_df[bin_df['total'] > 0]\n",
    "    iv_val = sum(filter(lambda x: x != float('inf'), bin_df['iv']))\n",
    "    \n",
    "    return bin_df, col, iv_val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>total</th>\n",
       "      <th>abuse</th>\n",
       "      <th>normal</th>\n",
       "      <th>normal_dist</th>\n",
       "      <th>abuse_dist</th>\n",
       "      <th>woe</th>\n",
       "      <th>iv</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Credit_history</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>A30</th>\n",
       "      <td>40</td>\n",
       "      <td>25</td>\n",
       "      <td>15</td>\n",
       "      <td>0.021429</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>-1.358123</td>\n",
       "      <td>0.084074</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>A31</th>\n",
       "      <td>49</td>\n",
       "      <td>28</td>\n",
       "      <td>21</td>\n",
       "      <td>0.030000</td>\n",
       "      <td>0.093333</td>\n",
       "      <td>-1.134980</td>\n",
       "      <td>0.071882</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>A32</th>\n",
       "      <td>530</td>\n",
       "      <td>169</td>\n",
       "      <td>361</td>\n",
       "      <td>0.515714</td>\n",
       "      <td>0.563333</td>\n",
       "      <td>-0.088319</td>\n",
       "      <td>0.004206</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>A33</th>\n",
       "      <td>88</td>\n",
       "      <td>28</td>\n",
       "      <td>60</td>\n",
       "      <td>0.085714</td>\n",
       "      <td>0.093333</td>\n",
       "      <td>-0.085158</td>\n",
       "      <td>0.000649</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>A34</th>\n",
       "      <td>293</td>\n",
       "      <td>50</td>\n",
       "      <td>243</td>\n",
       "      <td>0.347143</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>0.733741</td>\n",
       "      <td>0.132423</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                total  abuse  normal  normal_dist  abuse_dist       woe  \\\n",
       "Credit_history                                                            \n",
       "A30                40     25      15     0.021429    0.083333 -1.358123   \n",
       "A31                49     28      21     0.030000    0.093333 -1.134980   \n",
       "A32               530    169     361     0.515714    0.563333 -0.088319   \n",
       "A33                88     28      60     0.085714    0.093333 -0.085158   \n",
       "A34               293     50     243     0.347143    0.166667  0.733741   \n",
       "\n",
       "                      iv  \n",
       "Credit_history            \n",
       "A30             0.084074  \n",
       "A31             0.071882  \n",
       "A32             0.004206  \n",
       "A33             0.000649  \n",
       "A34             0.132423  "
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ch_df, ch, ch_i_val = calc_iv(dataset,'Credit_history', 'Target')\n",
    "ch_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "information value 0.2932335473908263\n"
     ]
    }
   ],
   "source": [
    "# &#51473;&#44036; &#51221;&#46020;&#51032; &#50696;&#52769; &#45733;&#47141;\n",
    "print('information value', ch_i_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>total</th>\n",
       "      <th>abuse</th>\n",
       "      <th>normal</th>\n",
       "      <th>normal_dist</th>\n",
       "      <th>abuse_dist</th>\n",
       "      <th>woe</th>\n",
       "      <th>iv</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bins</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>(0.999, 100.9]</th>\n",
       "      <td>100</td>\n",
       "      <td>11</td>\n",
       "      <td>89</td>\n",
       "      <td>0.127143</td>\n",
       "      <td>0.036667</td>\n",
       "      <td>1.243443</td>\n",
       "      <td>0.112502</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(100.9, 200.8]</th>\n",
       "      <td>100</td>\n",
       "      <td>20</td>\n",
       "      <td>80</td>\n",
       "      <td>0.114286</td>\n",
       "      <td>0.066667</td>\n",
       "      <td>0.538997</td>\n",
       "      <td>0.025667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(200.8, 300.7]</th>\n",
       "      <td>100</td>\n",
       "      <td>27</td>\n",
       "      <td>73</td>\n",
       "      <td>0.104286</td>\n",
       "      <td>0.090000</td>\n",
       "      <td>0.147325</td>\n",
       "      <td>0.002105</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(300.7, 400.6]</th>\n",
       "      <td>100</td>\n",
       "      <td>25</td>\n",
       "      <td>75</td>\n",
       "      <td>0.107143</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.251314</td>\n",
       "      <td>0.005984</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(400.6, 500.5]</th>\n",
       "      <td>100</td>\n",
       "      <td>26</td>\n",
       "      <td>74</td>\n",
       "      <td>0.105714</td>\n",
       "      <td>0.086667</td>\n",
       "      <td>0.198671</td>\n",
       "      <td>0.003784</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(500.5, 600.4]</th>\n",
       "      <td>100</td>\n",
       "      <td>38</td>\n",
       "      <td>62</td>\n",
       "      <td>0.088571</td>\n",
       "      <td>0.126667</td>\n",
       "      <td>-0.357750</td>\n",
       "      <td>0.013629</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(600.4, 700.3]</th>\n",
       "      <td>100</td>\n",
       "      <td>32</td>\n",
       "      <td>68</td>\n",
       "      <td>0.097143</td>\n",
       "      <td>0.106667</td>\n",
       "      <td>-0.093526</td>\n",
       "      <td>0.000891</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(700.3, 800.2]</th>\n",
       "      <td>100</td>\n",
       "      <td>31</td>\n",
       "      <td>69</td>\n",
       "      <td>0.098571</td>\n",
       "      <td>0.103333</td>\n",
       "      <td>-0.047179</td>\n",
       "      <td>0.000225</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(800.2, 900.1]</th>\n",
       "      <td>100</td>\n",
       "      <td>42</td>\n",
       "      <td>58</td>\n",
       "      <td>0.082857</td>\n",
       "      <td>0.140000</td>\n",
       "      <td>-0.524524</td>\n",
       "      <td>0.029973</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(900.1, 1000.0]</th>\n",
       "      <td>100</td>\n",
       "      <td>48</td>\n",
       "      <td>52</td>\n",
       "      <td>0.074286</td>\n",
       "      <td>0.160000</td>\n",
       "      <td>-0.767255</td>\n",
       "      <td>0.065765</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 total  abuse  normal  normal_dist  abuse_dist       woe  \\\n",
       "bins                                                                       \n",
       "(0.999, 100.9]     100     11      89     0.127143    0.036667  1.243443   \n",
       "(100.9, 200.8]     100     20      80     0.114286    0.066667  0.538997   \n",
       "(200.8, 300.7]     100     27      73     0.104286    0.090000  0.147325   \n",
       "(300.7, 400.6]     100     25      75     0.107143    0.083333  0.251314   \n",
       "(400.6, 500.5]     100     26      74     0.105714    0.086667  0.198671   \n",
       "(500.5, 600.4]     100     38      62     0.088571    0.126667 -0.357750   \n",
       "(600.4, 700.3]     100     32      68     0.097143    0.106667 -0.093526   \n",
       "(700.3, 800.2]     100     31      69     0.098571    0.103333 -0.047179   \n",
       "(800.2, 900.1]     100     42      58     0.082857    0.140000 -0.524524   \n",
       "(900.1, 1000.0]    100     48      52     0.074286    0.160000 -0.767255   \n",
       "\n",
       "                       iv  \n",
       "bins                       \n",
       "(0.999, 100.9]   0.112502  \n",
       "(100.9, 200.8]   0.025667  \n",
       "(200.8, 300.7]   0.002105  \n",
       "(300.7, 400.6]   0.005984  \n",
       "(400.6, 500.5]   0.003784  \n",
       "(500.5, 600.4]   0.013629  \n",
       "(600.4, 700.3]   0.000891  \n",
       "(700.3, 800.2]   0.000225  \n",
       "(800.2, 900.1]   0.029973  \n",
       "(900.1, 1000.0]  0.065765  "
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dim_df, dim, dim_i_val = calc_iv(dataset,'Duration_in_month', 'Target')\n",
    "dim_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.2605225223321392"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# &#45458;&#51008; &#50696;&#52769; &#45733;&#47141;\n",
    "dim_i_val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "# &#54632;&#49688;&#47484; &#47564;&#46308;&#50612;&#49436; &#51204;&#52404; iv&#47484; &#49332;&#54196;&#48372;&#51088;\n",
    "col_iv = {}\n",
    "for col in [idx for idx in dataset.columns.tolist()]:\n",
    "    if col == 'Target':\n",
    "        continue\n",
    "    _, col, iv = calc_iv(dataset, col, 'Target')\n",
    "    col_iv[col] = iv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Status_of_existing_checking_account': 0.6660115033513336,\n",
       " 'Duration_in_month': 0.2605225223321392,\n",
       " 'Credit_history': 0.2932335473908263,\n",
       " 'Purpose': 0.16919506567307832,\n",
       " 'Credit_amount': 0.11342803024552867,\n",
       " 'Savings_account_bonds': 0.19600955690422672,\n",
       " 'Present_employment_since': 0.086433631026641,\n",
       " 'Installment_rate_in_percentage_of_disposable_income': 0.061554683786294126,\n",
       " 'Personal_status_and_sex': 0.04467067763379073,\n",
       " 'Other_debtors_guarantors': 0.032019322019485055,\n",
       " 'Present_residence_since': 0.04874371881018562,\n",
       " 'Property': 0.11263826240979674,\n",
       " 'Age_in_years': 0.10267245670259074,\n",
       " 'Other_installment_plans': 0.057614541955647885,\n",
       " 'Housing': 0.08329343361549926,\n",
       " 'Number_of_existing_credits_at_this_bank': 0.09779114631307396,\n",
       " 'Job': 0.008762765707428294,\n",
       " 'Number_of_people_being_liable_to_provide_maintenance_for': 0.03408883520785682,\n",
       " 'Telephone': 0.0063776050286746735,\n",
       " 'foreign_worker': 0.04387741201028899,\n",
       " 'bin_var': 0.03408883520785682}"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "col_iv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Status_of_existing_checking_account', 0.6660115033513336),\n",
       " ('Credit_history', 0.2932335473908263),\n",
       " ('Duration_in_month', 0.2605225223321392),\n",
       " ('Savings_account_bonds', 0.19600955690422672),\n",
       " ('Purpose', 0.16919506567307832),\n",
       " ('Credit_amount', 0.11342803024552867),\n",
       " ('Property', 0.11263826240979674),\n",
       " ('Age_in_years', 0.10267245670259074),\n",
       " ('Number_of_existing_credits_at_this_bank', 0.09779114631307396),\n",
       " ('Present_employment_since', 0.086433631026641),\n",
       " ('Housing', 0.08329343361549926),\n",
       " ('Installment_rate_in_percentage_of_disposable_income', 0.061554683786294126),\n",
       " ('Other_installment_plans', 0.057614541955647885),\n",
       " ('Present_residence_since', 0.04874371881018562),\n",
       " ('Personal_status_and_sex', 0.04467067763379073),\n",
       " ('foreign_worker', 0.04387741201028899),\n",
       " ('Number_of_people_being_liable_to_provide_maintenance_for',\n",
       "  0.03408883520785682),\n",
       " ('bin_var', 0.03408883520785682),\n",
       " ('Other_debtors_guarantors', 0.032019322019485055),\n",
       " ('Job', 0.008762765707428294),\n",
       " ('Telephone', 0.0063776050286746735)]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import operator\n",
    "candidates = sorted(col_iv.items(), key=operator.itemgetter(1), reverse=True)\n",
    "display(candidates)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Status_of_existing_checking_account&#44032; &#44032;&#51109; &#44053;&#47141;&#54620; &#50696;&#52769; &#45733;&#47141;(&#51032;&#49900; &#46112; &#51221;&#46020;&#47196;)&#47484; &#44032;&#51276;&#44256;, &#45796;&#51020;&#51060; Credit_history&#51060;&#45796;. Telephone&#51008; &#44032;&#51109; &#45230;&#51008; &#50696;&#52769;&#47141;&#51012; &#44032;&#51648;&#44256; &#51080;&#45796;."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "&#49324;&#50857;&#54624; &#52972;&#47100;&#51008; &#48376;&#51064;&#51060; &#49440;&#53469;&#54644;&#50556;&#54620;&#45796;. &#45208;&#51008; &#50696;&#52769;&#51012; &#50948;&#54644;, &#47784;&#45944;&#51012; &#49373;&#49457;&#54616;&#44256;, &#53580;&#49828;&#53944;&#54616;&#44256; &#45796;&#49884; &#46028;&#50500;&#50752;&#49436; &#52972;&#47100;&#51012; &#49373;&#49457;&#54616;&#44256;&#47484; &#48152;&#48373;&#54620;&#45796;."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "&#50864;&#49440; &#49345;&#50948; 15&#44060;&#47564;&#51012; &#49440;&#53469;&#54616;&#51088;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "15"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "['Age_in_years',\n",
       " 'Credit_amount',\n",
       " 'Credit_history',\n",
       " 'Duration_in_month',\n",
       " 'Housing',\n",
       " 'Installment_rate_in_percentage_of_disposable_income',\n",
       " 'Number_of_existing_credits_at_this_bank',\n",
       " 'Other_installment_plans',\n",
       " 'Personal_status_and_sex',\n",
       " 'Present_employment_since',\n",
       " 'Present_residence_since',\n",
       " 'Property',\n",
       " 'Purpose',\n",
       " 'Savings_account_bonds',\n",
       " 'Status_of_existing_checking_account']"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# &#48516;&#49437;&#54616;&#47140;&#45716; feature&#51032; &#44079;&#49688;&#50752; &#50696;&#52769;&#47141;&#51032; trade-off&#47484; &#51312;&#51221;&#54616;&#50688;&#49845;&#45768;&#45796;.\n",
    "iv_cols = [key for key, iv in candidates if iv >= 0.044]\n",
    "display(len(iv_cols))\n",
    "display(sorted(iv_cols))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "# &#49345;&#49688;&#47196; &#49444;&#51221;&#54633;&#45768;&#45796;.\n",
    "IV_COL = iv_cols[:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Information Value&#50640; &#51032;&#54616;&#50668; &#49345;&#50948; 15&#44060;&#51032; feature&#47484; &#49440;&#51221;&#54616;&#50688;&#49845;&#45768;&#45796;. &#51204;&#52404; feature&#47484; &#49324;&#50857;&#54616;&#50668; &#47196;&#51648;&#49828;&#54001; &#54924;&#44480;&#47484; &#44396;&#54664;&#51012; &#46412;&#51032; accuracy&#50752; &#49888;&#50857;&#48520;&#47049;&#51088;&#50640; &#45824;&#54620; precision&#51032; &#44048;&#49548;&#54253;&#51060; &#51201;&#51004;&#47732;&#49436; (&#51204;&#52404; &#45936;&#51060;&#53552;&#49483;&#51012; &#44032;&#51109; &#51096; &#45824;&#48320;&#54644; &#51460; &#49688; &#51080;&#51004;&#47732;&#49436;) feature&#51032; &#49688;&#47484; &#51460;&#51060;&#45716; &#44284;&#51221;&#51012; &#47751; &#48264; &#53580;&#49828;&#53944; &#54616;&#47732;&#49436; &#50619;&#51008; &#44050;&#51077;&#45768;&#45796;.\n",
    "\n",
    "&#47676;&#51200;, IV&#50640;&#49436; &#49440;&#53469;&#46108; &#52972;&#47100;&#46308;&#47196; &#51060;&#47336;&#50612;&#51652; &#47784;&#45944;&#50640;&#49436;&#48512;&#53552; &#49884;&#51089;&#54633;&#45768;&#45796;."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Category column&#46308;&#51008; one hot vectore&#47196; &#48320;&#54872;&#54633;&#45768;&#45796;.\n",
    "cate_features = {}\n",
    "num_fetures = []\n",
    "for col in IV_COL:\n",
    "    if dataset[col].dtype == 'object':\n",
    "        cate_features[col] = pd.get_dummies(dataset[col], prefix=col)\n",
    "    else:\n",
    "        num_fetures.append(col)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['Status_of_existing_checking_account', 'Credit_history', 'Savings_account_bonds', 'Purpose', 'Property', 'Present_employment_since', 'Housing', 'Other_installment_plans', 'Personal_status_and_sex'])"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cate_features.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Status_of_existing_checking_account_A11</th>\n",
       "      <th>Status_of_existing_checking_account_A12</th>\n",
       "      <th>Status_of_existing_checking_account_A13</th>\n",
       "      <th>Status_of_existing_checking_account_A14</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Status_of_existing_checking_account_A11  \\\n",
       "0                                        1   \n",
       "1                                        0   \n",
       "2                                        0   \n",
       "3                                        1   \n",
       "4                                        1   \n",
       "\n",
       "   Status_of_existing_checking_account_A12  \\\n",
       "0                                        0   \n",
       "1                                        1   \n",
       "2                                        0   \n",
       "3                                        0   \n",
       "4                                        0   \n",
       "\n",
       "   Status_of_existing_checking_account_A13  \\\n",
       "0                                        0   \n",
       "1                                        0   \n",
       "2                                        0   \n",
       "3                                        0   \n",
       "4                                        0   \n",
       "\n",
       "   Status_of_existing_checking_account_A14  \n",
       "0                                        0  \n",
       "1                                        0  \n",
       "2                                        1  \n",
       "3                                        0  \n",
       "4                                        0  "
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cate_features['Status_of_existing_checking_account'].head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "&#45908;&#48120; &#48320;&#49688;&#47484; &#49373;&#49457;&#54616;&#44172; &#46104;&#47732;, &#51088;&#50976;&#46020;&#45716; &#45908;&#48120; &#48320;&#49688;&#44032; &#44032;&#51656; &#49688; &#51080;&#45716; unique &#44050;&#51060; $n$ &#51060;&#46972;&#44256; &#54664;&#51012;&#46412; $n-1$&#51060; &#46104;&#48064;&#47196;, &#44033; &#48320;&#49688;&#50640;&#49436; &#45908;&#48120;&#48320;&#49688; &#52629;&#51012; &#54620;&#44060;&#50473; &#51228;&#44144;&#54644;&#51456;&#45796;. &#51228;&#44144;&#54616;&#51648; &#50506;&#51004;&#47732; `NaN`&#44050;&#51060; &#48156;&#49373;&#54633;&#45768;&#45796;.\n",
    "\n",
    "&#54980;&#51652; &#51228;&#44144;&#48277;&#51012; &#53685;&#54644; &#47924;&#51032;&#48120;&#54620; &#48320;&#49688;&#47484; &#54616;&#45208;&#50473; &#51228;&#44144;&#54644; &#45208;&#44032;&#44592; &#50948;&#54644; &#48324;&#46020;&#51032; &#47532;&#49828;&#53944;&#47484; &#54616;&#45208; &#49373;&#49457;&#54633;&#45768;&#45796;. &#51060; &#47532;&#49828;&#53944;&#45716; &#49892;&#54665;&#51060; &#48152;&#48373;&#46112; &#46412;&#47560;&#45796; &#44032;&#51109; &#47924;&#51032;&#48120;&#54620; &#48320;&#49688;(p-value&#44032; &#53360; &#48320;&#49688;)&#50752; &#45796;&#51473;&#44277;&#49440;&#49457;&#51012; &#44054;&#45716; &#48320;&#49688;&#47484; &#51228;&#44144; &#54620;&#45796;."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [],
   "source": [
    "removed_features = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [],
   "source": [
    "for col, dummies in cate_features.items():\n",
    "    dropped_col = dummies.columns[-1]\n",
    "    removed_features.append(dropped_col)\n",
    "    cate_features[col] = dummies.drop(dropped_col, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Status_of_existing_checking_account_A14',\n",
       " 'Credit_history_A34',\n",
       " 'Savings_account_bonds_A65',\n",
       " 'Purpose_A49',\n",
       " 'Property_A124',\n",
       " 'Present_employment_since_A75',\n",
       " 'Housing_A153',\n",
       " 'Other_installment_plans_A143',\n",
       " 'Personal_status_and_sex_A94']"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "removed_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Status_of_existing_checking_account_A11</th>\n",
       "      <th>Status_of_existing_checking_account_A12</th>\n",
       "      <th>Status_of_existing_checking_account_A13</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Status_of_existing_checking_account_A11  \\\n",
       "0                                        1   \n",
       "1                                        0   \n",
       "2                                        0   \n",
       "3                                        1   \n",
       "4                                        1   \n",
       "\n",
       "   Status_of_existing_checking_account_A12  \\\n",
       "0                                        0   \n",
       "1                                        1   \n",
       "2                                        0   \n",
       "3                                        0   \n",
       "4                                        0   \n",
       "\n",
       "   Status_of_existing_checking_account_A13  \n",
       "0                                        0  \n",
       "1                                        0  \n",
       "2                                        0  \n",
       "3                                        0  \n",
       "4                                        0  "
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cate_features['Status_of_existing_checking_account'].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_dataset = dataset[num_fetures]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "for col, df in cate_features.items():\n",
    "    final_dataset = pd.concat([final_dataset, df], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1000, 40)"
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_dataset.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Duration_in_month</th>\n",
       "      <th>Credit_amount</th>\n",
       "      <th>Age_in_years</th>\n",
       "      <th>Number_of_existing_credits_at_this_bank</th>\n",
       "      <th>Installment_rate_in_percentage_of_disposable_income</th>\n",
       "      <th>Present_residence_since</th>\n",
       "      <th>Status_of_existing_checking_account_A11</th>\n",
       "      <th>Status_of_existing_checking_account_A12</th>\n",
       "      <th>Status_of_existing_checking_account_A13</th>\n",
       "      <th>Credit_history_A30</th>\n",
       "      <th>Credit_history_A31</th>\n",
       "      <th>Credit_history_A32</th>\n",
       "      <th>Credit_history_A33</th>\n",
       "      <th>Savings_account_bonds_A61</th>\n",
       "      <th>Savings_account_bonds_A62</th>\n",
       "      <th>Savings_account_bonds_A63</th>\n",
       "      <th>Savings_account_bonds_A64</th>\n",
       "      <th>Purpose_A40</th>\n",
       "      <th>Purpose_A41</th>\n",
       "      <th>Purpose_A410</th>\n",
       "      <th>Purpose_A42</th>\n",
       "      <th>Purpose_A43</th>\n",
       "      <th>Purpose_A44</th>\n",
       "      <th>Purpose_A45</th>\n",
       "      <th>Purpose_A46</th>\n",
       "      <th>Purpose_A48</th>\n",
       "      <th>Property_A121</th>\n",
       "      <th>Property_A122</th>\n",
       "      <th>Property_A123</th>\n",
       "      <th>Present_employment_since_A71</th>\n",
       "      <th>Present_employment_since_A72</th>\n",
       "      <th>Present_employment_since_A73</th>\n",
       "      <th>Present_employment_since_A74</th>\n",
       "      <th>Housing_A151</th>\n",
       "      <th>Housing_A152</th>\n",
       "      <th>Other_installment_plans_A141</th>\n",
       "      <th>Other_installment_plans_A142</th>\n",
       "      <th>Personal_status_and_sex_A91</th>\n",
       "      <th>Personal_status_and_sex_A92</th>\n",
       "      <th>Personal_status_and_sex_A93</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6</td>\n",
       "      <td>1169</td>\n",
       "      <td>67</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>48</td>\n",
       "      <td>5951</td>\n",
       "      <td>22</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>12</td>\n",
       "      <td>2096</td>\n",
       "      <td>49</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>42</td>\n",
       "      <td>7882</td>\n",
       "      <td>45</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>24</td>\n",
       "      <td>4870</td>\n",
       "      <td>53</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Duration_in_month  Credit_amount  Age_in_years  \\\n",
       "0                  6           1169            67   \n",
       "1                 48           5951            22   \n",
       "2                 12           2096            49   \n",
       "3                 42           7882            45   \n",
       "4                 24           4870            53   \n",
       "\n",
       "   Number_of_existing_credits_at_this_bank  \\\n",
       "0                                        2   \n",
       "1                                        1   \n",
       "2                                        1   \n",
       "3                                        1   \n",
       "4                                        2   \n",
       "\n",
       "   Installment_rate_in_percentage_of_disposable_income  \\\n",
       "0                                                  4     \n",
       "1                                                  2     \n",
       "2                                                  2     \n",
       "3                                                  2     \n",
       "4                                                  3     \n",
       "\n",
       "   Present_residence_since  Status_of_existing_checking_account_A11  \\\n",
       "0                        4                                        1   \n",
       "1                        2                                        0   \n",
       "2                        3                                        0   \n",
       "3                        4                                        1   \n",
       "4                        4                                        1   \n",
       "\n",
       "   Status_of_existing_checking_account_A12  \\\n",
       "0                                        0   \n",
       "1                                        1   \n",
       "2                                        0   \n",
       "3                                        0   \n",
       "4                                        0   \n",
       "\n",
       "   Status_of_existing_checking_account_A13  Credit_history_A30  \\\n",
       "0                                        0                   0   \n",
       "1                                        0                   0   \n",
       "2                                        0                   0   \n",
       "3                                        0                   0   \n",
       "4                                        0                   0   \n",
       "\n",
       "   Credit_history_A31  Credit_history_A32  Credit_history_A33  \\\n",
       "0                   0                   0                   0   \n",
       "1                   0                   1                   0   \n",
       "2                   0                   0                   0   \n",
       "3                   0                   1                   0   \n",
       "4                   0                   0                   1   \n",
       "\n",
       "   Savings_account_bonds_A61  Savings_account_bonds_A62  \\\n",
       "0                          0                          0   \n",
       "1                          1                          0   \n",
       "2                          1                          0   \n",
       "3                          1                          0   \n",
       "4                          1                          0   \n",
       "\n",
       "   Savings_account_bonds_A63  Savings_account_bonds_A64  Purpose_A40  \\\n",
       "0                          0                          0            0   \n",
       "1                          0                          0            0   \n",
       "2                          0                          0            0   \n",
       "3                          0                          0            0   \n",
       "4                          0                          0            1   \n",
       "\n",
       "   Purpose_A41  Purpose_A410  Purpose_A42  Purpose_A43  Purpose_A44  \\\n",
       "0            0             0            0            1            0   \n",
       "1            0             0            0            1            0   \n",
       "2            0             0            0            0            0   \n",
       "3            0             0            1            0            0   \n",
       "4            0             0            0            0            0   \n",
       "\n",
       "   Purpose_A45  Purpose_A46  Purpose_A48  Property_A121  Property_A122  \\\n",
       "0            0            0            0              1              0   \n",
       "1            0            0            0              1              0   \n",
       "2            0            1            0              1              0   \n",
       "3            0            0            0              0              1   \n",
       "4            0            0            0              0              0   \n",
       "\n",
       "   Property_A123  Present_employment_since_A71  Present_employment_since_A72  \\\n",
       "0              0                             0                             0   \n",
       "1              0                             0                             0   \n",
       "2              0                             0                             0   \n",
       "3              0                             0                             0   \n",
       "4              0                             0                             0   \n",
       "\n",
       "   Present_employment_since_A73  Present_employment_since_A74  Housing_A151  \\\n",
       "0                             0                             0             0   \n",
       "1                             1                             0             0   \n",
       "2                             0                             1             0   \n",
       "3                             0                             1             0   \n",
       "4                             1                             0             0   \n",
       "\n",
       "   Housing_A152  Other_installment_plans_A141  Other_installment_plans_A142  \\\n",
       "0             1                             0                             0   \n",
       "1             1                             0                             0   \n",
       "2             1                             0                             0   \n",
       "3             0                             0                             0   \n",
       "4             0                             0                             0   \n",
       "\n",
       "   Personal_status_and_sex_A91  Personal_status_and_sex_A92  \\\n",
       "0                            0                            0   \n",
       "1                            0                            1   \n",
       "2                            0                            0   \n",
       "3                            0                            0   \n",
       "4                            0                            0   \n",
       "\n",
       "   Personal_status_and_sex_A93  \n",
       "0                            1  \n",
       "1                            0  \n",
       "2                            1  \n",
       "3                            1  \n",
       "4                            1  "
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_dataset.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Training set, test set&#51012; &#45208;&#45589;&#45768;&#45796;. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_x, test_x, train_y, test_y = \\\n",
    "    train_test_split( \\\n",
    "        final_dataset, dataset['Target'], test_size=0.2, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_set (800, 40)\n",
      "test_set (200, 40)\n"
     ]
    }
   ],
   "source": [
    "print('train_set', train_x.shape)\n",
    "print('test_set', test_x.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Statsmodels&#47484; &#51060;&#50857;&#54616;&#50668; &#47196;&#51648;&#49828;&#54001; &#54924;&#44480;&#47484; &#46028;&#47549;&#45768;&#45796;."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/statsmodels/compat/pandas.py:56: FutureWarning: The pandas.core.datetools module is deprecated and will be removed in a future version. Please use the pandas.tseries module instead.\n",
      "  from pandas.core import datetools\n"
     ]
    }
   ],
   "source": [
    "import statsmodels.api as sm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimization terminated successfully.\n",
      "         Current function value: 0.460159\n",
      "         Iterations 7\n"
     ]
    }
   ],
   "source": [
    "logistic_model = sm.Logit(\n",
    "    train_y,\n",
    "    sm.add_constant(train_x)\n",
    ").fit()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### &#49457;&#45733; &#52769;&#51221;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import auc\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import classification_report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_pred = pd.DataFrame({\n",
    "    'probs': logistic_model.predict(sm.add_constant(train_x)),\n",
    "    'class': train_y\n",
    "})\n",
    "\n",
    "train_pred['y_pred'] = 0\n",
    "train_pred.loc[train_pred['probs'] > 0.5, 'y_pred'] = 1\n",
    "\n",
    "# Test prediction\n",
    "test_pred = pd.DataFrame({\n",
    "    'probs': logistic_model.predict(sm.add_constant(test_x)),\n",
    "    'class': test_y\n",
    "})\n",
    "\n",
    "test_pred['y_pred'] = 0\n",
    "test_pred.loc[test_pred['probs'] > 0.5, 'y_pred'] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Training Confusion matrix:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>Actual</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>All</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Predict</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>503</td>\n",
       "      <td>120</td>\n",
       "      <td>623</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>56</td>\n",
       "      <td>121</td>\n",
       "      <td>177</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>All</th>\n",
       "      <td>559</td>\n",
       "      <td>241</td>\n",
       "      <td>800</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Actual     0    1  All\n",
       "Predict               \n",
       "0        503  120  623\n",
       "1         56  121  177\n",
       "All      559  241  800"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Training Accuracy:  0.78\n",
      "\n",
      "Training classification report:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "          0     0.8074    0.8998    0.8511       559\n",
      "          1     0.6836    0.5021    0.5789       241\n",
      "\n",
      "avg / total     0.7701    0.7800    0.7691       800\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print('\\nTraining Confusion matrix:')\n",
    "display(pd.crosstab(train_pred['y_pred'], train_pred['class'], rownames=['Predict'], colnames=['Actual'], margins=True))\n",
    "print('\\nTraining Accuracy: ', round(accuracy_score(train_pred['class'], train_pred['y_pred']), 4))\n",
    "print('\\nTraining classification report:\\n', classification_report(train_pred['class'], train_pred['y_pred'], digits=4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Confusion matrix:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>Actual</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>All</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Predict</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>123</td>\n",
       "      <td>27</td>\n",
       "      <td>150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>18</td>\n",
       "      <td>32</td>\n",
       "      <td>50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>All</th>\n",
       "      <td>141</td>\n",
       "      <td>59</td>\n",
       "      <td>200</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Actual     0   1  All\n",
       "Predict              \n",
       "0        123  27  150\n",
       "1         18  32   50\n",
       "All      141  59  200"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Test Accuracy:  0.775\n",
      "\n",
      "Test classification report:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "          0     0.8200    0.8723    0.8454       141\n",
      "          1     0.6400    0.5424    0.5872        59\n",
      "\n",
      "avg / total     0.7669    0.7750    0.7692       200\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print('Test Confusion matrix:')\n",
    "display(pd.crosstab(test_pred['y_pred'], test_pred['class'], rownames=['Predict'], colnames=['Actual'], margins=True))\n",
    "print('\\nTest Accuracy: ', round(accuracy_score(test_pred['class'], test_pred['y_pred']), 4))\n",
    "print('\\nTest classification report:\\n', classification_report(test_pred['class'], test_pred['y_pred'], digits=4))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. &#54980;&#51652;&#51228;&#44144;&#48277;\n",
    " - &#44032;&#51109; &#47924;&#51032;&#48120;&#54620; &#48320;&#49688; (p-value)\n",
    " - &#45796;&#51473;&#44277;&#49440;&#49457;&#51012; &#44054;&#45716; &#48320;&#49688; (Variance Inflation Factor)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### (A) P-value\n",
    "P-value&#44032; &#44032;&#51109; &#45458;&#51008; &#48320;&#49688;&#47484; &#51228;&#44144;&#54633;&#45768;&#45796;. &#50668;&#44592;&#49436; P-value&#45716; &#44033; &#46021;&#47549;&#48320;&#49688;&#51032; &#50976;&#51032;&#49457; &#44160;&#51221;(t-test)&#50640;&#49436; &#50619;&#50612;&#51652; &#44050;&#51077;&#45768;&#45796;."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [],
   "source": [
    "unnecesarries = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "        <td>Model:</td>              <td>Logit</td>       <td>No. Iterations:</td>   <td>7.0000</td>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <td>Dependent Variable:</td>      <td>Target</td>      <td>Pseudo R-squared:</td>   <td>0.248</td>  \n",
       "</tr>\n",
       "<tr>\n",
       "         <td>Date:</td>        <td>2019-02-25 22:10</td>       <td>AIC:</td>        <td>818.2538</td> \n",
       "</tr>\n",
       "<tr>\n",
       "   <td>No. Observations:</td>         <td>800</td>             <td>BIC:</td>        <td>1010.3229</td>\n",
       "</tr>\n",
       "<tr>\n",
       "       <td>Df Model:</td>             <td>40</td>         <td>Log-Likelihood:</td>   <td>-368.13</td> \n",
       "</tr>\n",
       "<tr>\n",
       "     <td>Df Residuals:</td>           <td>759</td>           <td>LL-Null:</td>       <td>-489.54</td> \n",
       "</tr>\n",
       "<tr>\n",
       "      <td>Converged:</td>           <td>1.0000</td>           <td>Scale:</td>        <td>1.0000</td>  \n",
       "</tr>\n",
       "</table>\n",
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "                           <td></td>                            <th>Coef.</th>  <th>Std.Err.</th>    <th>z</th>     <th>P>|z|</th> <th>[0.025</th>  <th>0.975]</th> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>const</th>                                               <td>-3.6675</td>  <td>1.0471</td>  <td>-3.5026</td> <td>0.0005</td> <td>-5.7198</td> <td>-1.6153</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Duration_in_month</th>                                   <td>0.0230</td>   <td>0.0101</td>  <td>2.2759</td>  <td>0.0228</td> <td>0.0032</td>  <td>0.0429</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Credit_amount</th>                                       <td>0.0001</td>   <td>0.0000</td>  <td>2.4648</td>  <td>0.0137</td> <td>0.0000</td>  <td>0.0002</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Age_in_years</th>                                        <td>-0.0315</td>  <td>0.0104</td>  <td>-3.0316</td> <td>0.0024</td> <td>-0.0519</td> <td>-0.0111</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Number_of_existing_credits_at_this_bank</th>             <td>0.3461</td>   <td>0.2075</td>  <td>1.6682</td>  <td>0.0953</td> <td>-0.0605</td> <td>0.7528</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Installment_rate_in_percentage_of_disposable_income</th> <td>0.3172</td>   <td>0.0945</td>  <td>3.3582</td>  <td>0.0008</td> <td>0.1321</td>  <td>0.5024</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Present_residence_since</th>                             <td>0.0320</td>   <td>0.0948</td>  <td>0.3373</td>  <td>0.7359</td> <td>-0.1538</td> <td>0.2178</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Status_of_existing_checking_account_A11</th>             <td>1.5522</td>   <td>0.2525</td>  <td>6.1468</td>  <td>0.0000</td> <td>1.0573</td>  <td>2.0472</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Status_of_existing_checking_account_A12</th>             <td>1.1125</td>   <td>0.2474</td>  <td>4.4969</td>  <td>0.0000</td> <td>0.6276</td>  <td>1.5974</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Status_of_existing_checking_account_A13</th>             <td>0.5329</td>   <td>0.4364</td>  <td>1.2211</td>  <td>0.2220</td> <td>-0.3224</td> <td>1.3881</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Credit_history_A30</th>                                  <td>1.5371</td>   <td>0.4723</td>  <td>3.2545</td>  <td>0.0011</td> <td>0.6114</td>  <td>2.4628</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Credit_history_A31</th>                                  <td>1.4798</td>   <td>0.4815</td>  <td>3.0734</td>  <td>0.0021</td> <td>0.5361</td>  <td>2.4235</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Credit_history_A32</th>                                  <td>0.8741</td>   <td>0.2821</td>  <td>3.0988</td>  <td>0.0019</td> <td>0.3212</td>  <td>1.4269</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Credit_history_A33</th>                                  <td>0.5364</td>   <td>0.3691</td>  <td>1.4532</td>  <td>0.1462</td> <td>-0.1871</td> <td>1.2599</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Savings_account_bonds_A61</th>                           <td>0.6851</td>   <td>0.2741</td>  <td>2.4997</td>  <td>0.0124</td> <td>0.1479</td>  <td>1.2224</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Savings_account_bonds_A62</th>                           <td>0.5970</td>   <td>0.3698</td>  <td>1.6144</td>  <td>0.1064</td> <td>-0.1278</td> <td>1.3217</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Savings_account_bonds_A63</th>                           <td>0.2964</td>   <td>0.4700</td>  <td>0.6305</td>  <td>0.5284</td> <td>-0.6249</td> <td>1.2176</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Savings_account_bonds_A64</th>                           <td>-0.2078</td>  <td>0.5857</td>  <td>-0.3547</td> <td>0.7228</td> <td>-1.3558</td> <td>0.9402</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Purpose_A40</th>                                         <td>0.6410</td>   <td>0.3599</td>  <td>1.7810</td>  <td>0.0749</td> <td>-0.0644</td> <td>1.3464</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Purpose_A41</th>                                         <td>-0.9585</td>  <td>0.4654</td>  <td>-2.0593</td> <td>0.0395</td> <td>-1.8707</td> <td>-0.0462</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Purpose_A410</th>                                        <td>-0.6272</td>  <td>0.8378</td>  <td>-0.7486</td> <td>0.4541</td> <td>-2.2693</td> <td>1.0149</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Purpose_A42</th>                                         <td>-0.2957</td>  <td>0.3846</td>  <td>-0.7686</td> <td>0.4421</td> <td>-1.0496</td> <td>0.4582</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Purpose_A43</th>                                         <td>-0.3352</td>  <td>0.3618</td>  <td>-0.9263</td> <td>0.3543</td> <td>-1.0443</td> <td>0.3740</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Purpose_A44</th>                                         <td>0.1854</td>   <td>0.8089</td>  <td>0.2292</td>  <td>0.8187</td> <td>-1.4000</td> <td>1.7709</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Purpose_A45</th>                                         <td>0.2867</td>   <td>0.6258</td>  <td>0.4581</td>  <td>0.6469</td> <td>-0.9398</td> <td>1.5131</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Purpose_A46</th>                                         <td>0.9372</td>   <td>0.5224</td>  <td>1.7941</td>  <td>0.0728</td> <td>-0.0867</td> <td>1.9611</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Purpose_A48</th>                                         <td>-1.2412</td>  <td>1.2458</td>  <td>-0.9964</td> <td>0.3191</td> <td>-3.6829</td> <td>1.2005</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Property_A121</th>                                       <td>-0.6366</td>  <td>0.4530</td>  <td>-1.4053</td> <td>0.1599</td> <td>-1.5244</td> <td>0.2513</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Property_A122</th>                                       <td>-0.2833</td>  <td>0.4461</td>  <td>-0.6349</td> <td>0.5255</td> <td>-1.1576</td> <td>0.5911</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Property_A123</th>                                       <td>-0.2635</td>  <td>0.4302</td>  <td>-0.6126</td> <td>0.5401</td> <td>-1.1067</td> <td>0.5796</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Present_employment_since_A71</th>                        <td>0.2897</td>   <td>0.4022</td>  <td>0.7203</td>  <td>0.4713</td> <td>-0.4986</td> <td>1.0781</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Present_employment_since_A72</th>                        <td>0.1534</td>   <td>0.3273</td>  <td>0.4686</td>  <td>0.6394</td> <td>-0.4881</td> <td>0.7948</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Present_employment_since_A73</th>                        <td>0.1896</td>   <td>0.2776</td>  <td>0.6830</td>  <td>0.4946</td> <td>-0.3545</td> <td>0.7337</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Present_employment_since_A74</th>                        <td>-0.3076</td>  <td>0.3278</td>  <td>-0.9385</td> <td>0.3480</td> <td>-0.9500</td> <td>0.3348</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Housing_A151</th>                                        <td>0.2009</td>   <td>0.5064</td>  <td>0.3967</td>  <td>0.6916</td> <td>-0.7916</td> <td>1.1934</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Housing_A152</th>                                        <td>-0.3416</td>  <td>0.4803</td>  <td>-0.7112</td> <td>0.4770</td> <td>-1.2829</td> <td>0.5997</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Other_installment_plans_A141</th>                        <td>0.7211</td>   <td>0.2690</td>  <td>2.6809</td>  <td>0.0073</td> <td>0.1939</td>  <td>1.2483</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Other_installment_plans_A142</th>                        <td>0.5466</td>   <td>0.4384</td>  <td>1.2467</td>  <td>0.2125</td> <td>-0.3127</td> <td>1.4059</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Personal_status_and_sex_A91</th>                         <td>0.4998</td>   <td>0.5226</td>  <td>0.9563</td>  <td>0.3389</td> <td>-0.5245</td> <td>1.5241</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Personal_status_and_sex_A92</th>                         <td>0.0181</td>   <td>0.3437</td>  <td>0.0527</td>  <td>0.9580</td> <td>-0.6555</td> <td>0.6918</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Personal_status_and_sex_A93</th>                         <td>-0.4830</td>  <td>0.3432</td>  <td>-1.4074</td> <td>0.1593</td> <td>-1.1558</td> <td>0.1897</td> \n",
       "</tr>\n",
       "</table>"
      ],
      "text/plain": [
       "<class 'statsmodels.iolib.summary2.Summary'>\n",
       "\"\"\"\n",
       "                                          Results: Logit\n",
       "===================================================================================================\n",
       "Model:                           Logit                       No. Iterations:              7.0000   \n",
       "Dependent Variable:              Target                      Pseudo R-squared:            0.248    \n",
       "Date:                            2019-02-25 22:10            AIC:                         818.2538 \n",
       "No. Observations:                800                         BIC:                         1010.3229\n",
       "Df Model:                        40                          Log-Likelihood:              -368.13  \n",
       "Df Residuals:                    759                         LL-Null:                     -489.54  \n",
       "Converged:                       1.0000                      Scale:                       1.0000   \n",
       "---------------------------------------------------------------------------------------------------\n",
       "                                                     Coef.  Std.Err.    z    P>|z|   [0.025  0.975]\n",
       "---------------------------------------------------------------------------------------------------\n",
       "const                                               -3.6675   1.0471 -3.5026 0.0005 -5.7198 -1.6153\n",
       "Duration_in_month                                    0.0230   0.0101  2.2759 0.0228  0.0032  0.0429\n",
       "Credit_amount                                        0.0001   0.0000  2.4648 0.0137  0.0000  0.0002\n",
       "Age_in_years                                        -0.0315   0.0104 -3.0316 0.0024 -0.0519 -0.0111\n",
       "Number_of_existing_credits_at_this_bank              0.3461   0.2075  1.6682 0.0953 -0.0605  0.7528\n",
       "Installment_rate_in_percentage_of_disposable_income  0.3172   0.0945  3.3582 0.0008  0.1321  0.5024\n",
       "Present_residence_since                              0.0320   0.0948  0.3373 0.7359 -0.1538  0.2178\n",
       "Status_of_existing_checking_account_A11              1.5522   0.2525  6.1468 0.0000  1.0573  2.0472\n",
       "Status_of_existing_checking_account_A12              1.1125   0.2474  4.4969 0.0000  0.6276  1.5974\n",
       "Status_of_existing_checking_account_A13              0.5329   0.4364  1.2211 0.2220 -0.3224  1.3881\n",
       "Credit_history_A30                                   1.5371   0.4723  3.2545 0.0011  0.6114  2.4628\n",
       "Credit_history_A31                                   1.4798   0.4815  3.0734 0.0021  0.5361  2.4235\n",
       "Credit_history_A32                                   0.8741   0.2821  3.0988 0.0019  0.3212  1.4269\n",
       "Credit_history_A33                                   0.5364   0.3691  1.4532 0.1462 -0.1871  1.2599\n",
       "Savings_account_bonds_A61                            0.6851   0.2741  2.4997 0.0124  0.1479  1.2224\n",
       "Savings_account_bonds_A62                            0.5970   0.3698  1.6144 0.1064 -0.1278  1.3217\n",
       "Savings_account_bonds_A63                            0.2964   0.4700  0.6305 0.5284 -0.6249  1.2176\n",
       "Savings_account_bonds_A64                           -0.2078   0.5857 -0.3547 0.7228 -1.3558  0.9402\n",
       "Purpose_A40                                          0.6410   0.3599  1.7810 0.0749 -0.0644  1.3464\n",
       "Purpose_A41                                         -0.9585   0.4654 -2.0593 0.0395 -1.8707 -0.0462\n",
       "Purpose_A410                                        -0.6272   0.8378 -0.7486 0.4541 -2.2693  1.0149\n",
       "Purpose_A42                                         -0.2957   0.3846 -0.7686 0.4421 -1.0496  0.4582\n",
       "Purpose_A43                                         -0.3352   0.3618 -0.9263 0.3543 -1.0443  0.3740\n",
       "Purpose_A44                                          0.1854   0.8089  0.2292 0.8187 -1.4000  1.7709\n",
       "Purpose_A45                                          0.2867   0.6258  0.4581 0.6469 -0.9398  1.5131\n",
       "Purpose_A46                                          0.9372   0.5224  1.7941 0.0728 -0.0867  1.9611\n",
       "Purpose_A48                                         -1.2412   1.2458 -0.9964 0.3191 -3.6829  1.2005\n",
       "Property_A121                                       -0.6366   0.4530 -1.4053 0.1599 -1.5244  0.2513\n",
       "Property_A122                                       -0.2833   0.4461 -0.6349 0.5255 -1.1576  0.5911\n",
       "Property_A123                                       -0.2635   0.4302 -0.6126 0.5401 -1.1067  0.5796\n",
       "Present_employment_since_A71                         0.2897   0.4022  0.7203 0.4713 -0.4986  1.0781\n",
       "Present_employment_since_A72                         0.1534   0.3273  0.4686 0.6394 -0.4881  0.7948\n",
       "Present_employment_since_A73                         0.1896   0.2776  0.6830 0.4946 -0.3545  0.7337\n",
       "Present_employment_since_A74                        -0.3076   0.3278 -0.9385 0.3480 -0.9500  0.3348\n",
       "Housing_A151                                         0.2009   0.5064  0.3967 0.6916 -0.7916  1.1934\n",
       "Housing_A152                                        -0.3416   0.4803 -0.7112 0.4770 -1.2829  0.5997\n",
       "Other_installment_plans_A141                         0.7211   0.2690  2.6809 0.0073  0.1939  1.2483\n",
       "Other_installment_plans_A142                         0.5466   0.4384  1.2467 0.2125 -0.3127  1.4059\n",
       "Personal_status_and_sex_A91                          0.4998   0.5226  0.9563 0.3389 -0.5245  1.5241\n",
       "Personal_status_and_sex_A92                          0.0181   0.3437  0.0527 0.9580 -0.6555  0.6918\n",
       "Personal_status_and_sex_A93                         -0.4830   0.3432 -1.4074 0.1593 -1.1558  0.1897\n",
       "===================================================================================================\n",
       "\n",
       "\"\"\""
      ]
     },
     "execution_count": 180,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logistic_model.summary2()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "&#44032;&#51109; &#47924;&#51032;&#48120;&#54620; &#48320;&#49688;&#45716; P&#44050;&#51060; 0.9580&#51064; Personal_status_and_sex_A92&#51060;&#45796;."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "&#52404;&#53356;&#54616;&#44256; &#48320;&#49688;&#47484; 1&#44060;&#50473; &#51228;&#44144;&#54616;&#44256;&#47484; &#48152;&#48373;&#54644;&#50556;&#54620;&#45796;."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Help &#54632;&#49688;\n",
    "def get_unrelated_cols(model, pvalue):\n",
    "    cols = model.pvalues[model.pvalues >= pvalue].keys().tolist()\n",
    "    print(len(cols))\n",
    "    print(cols)\n",
    "    \n",
    "    return cols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n",
      "['Personal_status_and_sex_A92']\n"
     ]
    }
   ],
   "source": [
    "unrelated_cols = get_unrelated_cols(logistic_model, 0.8187)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Personal_status_and_sex_A92&#51012; &#49325;&#51228;&#54620;&#45796;."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [],
   "source": [
    "unnecesarries.append('Personal_status_and_sex_A92')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### (B) &#45796;&#51473;&#44277;&#49440;&#49457;\n",
    "&#53580;&#49828;&#53944;&#50752; &#49345;&#44288;&#50630;&#45796;&#44256; &#54032;&#45800;&#46104;&#45716; &#48320;&#49688;&#46308;&#51012; &#51228;&#44144;&#54620;&#54980;&#50640; &#45796;&#51473;&#44277;&#49440;&#49457;&#51012; &#52404;&#53356;&#54633;&#45768;&#45796;. **&#45796;&#51473;&#44277;&#49440;&#49457;(multicollinearity)** &#51060;&#46976; &#46021;&#47549;&#48320;&#49688;&#46308;&#44036;&#51032; &#49440;&#54805;&#44288;&#44228;&#44032; &#51316;&#51116;&#54616;&#45716; &#44163;&#51012; &#45208;&#53440;&#45253;&#45768;&#45796;. &#46021;&#47549;&#48320;&#49688; &#51204;&#52404;&#51032; set&#50640;&#49436; &#54616;&#45208;&#51032; &#46021;&#47549;&#48320;&#49688;&#47484; &#44264;&#46972; &#51333;&#49549;&#48320;&#49688;&#47196; &#48372;&#44256;, &#44536; &#46021;&#47549;&#48320;&#49688;&#47484; &#51228;&#50808;&#54620; set&#51012; &#51060;&#50857;&#54616;&#50668; &#49440;&#54805;&#54924;&#44480;&#48516;&#49437;&#51012; &#54616;&#50668; &#44396;&#54624; &#49688; &#51080;&#49845;&#45768;&#45796;.\n",
    "\n",
    "&#49324;&#49892; &#48320;&#49688;&#44036;&#51032; &#49440;&#54805;&#44288;&#44228; &#50508;&#50500;&#48372;&#44592; &#50948;&#54644;&#49436;&#45716; &#49328;&#51216;&#46020; or &#49345;&#44288;&#44228;&#49688;&#47484; &#49324;&#50857;&#54644;&#46020; &#46104;&#51648;&#47564; **VIF(&#48516;&#49328; &#54077;&#52285; &#51648;&#49688;, Variance Inflation Factor)** &#47484; &#51060;&#50857;&#54616;&#50668; &#44396;&#54620;&#45796;."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_max_vif(df, removal_cols):\n",
    "    vifs = []\n",
    "    cnames = df.drop(removal_cols, axis=1).columns.tolist()\n",
    "    for i in range(len(cnames)):\n",
    "        xvar = cnames[:]\n",
    "        yvar = xvar.pop(i)\n",
    "        model = sm.OLS(\n",
    "            df.drop(removal_cols, axis=1)[yvar], \n",
    "            sm.add_constant(df.drop(removal_cols, axis=1)[xvar]))\n",
    "        res = model.fit()\n",
    "        vif = 1 / (1 - res.rsquared)\n",
    "        vifs.append((yvar, round(vif, 3)))\n",
    "    vifs = sorted(vifs, key=operator.itemgetter(1), reverse=True)\n",
    "    return vifs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [],
   "source": [
    "vifs = get_max_vif(train_x, unnecesarries)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Housing_A152', 5.846),\n",
       " ('Property_A123', 4.999),\n",
       " ('Property_A121', 4.793),\n",
       " ('Housing_A151', 4.633),\n",
       " ('Property_A122', 4.281),\n",
       " ('Personal_status_and_sex_A93', 3.643),\n",
       " ('Personal_status_and_sex_A92', 3.302),\n",
       " ('Purpose_A43', 3.165),\n",
       " ('Purpose_A40', 2.983),\n",
       " ('Purpose_A42', 2.614),\n",
       " ('Credit_amount', 2.406),\n",
       " ('Purpose_A41', 2.121),\n",
       " ('Credit_history_A32', 2.011),\n",
       " ('Duration_in_month', 1.974),\n",
       " ('Present_employment_since_A73', 1.923),\n",
       " ('Savings_account_bonds_A61', 1.879),\n",
       " ('Present_employment_since_A72', 1.838),\n",
       " ('Present_employment_since_A74', 1.62),\n",
       " ('Purpose_A46', 1.564),\n",
       " ('Number_of_existing_credits_at_this_bank', 1.558),\n",
       " ('Personal_status_and_sex_A91', 1.53),\n",
       " ('Savings_account_bonds_A62', 1.478),\n",
       " ('Status_of_existing_checking_account_A11', 1.465),\n",
       " ('Age_in_years', 1.459),\n",
       " ('Credit_history_A31', 1.368),\n",
       " ('Status_of_existing_checking_account_A12', 1.365),\n",
       " ('Present_residence_since', 1.338),\n",
       " ('Savings_account_bonds_A63', 1.334),\n",
       " ('Credit_history_A33', 1.326),\n",
       " ('Present_employment_since_A71', 1.314),\n",
       " ('Installment_rate_in_percentage_of_disposable_income', 1.305),\n",
       " ('Savings_account_bonds_A64', 1.27),\n",
       " ('Purpose_A45', 1.26),\n",
       " ('Credit_history_A30', 1.259),\n",
       " ('Purpose_A410', 1.216),\n",
       " ('Purpose_A44', 1.191),\n",
       " ('Purpose_A48', 1.146),\n",
       " ('Status_of_existing_checking_account_A13', 1.14),\n",
       " ('Other_installment_plans_A141', 1.134),\n",
       " ('Other_installment_plans_A142', 1.107)]"
      ]
     },
     "execution_count": 185,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vifs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {},
   "outputs": [],
   "source": [
    "unnecesarries.append(vifs[0][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Personal_status_and_sex_A92', 'Housing_A152']"
      ]
     },
     "execution_count": 178,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "unnecesarries"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "&#47924;&#51032;&#48120;&#54620; &#48320;&#49688;(&#44032;&#51109; &#45458;&#51008; p-value)&#45716; Personal_status_and_sex_A92&#50688;&#44256; pvalue&#44032; 0.9580, VIF&#44032; 3.302&#51060;&#45796;.\n",
    "\n",
    "&#45796;&#51473;&#44277;&#49440;&#49457;&#51060; &#44032;&#51109; &#45458;&#50520;&#45912; &#48320;&#49688;&#45716; Housing_A152&#50688;&#44256; pvalue&#44032; 0.4770, VIF&#44032; 5.846&#51060;&#45796;.\n",
    "\n",
    "**&#47924;&#51032;&#48120;&#54620; &#48320;&#49688;&#47484; &#47676;&#51200; &#49325;&#51228;&#54644;&#50556;&#54616;&#47728;, &#51648;&#50864;&#44592; &#51204;&#50640;, &#46160; &#48320;&#49688;&#51032; pvalue&#50752; VIF&#47484; &#44033;&#44033; &#48708;&#44368;&#54644;&#48372;&#45716;&#44163;&#51060; &#51339;&#45796;**.\n",
    "\n",
    "&#54869;&#49892;&#55176; Personal_status_and_sex_A92&#45716; &#49325;&#51228;&#54644;&#50556; &#54620;&#45796;."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "&#47784;&#45944;&#51012; &#49325;&#51228;&#54616;&#44592; &#50526;&#49436; &#47784;&#45944; &#52292;&#53469; &#44592;&#51456;&#51012; &#49332;&#54196;&#48400;&#50556;&#54620;&#45796;."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. &#47784;&#45944; &#52292;&#53469; &#44592;&#51456;\n",
    "### (A) C&#53685;&#44228;&#47049; (concordance statistics)\n",
    " - &#47196;&#51648;&#49828;&#54001; &#54924;&#44480; &#47784;&#45944;&#50640;&#49436; &#51060;&#54637; &#44208;&#44284;&#51032; &#51201;&#54633;&#46020;(goodness of fit)&#50640; &#44288;&#54620; &#54408;&#51656;&#51012; &#52769;&#51221;&#54616;&#45716; &#52377;&#46020;\n",
    " - ROC curve&#50752; &#46041;&#51068;&#54616;&#45796;."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$C=Pr[\\pi(B|x_{i}) > \\pi(B|x_{j}) | Y_{i} = 1, Y_{j} = 0]$$\n",
    "$$C &#53685;&#44228;&#47049;(C &#51064;&#45937;&#49828;) = 0.5 + (\\frac{&#51068;&#52824;&#49933; &#48708;&#50984; - &#48520;&#51068;&#52824;&#49933; &#48708;&#50984;}{2})$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_c_stat(iv_pred):\n",
    "    noraml_test_df = iv_pred[iv_pred['class'] == 0][['class', 'probs']]\n",
    "    spammer_test_df = iv_pred[iv_pred['class'] == 1][['class', 'probs']]\n",
    "\n",
    "    noraml_test_df['key'] = 0\n",
    "    spammer_test_df['key'] = 0\n",
    "\n",
    "    cross_join_df = noraml_test_df.merge(spammer_test_df, how='outer', on='key').drop('key', axis=1)\n",
    "\n",
    "    cross_join_df['concordance'] = cross_join_df['probs_x'] < cross_join_df['probs_y']\n",
    "    cross_join_df['in_concordance'] = cross_join_df['probs_x'] > cross_join_df['probs_y']\n",
    "    cross_join_df['tie'] = cross_join_df['probs_x'] == cross_join_df['probs_y']\n",
    "\n",
    "    results = cross_join_df.agg({'concordance': np.sum, 'in_concordance': np.sum, 'tie': np.sum}) / len(cross_join_df)\n",
    "    c_stat = 0.5 + (results['concordance'] - results['in_concordance']) / 2\n",
    "    \n",
    "    return c_stat"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### (B) AIC, Likelihood\n",
    "\n",
    "- $AIC=-2 \\times ln(L) + 2 \\times k$\n",
    "\n",
    "Akaike information criterion&#45716; &#51452;&#50612;&#51652; &#45936;&#51060;&#53552; &#51665;&#54633;&#50640; &#44288;&#54644; &#53685;&#44228; &#47784;&#45944;&#51032; &#49345;&#45824;&#51201; &#54408;&#51656;&#51012; &#52769;&#51221;&#54620;&#45796;. &#51060; &#52377;&#46020;&#45716; &#54200;&#54693;&#44284; &#48516;&#49328;&#51032; &#53944;&#47112;&#51060;&#46300; &#50724;&#54532;&#51060;&#45796;. &#45908; &#51089;&#51008; AIC&#47484; &#49440;&#54840;&#54620;&#45796;.\n",
    "\n",
    "- $L = \\Pi_{i=1}^{n}p(x_i)^{y_i}(1-p(x_i))^{1-y_i}$\n",
    "\n",
    "log likelihood&#45716; &#53360; &#44050;&#51012; &#49440;&#54840;&#54620;&#45796;."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_aic_value(model):\n",
    "    return -2 * model.llf + 2 * (len(model.params) - 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> &#47924;&#51032;&#48120;&#54620; &#48320;&#49688;&#47484; &#51648;&#50864;&#44592; &#51204;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "c_stat: 0.8227198836095874\n",
      "aic: 816.2537855406207\n",
      "loglikehood: -368.12689277031035\n"
     ]
    }
   ],
   "source": [
    "c_stat, aic = get_c_stat(train_pred), get_aic_value(logistic_model)\n",
    "print('c_stat:', c_stat)\n",
    "print('aic:', aic)\n",
    "print('loglikehood:', logistic_model.llf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "c_stat: 0.8067075369635773\n",
      "aic: 816.2537855406207\n",
      "loglikehood: -368.12689277031035\n"
     ]
    }
   ],
   "source": [
    "c_stat, aic = get_c_stat(test_pred), get_aic_value(logistic_model)\n",
    "print('c_stat:', c_stat)\n",
    "print('aic:', aic)\n",
    "print('loglikehood:', logistic_model.llf)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> &#51648;&#50864;&#44256; &#45212; &#54980;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "metadata": {},
   "outputs": [],
   "source": [
    "unnecesarries.extend(['Personal_status_and_sex_A92', 'Housing_A152'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimization terminated successfully.\n",
      "         Current function value: 0.460475\n",
      "         Iterations 7\n"
     ]
    }
   ],
   "source": [
    "drop_logistic_model = sm.Logit(\n",
    "    train_y,\n",
    "    sm.add_constant(train_x.drop(unnecesarries, axis=1))\n",
    ").fit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_pred = pd.DataFrame({\n",
    "    'probs': drop_logistic_model.predict(sm.add_constant(train_x.drop(unnecesarries, axis=1))),\n",
    "    'class': train_y\n",
    "})\n",
    "\n",
    "train_pred['y_pred'] = 0\n",
    "train_pred.loc[train_pred['probs'] > 0.5, 'y_pred'] = 1\n",
    "\n",
    "# Test prediction\n",
    "test_pred = pd.DataFrame({\n",
    "    'probs': drop_logistic_model.predict(sm.add_constant(test_x.drop(unnecesarries, axis=1))),\n",
    "    'class': test_y\n",
    "})\n",
    "\n",
    "test_pred['y_pred'] = 0\n",
    "test_pred.loc[test_pred['probs'] > 0.5, 'y_pred'] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "c_stat: 0.8227198836095874\n",
      "aic: 812.7605967161102\n",
      "loglikehood: -368.3802983580551\n"
     ]
    }
   ],
   "source": [
    "c_stat, aic = get_c_stat(train_pred), get_aic_value(drop_logistic_model)\n",
    "print('c_stat:', c_stat)\n",
    "print('aic:', aic)\n",
    "print('loglikehood:', drop_logistic_model.llf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "c_stat: 0.8101935328765477\n",
      "aic: 812.7605967161102\n",
      "loglikehood: -368.3802983580551\n"
     ]
    }
   ],
   "source": [
    "c_stat, aic = get_c_stat(test_pred), get_aic_value(drop_logistic_model)\n",
    "print('c_stat:', c_stat)\n",
    "print('aic:', aic)\n",
    "print('loglikehood:', drop_logistic_model.llf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "        <td>Model:</td>              <td>Logit</td>       <td>No. Iterations:</td>   <td>7.0000</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <td>Dependent Variable:</td>      <td>Target</td>      <td>Pseudo R-squared:</td>   <td>0.247</td> \n",
       "</tr>\n",
       "<tr>\n",
       "         <td>Date:</td>        <td>2019-02-25 22:24</td>       <td>AIC:</td>        <td>814.7606</td>\n",
       "</tr>\n",
       "<tr>\n",
       "   <td>No. Observations:</td>         <td>800</td>             <td>BIC:</td>        <td>997.4605</td>\n",
       "</tr>\n",
       "<tr>\n",
       "       <td>Df Model:</td>             <td>38</td>         <td>Log-Likelihood:</td>   <td>-368.38</td>\n",
       "</tr>\n",
       "<tr>\n",
       "     <td>Df Residuals:</td>           <td>761</td>           <td>LL-Null:</td>       <td>-489.54</td>\n",
       "</tr>\n",
       "<tr>\n",
       "      <td>Converged:</td>           <td>1.0000</td>           <td>Scale:</td>        <td>1.0000</td> \n",
       "</tr>\n",
       "</table>\n",
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "                           <td></td>                            <th>Coef.</th>  <th>Std.Err.</th>    <th>z</th>     <th>P>|z|</th> <th>[0.025</th>  <th>0.975]</th> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>const</th>                                               <td>-3.8574</td>  <td>0.9727</td>  <td>-3.9656</td> <td>0.0001</td> <td>-5.7638</td> <td>-1.9509</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Duration_in_month</th>                                   <td>0.0231</td>   <td>0.0101</td>  <td>2.2787</td>  <td>0.0227</td> <td>0.0032</td>  <td>0.0429</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Credit_amount</th>                                       <td>0.0001</td>   <td>0.0000</td>  <td>2.4931</td>  <td>0.0127</td> <td>0.0000</td>  <td>0.0002</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Age_in_years</th>                                        <td>-0.0306</td>  <td>0.0103</td>  <td>-2.9737</td> <td>0.0029</td> <td>-0.0508</td> <td>-0.0104</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Number_of_existing_credits_at_this_bank</th>             <td>0.3431</td>   <td>0.2072</td>  <td>1.6559</td>  <td>0.0977</td> <td>-0.0630</td> <td>0.7492</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Installment_rate_in_percentage_of_disposable_income</th> <td>0.3180</td>   <td>0.0944</td>  <td>3.3681</td>  <td>0.0008</td> <td>0.1330</td>  <td>0.5031</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Present_residence_since</th>                             <td>0.0415</td>   <td>0.0935</td>  <td>0.4440</td>  <td>0.6570</td> <td>-0.1417</td> <td>0.2248</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Status_of_existing_checking_account_A11</th>             <td>1.5561</td>   <td>0.2523</td>  <td>6.1684</td>  <td>0.0000</td> <td>1.0617</td>  <td>2.0506</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Status_of_existing_checking_account_A12</th>             <td>1.1076</td>   <td>0.2469</td>  <td>4.4869</td>  <td>0.0000</td> <td>0.6238</td>  <td>1.5915</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Status_of_existing_checking_account_A13</th>             <td>0.5488</td>   <td>0.4351</td>  <td>1.2612</td>  <td>0.2072</td> <td>-0.3041</td> <td>1.4016</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Credit_history_A30</th>                                  <td>1.5543</td>   <td>0.4713</td>  <td>3.2982</td>  <td>0.0010</td> <td>0.6307</td>  <td>2.4780</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Credit_history_A31</th>                                  <td>1.4789</td>   <td>0.4809</td>  <td>3.0756</td>  <td>0.0021</td> <td>0.5365</td>  <td>2.4214</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Credit_history_A32</th>                                  <td>0.8853</td>   <td>0.2813</td>  <td>3.1471</td>  <td>0.0016</td> <td>0.3340</td>  <td>1.4367</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Credit_history_A33</th>                                  <td>0.5490</td>   <td>0.3686</td>  <td>1.4893</td>  <td>0.1364</td> <td>-0.1735</td> <td>1.2715</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Savings_account_bonds_A61</th>                           <td>0.6956</td>   <td>0.2737</td>  <td>2.5413</td>  <td>0.0110</td> <td>0.1591</td>  <td>1.2321</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Savings_account_bonds_A62</th>                           <td>0.5991</td>   <td>0.3697</td>  <td>1.6203</td>  <td>0.1052</td> <td>-0.1256</td> <td>1.3237</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Savings_account_bonds_A63</th>                           <td>0.2976</td>   <td>0.4698</td>  <td>0.6335</td>  <td>0.5264</td> <td>-0.6231</td> <td>1.2183</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Savings_account_bonds_A64</th>                           <td>-0.2150</td>  <td>0.5863</td>  <td>-0.3668</td> <td>0.7138</td> <td>-1.3641</td> <td>0.9340</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Purpose_A40</th>                                         <td>0.6528</td>   <td>0.3596</td>  <td>1.8154</td>  <td>0.0695</td> <td>-0.0520</td> <td>1.3576</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Purpose_A41</th>                                         <td>-0.9272</td>  <td>0.4621</td>  <td>-2.0067</td> <td>0.0448</td> <td>-1.8329</td> <td>-0.0216</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Purpose_A410</th>                                        <td>-0.6423</td>  <td>0.8345</td>  <td>-0.7698</td> <td>0.4414</td> <td>-2.2779</td> <td>0.9932</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Purpose_A42</th>                                         <td>-0.2835</td>  <td>0.3841</td>  <td>-0.7380</td> <td>0.4605</td> <td>-1.0364</td> <td>0.4694</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Purpose_A43</th>                                         <td>-0.3321</td>  <td>0.3613</td>  <td>-0.9192</td> <td>0.3580</td> <td>-1.0402</td> <td>0.3760</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Purpose_A44</th>                                         <td>0.1707</td>   <td>0.8124</td>  <td>0.2102</td>  <td>0.8335</td> <td>-1.4216</td> <td>1.7630</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Purpose_A45</th>                                         <td>0.2766</td>   <td>0.6279</td>  <td>0.4406</td>  <td>0.6595</td> <td>-0.9540</td> <td>1.5073</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Purpose_A46</th>                                         <td>0.9581</td>   <td>0.5227</td>  <td>1.8327</td>  <td>0.0668</td> <td>-0.0665</td> <td>1.9826</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Purpose_A48</th>                                         <td>-1.2385</td>  <td>1.2279</td>  <td>-1.0086</td> <td>0.3132</td> <td>-3.6451</td> <td>1.1682</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Property_A121</th>                                       <td>-0.8475</td>  <td>0.3403</td>  <td>-2.4908</td> <td>0.0127</td> <td>-1.5144</td> <td>-0.1806</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Property_A122</th>                                       <td>-0.5012</td>  <td>0.3230</td>  <td>-1.5515</td> <td>0.1208</td> <td>-1.1342</td> <td>0.1319</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Property_A123</th>                                       <td>-0.4812</td>  <td>0.3012</td>  <td>-1.5973</td> <td>0.1102</td> <td>-1.0716</td> <td>0.1092</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Present_employment_since_A71</th>                        <td>0.3068</td>   <td>0.4011</td>  <td>0.7649</td>  <td>0.4444</td> <td>-0.4793</td> <td>1.0929</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Present_employment_since_A72</th>                        <td>0.1538</td>   <td>0.3268</td>  <td>0.4706</td>  <td>0.6380</td> <td>-0.4867</td> <td>0.7943</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Present_employment_since_A73</th>                        <td>0.1967</td>   <td>0.2771</td>  <td>0.7097</td>  <td>0.4779</td> <td>-0.3464</td> <td>0.7398</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Present_employment_since_A74</th>                        <td>-0.3067</td>  <td>0.3273</td>  <td>-0.9372</td> <td>0.3487</td> <td>-0.9482</td> <td>0.3348</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Housing_A151</th>                                        <td>0.5122</td>   <td>0.2563</td>  <td>1.9984</td>  <td>0.0457</td> <td>0.0099</td>  <td>1.0146</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Other_installment_plans_A141</th>                        <td>0.7157</td>   <td>0.2689</td>  <td>2.6619</td>  <td>0.0078</td> <td>0.1887</td>  <td>1.2427</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Other_installment_plans_A142</th>                        <td>0.5353</td>   <td>0.4381</td>  <td>1.2218</td>  <td>0.2218</td> <td>-0.3234</td> <td>1.3939</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Personal_status_and_sex_A91</th>                         <td>0.4805</td>   <td>0.4459</td>  <td>1.0775</td>  <td>0.2813</td> <td>-0.3935</td> <td>1.3544</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Personal_status_and_sex_A93</th>                         <td>-0.4934</td>  <td>0.2122</td>  <td>-2.3255</td> <td>0.0200</td> <td>-0.9092</td> <td>-0.0776</td>\n",
       "</tr>\n",
       "</table>"
      ],
      "text/plain": [
       "<class 'statsmodels.iolib.summary2.Summary'>\n",
       "\"\"\"\n",
       "                                          Results: Logit\n",
       "===================================================================================================\n",
       "Model:                          Logit                        No. Iterations:               7.0000  \n",
       "Dependent Variable:             Target                       Pseudo R-squared:             0.247   \n",
       "Date:                           2019-02-25 22:24             AIC:                          814.7606\n",
       "No. Observations:               800                          BIC:                          997.4605\n",
       "Df Model:                       38                           Log-Likelihood:               -368.38 \n",
       "Df Residuals:                   761                          LL-Null:                      -489.54 \n",
       "Converged:                      1.0000                       Scale:                        1.0000  \n",
       "---------------------------------------------------------------------------------------------------\n",
       "                                                     Coef.  Std.Err.    z    P>|z|   [0.025  0.975]\n",
       "---------------------------------------------------------------------------------------------------\n",
       "const                                               -3.8574   0.9727 -3.9656 0.0001 -5.7638 -1.9509\n",
       "Duration_in_month                                    0.0231   0.0101  2.2787 0.0227  0.0032  0.0429\n",
       "Credit_amount                                        0.0001   0.0000  2.4931 0.0127  0.0000  0.0002\n",
       "Age_in_years                                        -0.0306   0.0103 -2.9737 0.0029 -0.0508 -0.0104\n",
       "Number_of_existing_credits_at_this_bank              0.3431   0.2072  1.6559 0.0977 -0.0630  0.7492\n",
       "Installment_rate_in_percentage_of_disposable_income  0.3180   0.0944  3.3681 0.0008  0.1330  0.5031\n",
       "Present_residence_since                              0.0415   0.0935  0.4440 0.6570 -0.1417  0.2248\n",
       "Status_of_existing_checking_account_A11              1.5561   0.2523  6.1684 0.0000  1.0617  2.0506\n",
       "Status_of_existing_checking_account_A12              1.1076   0.2469  4.4869 0.0000  0.6238  1.5915\n",
       "Status_of_existing_checking_account_A13              0.5488   0.4351  1.2612 0.2072 -0.3041  1.4016\n",
       "Credit_history_A30                                   1.5543   0.4713  3.2982 0.0010  0.6307  2.4780\n",
       "Credit_history_A31                                   1.4789   0.4809  3.0756 0.0021  0.5365  2.4214\n",
       "Credit_history_A32                                   0.8853   0.2813  3.1471 0.0016  0.3340  1.4367\n",
       "Credit_history_A33                                   0.5490   0.3686  1.4893 0.1364 -0.1735  1.2715\n",
       "Savings_account_bonds_A61                            0.6956   0.2737  2.5413 0.0110  0.1591  1.2321\n",
       "Savings_account_bonds_A62                            0.5991   0.3697  1.6203 0.1052 -0.1256  1.3237\n",
       "Savings_account_bonds_A63                            0.2976   0.4698  0.6335 0.5264 -0.6231  1.2183\n",
       "Savings_account_bonds_A64                           -0.2150   0.5863 -0.3668 0.7138 -1.3641  0.9340\n",
       "Purpose_A40                                          0.6528   0.3596  1.8154 0.0695 -0.0520  1.3576\n",
       "Purpose_A41                                         -0.9272   0.4621 -2.0067 0.0448 -1.8329 -0.0216\n",
       "Purpose_A410                                        -0.6423   0.8345 -0.7698 0.4414 -2.2779  0.9932\n",
       "Purpose_A42                                         -0.2835   0.3841 -0.7380 0.4605 -1.0364  0.4694\n",
       "Purpose_A43                                         -0.3321   0.3613 -0.9192 0.3580 -1.0402  0.3760\n",
       "Purpose_A44                                          0.1707   0.8124  0.2102 0.8335 -1.4216  1.7630\n",
       "Purpose_A45                                          0.2766   0.6279  0.4406 0.6595 -0.9540  1.5073\n",
       "Purpose_A46                                          0.9581   0.5227  1.8327 0.0668 -0.0665  1.9826\n",
       "Purpose_A48                                         -1.2385   1.2279 -1.0086 0.3132 -3.6451  1.1682\n",
       "Property_A121                                       -0.8475   0.3403 -2.4908 0.0127 -1.5144 -0.1806\n",
       "Property_A122                                       -0.5012   0.3230 -1.5515 0.1208 -1.1342  0.1319\n",
       "Property_A123                                       -0.4812   0.3012 -1.5973 0.1102 -1.0716  0.1092\n",
       "Present_employment_since_A71                         0.3068   0.4011  0.7649 0.4444 -0.4793  1.0929\n",
       "Present_employment_since_A72                         0.1538   0.3268  0.4706 0.6380 -0.4867  0.7943\n",
       "Present_employment_since_A73                         0.1967   0.2771  0.7097 0.4779 -0.3464  0.7398\n",
       "Present_employment_since_A74                        -0.3067   0.3273 -0.9372 0.3487 -0.9482  0.3348\n",
       "Housing_A151                                         0.5122   0.2563  1.9984 0.0457  0.0099  1.0146\n",
       "Other_installment_plans_A141                         0.7157   0.2689  2.6619 0.0078  0.1887  1.2427\n",
       "Other_installment_plans_A142                         0.5353   0.4381  1.2218 0.2218 -0.3234  1.3939\n",
       "Personal_status_and_sex_A91                          0.4805   0.4459  1.0775 0.2813 -0.3935  1.3544\n",
       "Personal_status_and_sex_A93                         -0.4934   0.2122 -2.3255 0.0200 -0.9092 -0.0776\n",
       "===================================================================================================\n",
       "\n",
       "\"\"\""
      ]
     },
     "execution_count": 206,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "drop_logistic_model.summary2()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. &#47924;&#51032;&#48120;&#54620; &#48320;&#49688;&#50752; &#45796;&#51473;&#44277;&#49440;&#49457;&#51060; &#51080;&#45716; &#48320;&#49688;&#44032; &#50630;&#51012;&#46412;&#44620;&#51648; &#48152;&#48373;&#54620;&#45796;. (&#44592;&#51456;&#51008; C&#53685;&#44228;&#47049;, AIC, Likelihood)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "&#50896;&#47000;&#45716; &#51060; &#44284;&#51221;&#51012; &#44033;&#44033; &#48320;&#49688;&#47484; &#54869;&#51064;&#54616;&#47732;&#49436;, &#48320;&#49688;&#47484; &#54616;&#45208;&#50473; &#51228;&#44144;&#54644;&#50556; &#54620;&#45796;. &#54616;&#51648;&#47564; &#49884;&#44036; &#44288;&#44228;&#49345;.."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Step 1. Pvalue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 218,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6\n",
      "['Present_residence_since', 'Savings_account_bonds_A63', 'Savings_account_bonds_A64', 'Purpose_A44', 'Purpose_A45', 'Present_employment_since_A72']\n"
     ]
    }
   ],
   "source": [
    "# Pvalue&#47484; &#54869;&#51064;&#54616;&#44256;, &#49345;&#54620;&#49440;&#51012; &#44396;&#54620;&#45796;.\n",
    "unrelated_cols = get_unrelated_cols(drop_logistic_model, 0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "metadata": {},
   "outputs": [],
   "source": [
    "unnecesarries.extend(unrelated_cols)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Step 2. VIF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 221,
   "metadata": {},
   "outputs": [],
   "source": [
    "vifs = get_max_vif(train_x, unnecesarries)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Property_A121', 2.58),\n",
       " ('Property_A123', 2.487),\n",
       " ('Purpose_A43', 2.431),\n",
       " ('Credit_amount', 2.354),\n",
       " ('Purpose_A40', 2.32),\n",
       " ('Property_A122', 2.283),\n",
       " ('Purpose_A42', 2.111),\n",
       " ('Credit_history_A32', 1.977),\n",
       " ('Duration_in_month', 1.947),\n",
       " ('Purpose_A41', 1.79)]"
      ]
     },
     "execution_count": 222,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vifs[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 226,
   "metadata": {},
   "outputs": [],
   "source": [
    "unnecesarries.extend([elem[0] for elem in vifs[:10]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 228,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimization terminated successfully.\n",
      "         Current function value: 0.506885\n",
      "         Iterations 6\n"
     ]
    }
   ],
   "source": [
    "drop_logistic_model = sm.Logit(\n",
    "    train_y,\n",
    "    sm.add_constant(train_x.drop(unnecesarries, axis=1))\n",
    ").fit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 229,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_pred = pd.DataFrame({\n",
    "    'probs': drop_logistic_model.predict(sm.add_constant(train_x.drop(unnecesarries, axis=1))),\n",
    "    'class': train_y\n",
    "})\n",
    "\n",
    "train_pred['y_pred'] = 0\n",
    "train_pred.loc[train_pred['probs'] > 0.5, 'y_pred'] = 1\n",
    "\n",
    "# Test prediction\n",
    "test_pred = pd.DataFrame({\n",
    "    'probs': drop_logistic_model.predict(sm.add_constant(test_x.drop(unnecesarries, axis=1))),\n",
    "    'class': test_y\n",
    "})\n",
    "\n",
    "test_pred['y_pred'] = 0\n",
    "test_pred.loc[test_pred['probs'] > 0.5, 'y_pred'] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 230,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "c_stat: 0.7741112983320838\n",
      "aic: 855.0167324752877\n",
      "loglikehood: -405.5083662376438\n"
     ]
    }
   ],
   "source": [
    "c_stat, aic = get_c_stat(train_pred), get_aic_value(drop_logistic_model)\n",
    "print('c_stat:', c_stat)\n",
    "print('aic:', aic)\n",
    "print('loglikehood:', drop_logistic_model.llf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 231,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "c_stat: 0.7756941940137035\n",
      "aic: 855.0167324752877\n",
      "loglikehood: -405.5083662376438\n"
     ]
    }
   ],
   "source": [
    "c_stat, aic = get_c_stat(test_pred), get_aic_value(drop_logistic_model)\n",
    "print('c_stat:', c_stat)\n",
    "print('aic:', aic)\n",
    "print('loglikehood:', drop_logistic_model.llf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 232,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "        <td>Model:</td>              <td>Logit</td>       <td>No. Iterations:</td>   <td>6.0000</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <td>Dependent Variable:</td>      <td>Target</td>      <td>Pseudo R-squared:</td>   <td>0.172</td> \n",
       "</tr>\n",
       "<tr>\n",
       "         <td>Date:</td>        <td>2019-02-25 22:41</td>       <td>AIC:</td>        <td>857.0167</td>\n",
       "</tr>\n",
       "<tr>\n",
       "   <td>No. Observations:</td>         <td>800</td>             <td>BIC:</td>        <td>964.7628</td>\n",
       "</tr>\n",
       "<tr>\n",
       "       <td>Df Model:</td>             <td>22</td>         <td>Log-Likelihood:</td>   <td>-405.51</td>\n",
       "</tr>\n",
       "<tr>\n",
       "     <td>Df Residuals:</td>           <td>777</td>           <td>LL-Null:</td>       <td>-489.54</td>\n",
       "</tr>\n",
       "<tr>\n",
       "      <td>Converged:</td>           <td>1.0000</td>           <td>Scale:</td>        <td>1.0000</td> \n",
       "</tr>\n",
       "</table>\n",
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "                           <td></td>                            <th>Coef.</th>  <th>Std.Err.</th>    <th>z</th>     <th>P>|z|</th> <th>[0.025</th>  <th>0.975]</th> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>const</th>                                               <td>-2.1483</td>  <td>0.5231</td>  <td>-4.1067</td> <td>0.0000</td> <td>-3.1737</td> <td>-1.1230</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Age_in_years</th>                                        <td>-0.0256</td>  <td>0.0087</td>  <td>-2.9609</td> <td>0.0031</td> <td>-0.0426</td> <td>-0.0087</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Number_of_existing_credits_at_this_bank</th>             <td>-0.0049</td>  <td>0.1631</td>  <td>-0.0303</td> <td>0.9758</td> <td>-0.3245</td> <td>0.3146</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Installment_rate_in_percentage_of_disposable_income</th> <td>0.1913</td>   <td>0.0806</td>  <td>2.3732</td>  <td>0.0176</td> <td>0.0333</td>  <td>0.3494</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Status_of_existing_checking_account_A11</th>             <td>1.6818</td>   <td>0.2321</td>  <td>7.2451</td>  <td>0.0000</td> <td>1.2268</td>  <td>2.1367</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Status_of_existing_checking_account_A12</th>             <td>1.2142</td>   <td>0.2300</td>  <td>5.2782</td>  <td>0.0000</td> <td>0.7633</td>  <td>1.6650</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Status_of_existing_checking_account_A13</th>             <td>0.4189</td>   <td>0.4152</td>  <td>1.0090</td>  <td>0.3130</td> <td>-0.3948</td> <td>1.2326</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Credit_history_A30</th>                                  <td>1.4395</td>   <td>0.4109</td>  <td>3.5034</td>  <td>0.0005</td> <td>0.6342</td>  <td>2.2449</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Credit_history_A31</th>                                  <td>1.0135</td>   <td>0.3945</td>  <td>2.5689</td>  <td>0.0102</td> <td>0.2402</td>  <td>1.7868</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Credit_history_A33</th>                                  <td>0.3081</td>   <td>0.3093</td>  <td>0.9960</td>  <td>0.3193</td> <td>-0.2982</td> <td>0.9143</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Savings_account_bonds_A61</th>                           <td>0.5528</td>   <td>0.2132</td>  <td>2.5935</td>  <td>0.0095</td> <td>0.1350</td>  <td>0.9706</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Savings_account_bonds_A62</th>                           <td>0.6598</td>   <td>0.3173</td>  <td>2.0793</td>  <td>0.0376</td> <td>0.0379</td>  <td>1.2817</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Purpose_A410</th>                                        <td>0.0894</td>   <td>0.6874</td>  <td>0.1301</td>  <td>0.8965</td> <td>-1.2578</td> <td>1.4367</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Purpose_A46</th>                                         <td>1.1777</td>   <td>0.3936</td>  <td>2.9919</td>  <td>0.0028</td> <td>0.4062</td>  <td>1.9492</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Purpose_A48</th>                                         <td>-1.6748</td>  <td>1.2032</td>  <td>-1.3919</td> <td>0.1639</td> <td>-4.0330</td> <td>0.6835</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Present_employment_since_A71</th>                        <td>0.3893</td>   <td>0.3416</td>  <td>1.1397</td>  <td>0.2544</td> <td>-0.2802</td> <td>1.0589</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Present_employment_since_A73</th>                        <td>0.0832</td>   <td>0.2046</td>  <td>0.4069</td>  <td>0.6841</td> <td>-0.3177</td> <td>0.4842</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Present_employment_since_A74</th>                        <td>-0.2333</td>  <td>0.2595</td>  <td>-0.8990</td> <td>0.3686</td> <td>-0.7420</td> <td>0.2754</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Housing_A151</th>                                        <td>0.4031</td>   <td>0.2296</td>  <td>1.7556</td>  <td>0.0791</td> <td>-0.0469</td> <td>0.8531</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Other_installment_plans_A141</th>                        <td>0.6645</td>   <td>0.2541</td>  <td>2.6149</td>  <td>0.0089</td> <td>0.1664</td>  <td>1.1625</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Other_installment_plans_A142</th>                        <td>0.4337</td>   <td>0.4208</td>  <td>1.0306</td>  <td>0.3027</td> <td>-0.3911</td> <td>1.2585</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Personal_status_and_sex_A91</th>                         <td>0.4378</td>   <td>0.4224</td>  <td>1.0365</td>  <td>0.3000</td> <td>-0.3900</td> <td>1.2657</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Personal_status_and_sex_A93</th>                         <td>-0.2726</td>  <td>0.1908</td>  <td>-1.4281</td> <td>0.1533</td> <td>-0.6466</td> <td>0.1015</td> \n",
       "</tr>\n",
       "</table>"
      ],
      "text/plain": [
       "<class 'statsmodels.iolib.summary2.Summary'>\n",
       "\"\"\"\n",
       "                                          Results: Logit\n",
       "===================================================================================================\n",
       "Model:                          Logit                        No. Iterations:               6.0000  \n",
       "Dependent Variable:             Target                       Pseudo R-squared:             0.172   \n",
       "Date:                           2019-02-25 22:41             AIC:                          857.0167\n",
       "No. Observations:               800                          BIC:                          964.7628\n",
       "Df Model:                       22                           Log-Likelihood:               -405.51 \n",
       "Df Residuals:                   777                          LL-Null:                      -489.54 \n",
       "Converged:                      1.0000                       Scale:                        1.0000  \n",
       "---------------------------------------------------------------------------------------------------\n",
       "                                                     Coef.  Std.Err.    z    P>|z|   [0.025  0.975]\n",
       "---------------------------------------------------------------------------------------------------\n",
       "const                                               -2.1483   0.5231 -4.1067 0.0000 -3.1737 -1.1230\n",
       "Age_in_years                                        -0.0256   0.0087 -2.9609 0.0031 -0.0426 -0.0087\n",
       "Number_of_existing_credits_at_this_bank             -0.0049   0.1631 -0.0303 0.9758 -0.3245  0.3146\n",
       "Installment_rate_in_percentage_of_disposable_income  0.1913   0.0806  2.3732 0.0176  0.0333  0.3494\n",
       "Status_of_existing_checking_account_A11              1.6818   0.2321  7.2451 0.0000  1.2268  2.1367\n",
       "Status_of_existing_checking_account_A12              1.2142   0.2300  5.2782 0.0000  0.7633  1.6650\n",
       "Status_of_existing_checking_account_A13              0.4189   0.4152  1.0090 0.3130 -0.3948  1.2326\n",
       "Credit_history_A30                                   1.4395   0.4109  3.5034 0.0005  0.6342  2.2449\n",
       "Credit_history_A31                                   1.0135   0.3945  2.5689 0.0102  0.2402  1.7868\n",
       "Credit_history_A33                                   0.3081   0.3093  0.9960 0.3193 -0.2982  0.9143\n",
       "Savings_account_bonds_A61                            0.5528   0.2132  2.5935 0.0095  0.1350  0.9706\n",
       "Savings_account_bonds_A62                            0.6598   0.3173  2.0793 0.0376  0.0379  1.2817\n",
       "Purpose_A410                                         0.0894   0.6874  0.1301 0.8965 -1.2578  1.4367\n",
       "Purpose_A46                                          1.1777   0.3936  2.9919 0.0028  0.4062  1.9492\n",
       "Purpose_A48                                         -1.6748   1.2032 -1.3919 0.1639 -4.0330  0.6835\n",
       "Present_employment_since_A71                         0.3893   0.3416  1.1397 0.2544 -0.2802  1.0589\n",
       "Present_employment_since_A73                         0.0832   0.2046  0.4069 0.6841 -0.3177  0.4842\n",
       "Present_employment_since_A74                        -0.2333   0.2595 -0.8990 0.3686 -0.7420  0.2754\n",
       "Housing_A151                                         0.4031   0.2296  1.7556 0.0791 -0.0469  0.8531\n",
       "Other_installment_plans_A141                         0.6645   0.2541  2.6149 0.0089  0.1664  1.1625\n",
       "Other_installment_plans_A142                         0.4337   0.4208  1.0306 0.3027 -0.3911  1.2585\n",
       "Personal_status_and_sex_A91                          0.4378   0.4224  1.0365 0.3000 -0.3900  1.2657\n",
       "Personal_status_and_sex_A93                         -0.2726   0.1908 -1.4281 0.1533 -0.6466  0.1015\n",
       "===================================================================================================\n",
       "\n",
       "\"\"\""
      ]
     },
     "execution_count": 232,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "drop_logistic_model.summary2()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 235,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Confusion matrix:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>Actual</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>All</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Predict</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>504</td>\n",
       "      <td>150</td>\n",
       "      <td>654</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>55</td>\n",
       "      <td>91</td>\n",
       "      <td>146</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>All</th>\n",
       "      <td>559</td>\n",
       "      <td>241</td>\n",
       "      <td>800</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Actual     0    1  All\n",
       "Predict               \n",
       "0        504  150  654\n",
       "1         55   91  146\n",
       "All      559  241  800"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "print('Train Confusion matrix:')\n",
    "display(pd.crosstab(train_pred['y_pred'], train_pred['class'], rownames=['Predict'], colnames=['Actual'], margins=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 233,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Confusion matrix:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>Actual</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>All</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Predict</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>122</td>\n",
       "      <td>34</td>\n",
       "      <td>156</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>19</td>\n",
       "      <td>25</td>\n",
       "      <td>44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>All</th>\n",
       "      <td>141</td>\n",
       "      <td>59</td>\n",
       "      <td>200</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Actual     0   1  All\n",
       "Predict              \n",
       "0        122  34  156\n",
       "1         19  25   44\n",
       "All      141  59  200"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "print('Test Confusion matrix:')\n",
    "display(pd.crosstab(test_pred['y_pred'], test_pred['class'], rownames=['Predict'], colnames=['Actual'], margins=True))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5. ROC curve, AUC\n",
    " - ROC = FPR vs TPR(Recall) &#51032; &#48708;&#50984;\n",
    " - AUC = ROC &#44257;&#49440; &#50500;&#47000;&#51032; &#47732;&#51201;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 237,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "from sklearn import metrics\n",
    "from sklearn.metrics import auc\n",
    "\n",
    "plt.rcParams[\"figure.figsize\"] = (7,7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 238,
   "metadata": {},
   "outputs": [],
   "source": [
    "fpr, tpr, thresholds = metrics.roc_curve(train_pred['class'], train_pred['probs'], pos_label=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 240,
   "metadata": {},
   "outputs": [],
   "source": [
    "roc_auc = auc(fpr, tpr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 241,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzs3Xd4FGXXx/HvSSEFQu9GOtKbBBARRRBEqgUBsQOiIGJDRdQHFRUrKo9YUJBHfSk2pIkgiqII0qVKRwhFemgJpJz3j1lSIGWBbGaTnM915dqZ2ZnZX4aQk5m5575FVTHGGGMyEuB2AGOMMf7NCoUxxphMWaEwxhiTKSsUxhhjMmWFwhhjTKasUBhjjMmUFQpjjDGZskJh8hwR2S4isSJyXET2ish4ESl01jpXisjPInJMRGJEZLqI1D5rncIi8o6I7PDsa7NnvmTOfkfGuMsKhcmrOqtqIaAh0Ah4+swbItIcmANMBcoDlYG/gAUiUsWzTgHgJ6AO0B4oDFwJHASa+iq0iAT5at/GXCgrFCZPU9W9wGycgnHG68Bnqvquqh5T1UOq+iywCHjes85dQAXgJlVdp6pJqrpPVYer6vfpfZaI1BGRH0XkkIj8KyJDPcvHi8hLqdZrJSLRqea3i8hTIrIKOCEiz4rI12ft+10RGeWZLiIiY0Vkj4jsEpGXRCTwIg+VMRmyQmHyNBGJBG4ANnvmw3HODL5KZ/Uvgbae6euAH1T1uJefEwHMBX7AOUuphnNG4q3bgI5AUeBzoIOIFPbsOxDoDkzwrPs/IMHzGY2AdkDf8/gsY86LFQqTV30nIseAncA+YJhneXGcn/s96WyzBzhz/6FEButkpBOwV1XfUtU4z5nKn+ex/ShV3amqsar6D7AcuNHzXmvgpKouEpEyOIXvEVU9oar7gLeBnufxWcacFysUJq+6UVUjgFZATVIKwGEgCSiXzjblgAOe6YMZrJORS4EtF5TUsfOs+Qk4ZxkAvUg5m6gIBAN7ROSIiBwBPgJKX8RnG5MpKxQmT1PVX4HxwJue+RPAQuDWdFbvTsrlornA9SJS0MuP2glUzeC9E0B4qvmy6UU9a/4roJXn0tlNpBSKncApoKSqFvV8FVbVOl7mNOa8WaEw+cE7QFsROXNDewhwt4gMEpEIESnmudncHHjBs87nOL+UvxGRmiISICIlRGSoiHRI5zNmAGVF5BERCfHst5nnvZU49xyKi0hZ4JGsAqvqfuAX4FNgm6qu9yzfg9Ni6y1P890AEakqItdcwHExxitWKEye5/ml+xnwnGf+d+B64Gac+xD/4NwUvkpVN3nWOYVzQ/tv4EfgKLAY5xLWOfceVPUYzo3wzsBeYBNwreftz3Ga327H+SU/2cvoEzwZJpy1/C6gALAO51La15zfZTJjzovYwEXGGGMyY2cUxhhjMuWzQiEi40Rkn4isyeB9EZFRnm4RVonI5b7KYowx5sL58oxiPE7XBxm5Aaju+eoHfODDLMYYYy6QzwqFqs4HDmWySlecbhRUVRcBRUXEbsgZY4yfcbMDsktI+5BRtGfZOU/Dikg/nLMOChYs2LhmzZo5EtAYY3Kd2P1wdAcAMbEh/HOkCKpCQtK/B1S11IXs0s1CIeksS7cJlqqOAcYAREVF6dKlS32Zyxhj/JcqzLoL/l2W/vuHdrD/eDiPTG3PhBX1AWhSN4Qla57+50I/0s1CEY3T7cEZkcBul7IYY/KjhFNk8Pdp9lv3Bfw2BDTx4vZz6kiGb6nC5JV1eei7GzhwoiBhYUG89FJrHn64GUFBT2e4XVbcLBTTgIEiMgloBsR4njo1xhjf+/5OWP+F2ykuXIk60PnLNIu+nR7Nbf+3EIBrr63Exx93pmrV4hf9UT4rFCIyEadDtpKevveH4XRmhqp+CHwPdMDp/vkkcK+vshhj8iFNgn0rISHu3Peif0lbJAJDciZTgQjo9iMUrnjx+wopCpL2Cn7XO2rSdsK/dO9ehz59GiGS3hX+85frnsy2exTGGK8sehkWPJv1eo+chsBg3+fxgS1bDjF48I+MHt2B8uUjAFDVdAuEiCxT1agL+RwbdtEYk3skxsPGryHuYNbrbp3pvBapAuFlzn0/OAxavpYri0RiYhLvvLOI556bR2xsAkWKhDB+vDN8SXadRaRmhcIY49/iY2HVhxB7ADZ9C4f+Pr/tr3wBat/hm2wuWLNmH717T2XJEqftz+231+PNN9v59DOtUBhj/MuRrbD0LUiIdeY3fgnxJ85dr+GDWe8rtARU65q9+Vxy6lQCI0b8ziuv/EZ8fBKRkYX58MOOdOx4mc8/2wqFMSbnbfsBVvzXueF8tu0/ZLxdi5cgKBRq3QEF07mclIetW7ef4cPnk5Sk9O8fxauvXkfhwjlzE94KhTEmZ/3xAix8Puv1qnSEajc700EhUKUzhBT2aTR/c/p0IgUKBALQqFE53nijLY0bl+OaayrlaA4rFMaYzKlC9Hz44R5IOHnx+zu5L2W67UcQcem564QUg3LNzmn+mZ/8/PM27rtvOqNGtU++vPTYY81dyWKFwhiTVuyhlPsDMdvgq9aQFJ+9nyEBcP8uKJje8OH525EjcTzxxBw++WQFAO+/vzRH7kNkxgqFMSbFXx/B3AfSfy8gCDpOhEtaXvznhJVw9mfSmDZtA/37z2T37mMUKBDIc89dzVNPtXA7lhUKY4zHyf1pi0Sh8s5rQLDzvEHNHu7kygcOHYplwICZTJ68FoArrohk7Ngu1K59QZ29ZjsrFMbkRarOA2cnzqOfzR/vT5m+Zz2UsO78c0pQUAALFuwkPDyYV15pzcCBTQkM9J+Rqq1QGJMX7fkTvut8YdvW62tFIgfs3BlDiRLhhIcHU7hwCJMnd6NcuUJUrlzM7WjnsEJhTF60YZLzWigSKt/g/XaFykOzZ3yTyQCQlKSMGbOMJ5/8kX79Gic/VX3llem0/vITViiMyUt2/QHLRsKmb5z50g2h3Rh3M5lkmzYdpG/f6cyf74wh9M8/MSQlKQEB/t0M2AqFMf7o8GaY2x9Ox5zfdnuXpJ2/9p3sy2QuWEJCEiNHLmTYsF+Ii0ugdOmCjB7dgVtuqeWTTvyymxUKY/yNKoyrfnH7uPIFqNETilbNnkzmgh09eorWrf/HsmXOuGx33dWAkSPbUaJEuMvJvGeFwpiLkRAHp495t+78J2DtZ87DZplJPVTm5Q9DzV7nl6lgmewZGMdki8KFQ6hQoQj795/ko4860b59NbcjnTcrFMZciNPHYc1Y+P1ZiD9+ftt6O2ZyhdZ26SiXWrQomkKFClC3bmkAxozpTEhIIBEROTSSXjazQmHM+YqPhf9GpMyHFs/6LOGMYpdBt7neDZZjTy7nOidOnOaZZ35m1Kg/ady4PAsX9iEoKICSJXPPZab02E+iMd5KSnAeYpv/VMqy9uOhzt2uRTL+Y+7crdx333S2bz9CYKDQtm0VEhOTCArynwfnLpQVCmO8ceJfmNrVeZDtjKtftyJhOHw4lsGD5zBu3EoAGjYsy9ixXbj88nIuJ8s+ViiM8caS19IWiVtmQyXfDj9p/F9CQhLNmn3Cpk2HCAkJZNiwaxg8+EqCgwPdjpatrFAYk5mTB+D3ofDPHGc+ogL0WgSF8s5fi+bCBQUF8PDDzZgwYQ1jx3ahZs2SbkfyCVFVtzOcl6ioKF26dKnbMUx+8UlViNmaMn/jdKjayb08xlWqyhdfrCIxUbnnnoaA0yUH4PdPV4vIMlWNupBt7YzCmLMd2QIzekDsATjqdLVAmcZw9Rtw6TXuZjOu+eefIzzwwEx++GEzhQoV4Prrq1KuXITfF4jsYIXCmFNHYdrNsOt3Z+jNhLi070sg3L7Y+yawJk9JSlI++GAJQ4b8xPHjpylWLJS3376esmULuR0tx1ihMPnbvEdg+bvpv3flC1DrDoiItCKRT23YcIC+fafz++87AOjWrTb//e8N+apIgBUKk9/tWuC8BheEim2hwwRnPiAQAgu4l8v4hT59prFgwU7KlCnI++935Oaba7kdyRVWKEz+EX8Stv8ACbEpy+IOOq/d50HZJu7kMn5FVZN7dB09ugOjRv3Jm2+2o1ixMJeTuccKhckfjmyBzy+H00fTfz/Azh7yu7i4BIYP/5WtW48wceItADRoUJaxY7u6nMx9VihM/jB3QNoikbpH1qJVoVS9nM9k/MaCBTvo02caGzYcRASGDGlBgwZl3Y7lN6xQmLzpVAwseM5p4grwr+fZmyqdoeP/QYGIjLc1+caxY6cYOvQnRo9egirUrFmSsWO7WJE4ixUKk/fsWwmfN0r/vWvftiJhAJg9ezP9+s1gx44YgoICeOqpFjz77NWEhtqvxbPZETF5y5rxMPvelPkiVaDFcM90ZRvxzSSbPXsLO3bEcPnl5Rg7tgsNG9pZREasUJjcKz4WOKsLmlUfpUw3fx6ueNZp6moMsH//CUqVKgjA8OHXUqVKMR54ICpPdAXuS1YoTO604DlY9FLG73f+Ci7rlnN5jF/bs+cYAwfOYunS3axZ05+IiBAKFizAwIFN3Y6WK/i0jIpIexHZICKbRWRIOu9XEJF5IrJCRFaJSAdf5jF5yM5fnNfAAhAUlvareC2oaF2AG+eZiPHjV1K79vt8++16Dh2KZcWKvW7HynV8dkYhIoHAaKAtEA0sEZFpqrou1WrPAl+q6gciUhv4Hqjkq0wmD+o2FyJbup3C+KHt24/Qr990fvzR6f33hhuq8eGHnahQoYjLyXIfX156agpsVtWtACIyCegKpC4UChT2TBcBdvswjzEmn/jss78YMGAmJ07EU7x4GO++257bb6+X/MS1OT++LBSXADtTzUcDzc5a53lgjog8BBQErktvRyLSD+gHUKFChWwPanKJg+tg/QTQRIjZ5nYa48eKFQvlxIl4evSow6hRN1C6dEG3I+VqviwU6ZXus0dJug0Yr6pviUhz4HMRqauqSWk2Uh0DjAFn4CKfpDX+ZcmbcHBN2mVr/3fuegUKn7vM5Dvx8Yn89tsOWreuDEDnzjVYvLgvTZpc4nKyvMGXhSIauDTVfCTnXlrqA7QHUNWFIhIKlAT2+TCX8WdxR2BqV4ien/E6tW6HEnWgcEUoVT/nshm/tHz5Hnr3nsrq1fv488++REWVB7AikY2yLBQi0gS4A2gJlANigTXATGCCqh7LYNMlQHURqQzsAnoCvc5aZwfQBhgvIrWAUGD/BXwfJjc7uR9m9oKT/8KB1Wnfu/7TtPOFK0CF1jmXzfit2Nh4XnjhV9588w8SE5XKlYty+nSi27HypEwLhYjMAA4CU4G3cP7SDwUuA64FZorI66o64+xtVTVBRAYCs4FAYJyqrhWRF4GlqjoNeBz4WEQexbksdY/mtkG8zcXb+QvsmJt2Wbkr4JYfIMRaqJhz/fbbP/TtO52NG51O/B599AqGD7+WggWtF2BfyOqMoo+q/nvWsjhgsefrNREpndHGqvo9TpPX1Mv+k2p6HdDivBKbPMjzt0HFtnDNm06X38VrOMOSGnOWTz5Zzn33TQegdu1SjB3bhSuuiHQ5Vd6WaaFIp0ikt47dTzDZI6So3XMwWerQoTolS4YzYEAUQ4e2JCTEOpjwtawuPR3m3JZK4LRoUlUt7pNUxhjjcfDgSd57bzHPPns1gYEBlC8fwdatg4iICHE7Wr6RVSkumSMpTP4VPR+2z3E7hfFDqspXX61j4MDv2b//JBERITz2WHMAKxI5LKtCkdVTKhmMK2lMFhLjYfEI+GNYyrKgUPfyGL+ye/cxHnzwe7777m8ArrmmIl261HA5Vf6VVaFYi3PpKaOH5+wxaXP+VrwHi1+D49Epyxo9BA36u5fJ+AVVZdy4FTz++BxiYk4REVGAN99sR9++lxMQYI0b3JLVzexLM3vfmEwd3ux0Bx5/PGXZ8d2wb3na9e5YBmUuz9lsxi99/fU6+vZ1WjR17FidDz/sRGSkPX3vNq+bC4hIEaAqznMUAKjqH74IZXK5uMPw/R2w7fvM17t+HFTtAmElciaX8Xs331yLLl1q0LNnHXr2rGud+PkJrwqFiPQBHsPp6G810ARYBLTyWTKTe02+Ju0T1jV7Qc2eadcpf6UVCMPatft49NHZjBvXlcjIwgQGBjB1as+sNzQ5ytszikeAKGChqrYUkTo4Y0kYc64Te5zXyKuh3SdQrLq7eYzfOX06kdde+53hw+cTH5/Ec8/N49NPu7ody2TA20IRp6qxIoKIFPB0xVHTp8lM7qNJcGAtJCU4852/gvAMH9w3+dSSJbvo02caq1c7z+ref39jXnst3REGjJ/wtlDsEZGiwHRgtogcArJ8atvkI6owuRXs+i3VQhuw3qQ4eTKeYcPmMXLkIpKSlKpVi/Hxx5259trKbkczWfCqUKhqF8/kcyLSBmc0upk+S2Vyj5MHYNtM2P1H2iJR/367B2HS2LjxIG+/vQiAwYOb88IL1xIeHuxyKuMNb29mNwHWq+pxVf1JRCKAesBSn6Yz/utUDKz5FH559Nz3Bp2E4LCcz2T8TmxsPGFhTjFo2LAs777bniZNLqFpUxsrIjfx9trAGOBkqvkTwEfZH8fkGqs+TlskilZ1ziLu+suKhAFg5syNVK/+X6ZO/Tt52YMPNrUikQt5e48iIPXwpKqaJCJ2zpifnfaMV3VJS6fpa/37ISDQ3UzGL+zff4JHHpnNhAlOE+nx4/+ia1dr+5KbeVsotolIf5wzCwX6A9t9Fcr4sRXvwfr/g2M7nfkKbaDhAHczGb+gqkyevJaHHprFgQMnCQsL4uWXWzNoUDO3o5mL5G2huB8YDQzHKRTzgPt8Fcr4scWvwvFdKfNFrMWKcc4i+vSZxvTpGwFo3boyH3/cmSpVirmczGQHb1s9/Qt083EW46/iT8Dyd2HhC5B42lnWdSoUrQIl6ribzfiFsLBgVq36lyJFQnjrrXb07t3Iut/IQ7xt9VQN54yirKo2EJH6QEdVHeHTdMZdqk6BOLtlU8l6UKUDBNjIYvnZ5s2HKFu2EIUKFaBQoQJ8/XV3ypePoHz5CLejmWzmbaunT4AXgDM3tFcDd/gkkfEfvw1JWySKVYf7tjstm6xI5FuJiUm8+eYf1Kv3Ac8881Py8qio8lYk8ihv/7cXVNU/zpxKqqqKSLzvYhm/cCilWSN3roDSDd3LYvzCmjX76N17KkuW7AbgyJFTJCWpjRWRx3lbKA6KSGU842eLyI3AXp+lMu46fRzWfQ6HNznzXb+zIpHPnT6dyCuv/MYrr/xGfHwSkZGF+eijTnToYB0+5gfeFoqBwFigpoj8A+wBbvNZKuOuNeNg3sMp88FZjYhr8rKYmDhatBjH2rX7AejfP4pXX72OwoVt3Or8wttWT5uB1p7Bi0RVj/g2lnHVqRjntdwVUKM7RF7jbh7jqiJFQqlTpzSnTyfyySdduPrqim5HMjksy0Ihzo2JIqp6RFVjRCRYRO4FHlfVur6PaHJM/AmYc5/TwR9AxeugcTp9OZk87+eft1G8eBgNG5YF4MMPOxIaGpTcb5PJXzJt9SQitwKHgY0i8rOIXA1sBm4GeudAPpOTdi2AvyfC0X+c+YgK7uYxOe7IkTjuu28abdp8xr33TiU+PhGAYsXCrEjkY1mdUQwDmqnqBk8Psr8Dd6jqV76PZnwm4RQkndVobfdC+OZ6Z7psU7jufSh9ec5nM66ZNm0D/fvPZPfuYxQoEEi3brXcjmT8RFaF4rSqbgBQ1SUist2KRC51Yq/Tkd+On2Bu/8zXrdYVyjTOmVzGdfv2nWDQoFlMnrwWgObNIxk7tgu1apVyOZnxF1kVitIiMijVfMHU86o6yjexTLbaMh2+63Lu8rNbMwUWgBs+hyodcyaXcV1CQhLNm49l69bDhIcHM2JEGx58sAmBgTY6oUmRVaH4FCiVybzJDQ6uc15DikJYSQguBG0/gnJN3c1lXBcUFMCTT17J11+vZ8yYTlSubJ34mXOJqrqd4bxERUXp0qU2sF66on+Hf9M5Njt+gq0zoMmTcPVrOZ/L+I2kJGXMmGUEBAj9+jmXF8/8DrBO/PI2EVmmqlEXsm2mZxQiMgR4X1WPZvD+1UAhVf3+Qj7cZKOEOPimrfOakaDwnMtj/M6mTQfp23c68+f/Q3h4MF261KBs2UJWIEyWsrr0tAmYIyJHgWXAfiAUqA40Bn4FXvJpQuOdxNNOkQgITn8goeCC0OD+nM9lXJeQkMTIkQsZNuwX4uISKFOmIKNHd6Bs2UJuRzO5RKaFQlW/Ab4RkVpAC6AcEAt8DQxU1RO+j2jOS1AoXPuO2ymMn/jrr7307j2N5cv3AHD33Q0YOfJ6ihe3cc2N97ztwmM9sF5Ewjzzsd5sJyLtgXeBQOATVX01nXW6A8/jdDj4l6r28i66MSYzqsqDD37P8uV7qFChCGPGdOL666u5HcvkQt4OXFQHGI9zRiEiEg3cq6rrMtkmEGewo7ZANLBERKal3kZEqgNPAy1U9bCIlL7g7yS/06Ss1zH5QmJiEoGBAYgIH37YiTFjlvHyy62JiLBO/MyF8bax9EfAUFWNVNVLgGeAMVls0xTYrKpbVfU0MAnoetY69wGjVfUwgKru8z66SXZyP4yJdDuFcdnx46d55JEfuPXWr5JbMtWtW5pRo26wImEuirfdjEeo6o9nZlR1roi8lcU2lwA7U81HA83OWucyABFZgHN56nlV/eHsHYlIP6AfQIUK1v/QOQ6udTr0A6h2o7tZjCt+/HEL/frNYPv2IwQGCmvW7KNevTJuxzJ5hLdnFNtF5GkRifR8DQH+yWKb9Nrcnf3QRhBOC6pWOONbfCIiRc/ZSHWMqkapalSpUva8X7L4WNg6E6J/c+Yjr4EbPnM3k8lRhw/H0rv3VNq1+4Lt24/QsGFZliy5z4qEyVbenlH0BoYD3+MUgPnAvVlsEw1cmmo+EtidzjqLVDUe2CYiG3AKxxIvc+VvC56BZW+nzAdY7575yXff/U3//jPZu/c4ISGBPP98Kx5/vDnBwYFuRzN5jLetng4C6TTOz9QSoLpnCNVdQE/g7BZN3+GcSYwXkZI4l6K2nufn5E//LkspEmUaQ+FK0PBBVyOZnPXHHzvZu/c4V11VgU8+6UyNGiXdjmTyqKyezH5LVR8XkSmce9kIVb05o21VNUFEBgKzce4/jFPVtSLyIrBUVad53msnIuuAROAJT1EymTl5AL5I9SR+8+ehaifX4picoars2nWMyMjCADz/fCtq1izJPfc0JCDAnq42vpNpX08i0lRVF4tIm/TeV9WffJYsA/m6r6fEeJg3CP76MGXZlS9C06ecnl9NnvXPP0e4//4ZrFmzj7VrB1CkSKjbkUwuczF9PWV6M1tVF3sma6nqT6m/ABvVJKftXZK2SFS7EZo/Z0UiD0tKUt57bzF16rzP7NlbOHkynrVr97sdy+Qz3rZ6Sm/Y0z7ZGcR4QROc1xJ1oPs86GxjSOVlGzYc4JprxvPQQ7M4cSKebt1qs379g1x55aVZb2xMNsrqHkUPnJvQlUXk21RvRQBHfBnMpOPMZcLQ4nBpK1ejGN8aM2YZgwbN4tSpRMqWLcTo0R24+WY7iTfuyKrV02LgIE7T1tGplh8DVvgqlMnAtzd4JnLXGCLm/FWoUIRTpxK5996GvPVWO4oVs078jHuy6j12G7ANmJszcUyGVoyGBE9fjGWbuJvFZLu4uAR+/nkbHTpUB6B9+2qsXt2funWt+zPjPq/uUYhIExFZJCIxIhInIqc8Y1QYX0o8DRu+gpUfwM8DU5Zf86Z7mUy2W7BgBw0bfkinThNYtCg6ebkVCeMvvH0y+33gDpyO/ZoC95D2qWvjCxu+hFl3pl12zzoQG/g+Lzh27BRDh/7E6NFLUIWaNUsSGGjPQxj/422hCFDVDSIS5Olu42MR+QP4jw+zmTjPs4clakP5K6HyDVDCbmjmBbNnb6Zfvxns2BFDUFAAQ4a04NlnryYkxNv/ksbkHG9/Kk+ISAHgLxF5BdgD2DiKvrZ8lPNa4Tpo/a67WUy2+eCDJQwY4Awz37hxOcaO7UKDBmVdTmVMxry9hnGPZ92BOF1tVAe6+SiTSTgFU2+CGE+3VwXLuZvHZKubbqpF+fIRvPbadSxa1NeKhPF73nYKeKajvjjgOQAROXtsCZNddi+Azd+lzEc95l4Wc9H27DnGO+8s4uWX2xAUFEDZsoXYsmUQoaF2mcnkDlk9cBcA3IIzCNFsVV3vGQd7KFAMqOf7iPnQivec12KXwR1LrYuOXEpVGT9+JY89NocjR+IoWTKcJ55oAWBFwuQqWf20fgJUweky/AMR2YQzyNDTqvq1j7PlT9G/weYpzvT146BAhLt5zAXZtu0w998/gx9/dE7Gb7ihGj171nU5lTEXJqtC0Qyor6qJIhIGHACqqeoe30fLp/aleuDdHqzLdRITkxg9eglPP/0TJ0/GU6JEGO++255eveohYk1fTe6UVaE4paqJAKoaKyIbrEj42IE1zmujh+ySUy709dfrePhhZ9j3Hj3qMGrUDZQuXdDlVMZcnKwKRU0RWe6ZFqCGZ14AVdXLfZouv0lKhNUfO9OBIe5mMRfk1lvr8O23f9OrV126dq3pdhxjskVWhcJuVuekpISU6Qb93cthvLZs2W4efvgH/u//bqZixaIEBAiTJ1vLcZO3ZNUp4JacCpLvJSXC/Ced6cACULSKu3lMpmJj43n++V94882FJCUpL774K2PHdnU7ljE+YW30/MW+5bDC8yR2eBl3s5hMzZ//D337TmPTpkMEBAiPPXYFL754rduxjPEZKxT+IjE+ZbrHr+7lMBk6evQUQ4bM5YMPnDHb69QpxdixXWjWLNLlZMb4lteFwtPXUwVV3ezDPPnP6nEw72FI8hSKcs2hSGV3M5l0bd9+hI8/Xk5wcABDh7Zk6NCWFCgQ6HYsY3zOq0IhIh2BkUABnGFRGwLDVPUmX4bL8+Y+CH+9n3aZDXHqV44ePUXhwk4LtPr1y/Dhhx1p2vQS6tWzy4Mm//C2U8AXcR6+OwKgqiuBar4Klecd2Qo7f4Hts1KW3bEcBp2Alq+4FsukUFUmT15DtWqj+OabdcnL+/QLTwWJAAAgAElEQVS53IqEyXe8vfQUr6pHznqy1AZuPl9JibB2PMzpm3b53WugZB1XIplz7d59jP79ZzJt2gYAvvpqHbfcUtvlVMa4x9tCsV5EugMBIlIZeBhY5LtYedTm79IWichrnI7/bDAiv6CqjB27gsGD5xAT41xyeuONtvTta8+VmvzN20IxEGc0uyTgW2A2Tg+y5nysGZsyfevPUMGaVPqLvXuPc/vt3/Lzz9sA6NTpMj74oCORkYVdTmaM+7wtFFVU9SngKV+GydMOroNtnnsSTZ6yIuFnChcOYfv2I5QsGc6oUe3p2bOudeJnjIe3heJ9ESkOfAlMVtW/fZgpb1rx35TpJk+4l8MkW7t2H5deWoTChUMIDw/m22+7U758BKVKWSd+xqTmVasnVW0JtAOOAf8TkRUiMsSnyXK7xNMwpTN8Wtv5+nuiszzqCQgr4W62fO706URefPFXGjX6iCFD5iYvb9CgrBUJY9Lh9QN3qroLGCkis4CngeHAq74KlmttmQ4/9oMTe9N5U6BqpxyPZFIsWbKLPn2msXr1PsC5gZ2UpAQE2GUmYzLi7QN31YEeQDfgODAZu1+Rvr8npS0SFa6D1u860yHFoFA5d3LlcydPxjNs2DxGjlxEUpJStWoxPvmkC61aVXI7mjF+z9szignAJKCLqu7wYZ7cLfYg/D3BmW41Eur2hpAi7mYyHDkSR1TUGLZsOUxAgDB4cHNeeOFawsOD3Y5mTK7gVaFQVRuTMyuaBGs+TZmv2sWKhJ8oWjSUZs0iCQ8PZuzYLjRpconbkYzJVTItFCIyUVVvE5EVpH0S20a4S23bLFg6EnZ4boyWawZFq7qbKZ+bMWMj5coVonHj8gB88EFHQkODrBM/Yy5AVmcUZ9px2pBdZzu6E1Z/Aoc3woZJad+78gV3Mhn27z/Bww//wMSJa6hXrzRLl/ajQIHA5I79jDHnL9Pmsaoa7Znso6pbUn8BfbLauYi0F5ENIrI5s+a0ItJNRFREos4vvkuSEmHazbDoxbRFImow9N0Gla53L1s+papMmLCaWrVGM3HiGsLDg+nduxGBgdaayZiL5e3N7Pac22VHx3SWJRORQGA00BaIBpaIyDRVXXfWehHAIOBPb0O77tDf8K8zeA1XPAdB4VDtRihR091c+VR09FH695/JjBkbAWjTpjJjxnSmSpViLiczJm/I6h7F/cADwGUisjzVWxHAsiz23RTYrKpbPfuaBHQF1p213nDgdWDweeR2z7FomNzSmS7VAFq86G6efC4+PpEWLcaxY0cMRYqE8NZb7ejdu5F1v2FMNsrqjOJL4CdgBJD60tExVd2XxbaXADtTzUfjjGmRTEQaAZeq6gwRybBQiEg/oB9AhQoVsvhYHxtfG04fc6aLXeZuFkNwcCD/+c/VTJ++kfff70j58hFuRzImz8nqHsVhz9CnTwA7PfcmygHdRCSrbjXT+5MuueWUiAQAbwOPZxVSVceoapSqRpUqVSqr1X0r/oTzWr8fdJzgbpZ8KCEhiTff/IP33lucvKx370ZMmdLDioQxPuLtPYrvgCYiUhX4DJiJ8xBeZv1RRAOXppqPBHanmo8A6gK/eC4TlAWmiUgXVV3qZS4XeOpfm9EQ4HUPKCYbrFr1L336TGPp0t2EhQVx6621KVOmkF1mMsbHvB0KNUlV44GbgXdU9SGcS0uZWQJUF5HKIlIA6AlMO/OmqsaoaklVraSqlXAGQvLvInFgLWii2ynynVOnEhg2bB6NG49h6dLdXHppYb75pjtlyhRyO5ox+YK3fxIniMitwJ3AjZ5lmfZ/oKoJIjIQZ5CjQGCcqq4VkReBpao6LbPt/c7epfB/ngfUJYD0r6yZ7LZoUTR9+kxj3br9AAwYEMWIEdfZcxHG5CBvC0VvYADwuqpu9QyHOjGrjVT1e+D7s5b9J4N1W3mZxR0/3p8y3WokBNgTvr6mqjzxxI+sW7ef6tWLM3ZsF1q2rOh2LGPyHW/7elojIoOAaiJSE6fZ68u+jeYHVGHZ23B4AxzzNOC6+nW4/GF3c+Vx8fGJBAcHIiKMGdOJzz77i//85xrCwqwTP2Pc4G034y2Bz4FdONdcyorInaq6wJfhXHdkM/x6VqOsql3dyZIPHDkSx+DBc9i79zjTp9+GiFCrVilGjLjO7WjG5GveXnp6G+hw5qlqEamFUzhyR5cbFyohznktWA6aD4MilaG4PTvhC1On/k3//jPZs+c4BQoEsm7dfurUKe12LGMM3heKAqm73lDV9Z6WTPlDWAlocH/W65nz9u+/xxk06Ae+/HItAM2bRzJ2bBdq1XL5eRljTDJvC8VyEfkI5ywC4HZghW8i+YkDa+Gz+m6nyNMmTFjNQw/N4tChWAoWDGbEiDYMGNCEwEBvW20bY3KCt4XiAZyO+57EuUcxH/ivr0K5KikRDq5LWyQqtnMvTx62du0+Dh2KpW3bKowZ05lKlYq6HckYk44sC4WI1AOqAlNU9XXfR3KRKnzeCA6sTlnWcSLU7OlepjwkKUnZvv1Icq+uzz13DfXrl6F79zr2dLUxfizTc3wRGYrTfcftwI8i0jtHUrllx89pi0TjR61IZJONGw/SqtV4WrQYx+HDsQCEhgbRo0ddKxLG+LmszihuB+qr6gkRKYXz8Nw438dyyfFdKdOPxltfTtkgISGJkSMXMmzYL8TFJVCmTEE2bTpE06Y2brUxuUVWvwlPqeoJAFXd7+nxNQ/zdG5b+04rEtngr7/20rv3NJYv3wPAPfc05K232lG8eJjLyYwx5yOr34ZVRORbz7QAVVPNo6o3+yxZTju0EX73DNgXZk0zL9aoUX/y+ONzSEhIomLFIowZ05l27aq6HcsYcwGyKhS3nDX/nq+CuG71x3B8N5RrBk2fcjtNrle7dikSE5N46KGmvPJKGwoVyj+P3RiT12RaKFT1p5wK4rrE085rzdsg3J4IPl/Hj59m9uzN3HJLbQCuu64KGzc+RLVqxV1OZoy5WHn8noPJCXPmbKFu3fe59dav+P33HcnLrUgYkzfYHVtzwQ4fjuWxx+YwfvxKABo1KmvjRBiTB51XoRCREFU95aswrjq63e0Eucq3367nwQe/Z+/e44SEBPL88614/PHmBAfbOB3G5DVeXXoSkaYishrY5JlvICJ5pwuPkwdgi2fAPbFfdFl5991F3HLLl+zde5yrrqrAX389wJAhV1mRMCaP8vYexSigE3AQQFX/Aq71Vagct/fPlOlqN2a8ngHgttvqUalSUUaP7sCvv95DjRol3Y5kjPEhbwtFgKr+c9ayxOwOk+OO74YFz8GUTs584YoQEeluJj+0ffsRHnroe+LjnX/y0qULsnHjQAYMaEJAgHW/YUxe5+09ip0i0hRQEQkEHgI2+i5WDkhKgI/O6kbiinSH8863kpKU0aMX8/TTP3HiRDyRkYV56qmrAOwykzH5iLeFoj/O5acKwL/AXM+y3Gv5qJTp0pdDs6FQPe88aH6x/v77AH37TmPBAmes8Ftvrc099zR0OZUxxg1eFQpV3Qfk/m5UkxJh+q3OeBOHNzjLGjwArf9rfTt5xMcn8sYbf/DCC79y+nQiZcsW4v33O3DTTbXcjmaMcYlXvx1F5GOSe8xLoar9sj2RLx3ZDJunpMwHhkCzZ6xIpPLNN+t55pmfAejTpxFvvNGWYsWsEz9j8jNvf0POTTUdCtwE7Mz+OD628EXntXAluHmW01VHmD09rKrJY0J0716HH37YzB131Oe666q4nMwY4w+8vfQ0OfW8iHwO/OiTRL50PNp5LVIJStR0NYq/+P33HQwaNIuvv+5OlSrFCAgQxo+3JsLGmBQX2tdTZaBidgbJUc2fdzuB644dO8XAgd/TsuWnrFixl1df/d3tSMYYP+XtPYrDpNyjCAAOAUN8Fcr41g8/bOb++2ewY0cMQUEBPP30VTzzTEu3Yxlj/FSWhUKci9cNgDPjhCap6jk3tv3aus/hwFo4stXtJK46dCiWRx+dzWef/QVA48blGDeuK/Xrl3E5mTHGn2VZKFRVRWSKqjbOiUDZ7lg0zLor7bKQIu5kcdmePceYOHE1oaFBvPhiKx59tDlBQdbTvDEmc962elosIper6nKfpvGFTZ6RW0NLQNRgKFwBSjVwN1MOOnjwJMWLhyEi1KlTmnHjutKs2SVUr17C7WjGmFwi0z8nReRMIbkKp1hsEJHlIrJCRPy/aJz4F+Y97EwHhUCzIVCrF0je759IVfn00xVUq/ZfJk9em7z8jjvqW5EwxpyXrM4oFgOXA7mzveTOeSnTN//gXo4ctm3bYfr1m8Hcuc49mVmzNtOzZ12XUxljcqusCoUAqOqWHMiS/c7cc6/QBkrVczdLDkhMTOK99xYzdOjPnDwZT4kSYbz7bnt69cr737sxxneyKhSlROSxjN5U1ZHZnMc3wkq5ncDndu06yq23fsXChc5DhT171uXdd9tTunRBl5MZY3K7rJq8BAKFgIgMvjIlIu099zU2i8g5z12IyGMisk5EVonITyKSex/ic1nx4mEcOHCS8uUjmDq1JxMn3mJFwhiTLbI6o9ijqi9eyI4941aMBtoC0cASEZmmqutSrbYCiFLVkyLSH3gd6HEhn5eug2uybVf+aNmy3VStWpyiRUMJCwvmu+96Ur58BEWLhrodzRiTh2R1RnExzYOaAptVdauqngYmAV1Tr6Cq81T1pGd2EZC9w8v9+YrzGpC3BtmJjY3nqad+pGnTT3jyyZQut2rXLmVFwhiT7bI6o2hzEfu+hLQ9zEYDzTJZvw8wK703RKQf0A+gQoUK3icoEAGnj8HlD3u/jZ/79dft9O07nc2bDxEQIEREFEjT+6sxxmS3TAuFqh66iH2n95sr3a4/ROQOIAq4JoMcY4AxAFFRUefffUixGue9ib85evQUTz31Ix9+uAyAOnVKMXZsF5o1szG+jTG+5csRe6KBS1PNRwK7z15JRK4DngGuUdVTPsyTax0+HEuDBh+yc+dRgoMDGDq0JUOHtqRAgbx1Sc0Y4598WSiWANVFpDJOh4I9gV6pVxCRRsBHQHvPcKvZZ+evzmWnPKBYsTBat67MunX7GTu2C/XqWSd+xpic47NCoaoJIjIQmI3TzHacqq4VkReBpao6DXgDp/ntV55r7DtUtUu2BNj0Tcp0cO5qJqqqfPnlWipWLMoVVziXlkaP7kBoaBCBgdaJnzEmZ/l0sGhV/R74/qxl/0k1fZ1PPvjAGljxX2f6qldyVaunXbuOMmDA90ybtoFatUqyYsX9hIQEUbBgAbejGWPyKZ8WClds+ApmdE+Zr9zBvSznQVX55JPlDB78I0ePnqJw4RAeeeQKgoNzT5EzxuRNea9Q/OZ5ADysJFz5IpT2/y7Ft2w5xH33TWfevO0AdOp0GR980JHIyMLuBjPGGPJSoTiyBTZ8Ccedvo7ovQlCi7qbyQvx8Ym0avU/oqOPUrJkOP/97w306FHHnoswxviNvFEoEuJgbLWU+YBgCAxxL895CA4O5OWXWzNnzhbeeac9JUuGux3JGGPSyBuFYtadKdOVO0D9+yE4zL08mTh9OpERI34jIiKExx5rDsBddzXgrrv8/xKZMSZ/yhuFYst05zW8NNw0w29HsFu8eBd9+kxjzZp9hIYGceed9SlVKnc13TXG5D95o1F+kKcjvDtX+GWROHkynsGD59C8+VjWrNlHtWrFmTXrdisSxphcIW+cUZwR5H/X9+fN20bfvtPZuvUwAQHCE09cyfPPtyI8PNjtaMYY45W8VSj8jKrywgu/snXrYerVK824cV2JiirvdixjjDkvubtQJMTB7oWQGO92kjTi4hIIDQ1CRPj4485MnryWJ59sYZ34GWNypdx9j2LuA/BVa0jwjH3kclcd+/efoFevb+jSZSKqTm/o1auX4Nlnr7YiYYzJtXL3GcUxz8N1pRpC1S7OQEUuUFUmTlzDoEGzOHgwlvDwYP7++wC1apVyJY8xxmSn3Fsots+BHT8509e8ARV9079gVnbujKF//5nMnLkJgDZtKjNmTGeqVCnmSh5jjMluubNQ/PkK/P5MynyIO111jB27nEcfnc2xY6cpUiSEkSOv5957G1r3G8aYPCX3FYqk+LRF4obPoUxjV6Ls3HmUY8dO07VrDd5/vyPly7tz6csYY3wp9xUKTUqZvmctlKidYx+dkJDE5s2HqFmzJABDh7akceNydOp0mZ1F5HHx8fFER0cTFxfndhRjMhUaGkpkZCTBwdn3rFbuKxRnFK6Uo0Vi1ap/6dNnGjt2xLBu3QBKlAinQIFAOneukWMZjHuio6OJiIigUqVK9keB8VuqysGDB4mOjqZy5crZtt/c3Tw2B5w6lcB//jOPxo3HsHTpbkJCAvnnnxi3Y5kcFhcXR4kSJaxIGL8mIpQoUSLbz3xz7xlFDli0KJo+faaxbt1+AAYMiGLEiOsoXDh3dGFuspcVCZMb+OLn1ApFBt54YwFPPTUXVahevThjx3ahZcuKbscyxpgcZ5eeMtCkySUEBgYwZEgL/vrrASsSxnWBgYE0bNiQunXr0rlzZ44cOZL83tq1a2ndujWXXXYZ1atXZ/jw4cm9AwDMmjWLqKgoatWqRc2aNRk8eLAb30KmVqxYQd++fd2OkakRI0ZQrVo1atSowezZs9Ndp2XLljRs2JCGDRtSvnx5brzxRgDeeOON5OV169YlMDCQQ4cOJW+XmJhIo0aN6NSpU/Ky9957j2rVqiEiHDhwIHn5jBkzGDZsmI++y3Soaq76atywruqbqI6ppNnp8OFY/fzzv9Is27HjSLZ+hsm91q1b53YELViwYPL0XXfdpS+99JKqqp48eVKrVKmis2fPVlXVEydOaPv27fW9995TVdXVq1drlSpVdP369aqqGh8fr6NHj87WbPHx8Re9j27duunKlStz9DPPx9q1a7V+/foaFxenW7du1SpVqmhCQkKm29x88836v//975zl06ZN02uvvTbNsrfeektvu+027dixY/Ky5cuX67Zt27RixYq6f//+5OVJSUnasGFDPXHiRLqfm97PK7BUL/D3rl16Ar777m8GDJjJnj3HufTSwlxzTSUALr20iLvBjH96y0f3Kh7XrNfxaN68OatWrQJgwoQJtGjRgnbt2gEQHh7Oe++9R6tWrXjwwQd5/fXXeeaZZ6hZsyYAQUFBDBgw4Jx9Hj9+nIceeoilS5ciIgwbNoxbbrmFQoUKcfz4cQC+/vprZsyYwfjx47nnnnsoXrw4K1asoGHDhkyZMoWVK1dStKjzAGy1atVYsGABAQEBPPDAA+zYsQOAd955hxYtWqT57GPHjrFq1SoaNHBGely8eDGPPPIIsbGxhIWF8emnn1KjRg3Gjx/PzJkziYuL48SJE/z888+88cYbfPnll5w6dYqbbrqJF154AYAbb7yRnTt3EhcXx8MPP0y/fv28Pr7pmTp1Kj179iQkJITKlStTrVo1Fi9eTPPmzdNd/9ixY/z88898+umn57w3ceJEbrvttuT56OhoZs6cyTPPPMPIkSOTlzdq1CjdfYsIrVq1YsaMGXTv3v2ivi9v5OtC8e+/x3nooVl89dU6AJo3j6RMmUIupzImc4mJifz000/06dMHcC47NW6c9qHTqlWrcvz4cY4ePcqaNWt4/PHHs9zv8OHDKVKkCKtXrwbg8OHDWW6zceNG5s6dS2BgIElJSUyZMoV7772XP//8k0qVKlGmTBl69erFo48+ylVXXcWOHTu4/vrrWb9+fZr9LF26lLp16ybP16xZk/nz5xMUFMTcuXMZOnQo33zzDQALFy5k1apVFC9enDlz5rBp0yYWL16MqtKlSxfmz5/P1Vdfzbhx4yhevDixsbE0adKEW265hRIlSqT53EcffZR58+ad83317NmTIUOGpFm2a9currjiiuT5yMhIdu3aleGxmTJlCm3atKFw4cJplp88eZIffviB9957L3nZI488wuuvv86xY8cy3N/ZoqKi+O2336xQ+Iqq8sUXq3jkkdkcOhRLwYLBjBjRhgEDmhAYaLdtTBbO4y//7BQbG0vDhg3Zvn07jRs3pm3btoDz85xRS5fzaQEzd+5cJk2alDxfrFjW/ZXdeuutBAY6PSP36NGDF198kXvvvZdJkybRo0eP5P2uW7cueZujR49y7NgxIiJSejLYs2cPpUqldKIZExPD3XffzaZNmxAR4uNThhJo27YtxYsXB2DOnDnMmTMn+S/v48ePs2nTJq6++mpGjRrFlClTANi5cyebNm06p1C8/fbb3h0cSHPP54zMju/EiRPTvecyffp0WrRokfw9zJgxg9KlS9O4cWN++eUXr/OULl2a3bt3e73+xch9hSL2QNbrZGHkyIUMHvwjAG3bVmHMmM5UquROf1HGeCssLIyVK1cSExNDp06dGD16NIMGDaJOnTrMnz8/zbpbt26lUKFCREREUKdOHZYtW5Z8WScjGRWc1MvObp9fsGDKcL7Nmzdn8+bN7N+/n++++45nn30WgKSkJBYuXEhYWFim31vqfT/33HNce+21TJkyhe3bt9OqVat0P1NVefrpp7n//vvT7O+XX35h7ty5LFy4kPDwcFq1apXuswXnc0YRGRnJzp07k+ejo6MpXz79gcgOHjzI4sWLkwtVapMmTUpz2WnBggVMmzaN77//nri4OI4ePcodd9zBF198ke6+z4iLi8v0mGan3Pfnc/wJ5/Uixp64++6G1KhRgvHjuzJ79h1WJEyuUqRIEUaNGsWbb75JfHw8t99+O7///jtz584FnDOPQYMG8eSTTwLwxBNP8Morr7Bx40bA+cWd+jr4Ge3atUtzOeTMpacyZcqwfv365EtLGRERbrrpJh577DFq1aqV/Nf72ftduXLlOdvWqlWLzZs3J8/HxMRwySWXADB+/PgMP/P6669n3LhxyfdQdu3axb59+4iJiaFYsWKEh4fz999/s2jRonS3f/vtt1m5cuU5X2cXCYAuXbowadIkTp06xbZt29i0aRNNmzZNd79fffUVnTp1IjQ0NM3ymJgYfv31V7p27Zq8bMSIEURHR7N9+3YmTZpE69atsywS4Fz2S325zpdyb6FoM9rrTTZsOECfPlM5fToRgJIlw1m7dgB33209vZrcqVGjRjRo0IBJkyYRFhbG1KlTeemll6hRowb16tWjSZMmDBw4EID69evzzjvvcNttt1GrVi3q1q3Lnj17ztnns88+y+HDh6lbty4NGjRI/kv71VdfpVOnTrRu3Zpy5cplmqtHjx588cUXyZedAEaNGsXSpUupX78+tWvX5sMPPzxnu5o1axITE5N8jf7JJ5/k6aefpkWLFiQmJmb4ee3ataNXr140b96cevXq0a1bN44dO0b79u1JSEigfv36PPfcc2nuLVyoOnXq0L17d2rXrk379u0ZPXp08mW3Dh06pLkMdPZZwxlTpkyhXbt2ac6KMjNq1CgiIyOJjo6mfv36aS5lzZs3j44dO17kd+UdSe+6mz+LulR06SNAtx+zHIMiISGJN9/8g+ef/4VTpxIZMaINQ4ZclTNBTZ6yfv16atWq5XaMPO3tt98mIiLC75+l8Af//vsvvXr14qeffkr3/fR+XkVkmapGXcjn5b4zCoBLr4XyV2a6ysqVe2nW7BOefvonTp1K5J57GtKvnzvdkRtjsta/f39CQqx7HG/s2LGDt956K8c+L/fdzAbo/BUEh6f7VlxcAsOH/8prry0gMVGpWLEIY8Z0pl27qjkc0hhzPkJDQ7nzzjvdjpErNGnSJEc/L3cWikxMnfo3r7zyOyIwaFBTXn65DYUKFXA7lskDMmuGaoy/8MXthDxRKJKSlIAA5z9w9+51+OWX7dxxR31atKjgcjKTV4SGhnLw4EHratz4NfWMR3F2a6uLlTtvZm88AGFO07s5c7bwyCM/MHVqT6pXL5HF1sZcGBvhzuQWGY1wdzE3s3PtGcWhQ7E8/vgcxo932mS//fYi3n8/Z5qKmfwnODg4W0cMMyY38WmrJxFpLyIbRGSziJzzBIuIhIjIZM/7f4pIJW/2+813m6ldezTjx68kJCSQV19tw6hRN2R3fGOMMfjw0pOIBAIbgbZANLAEuE1V16VaZwBQX1UfEJGewE2q2iPdHXoUCy+uR2IfBuCqqyrwySedqVGjpE++B2OMySv89TmKpsBmVd2qqqeBSUDXs9bpCvzPM/010EayuFMYExtKoULBjB7dgV9/vceKhDHG+Jgvzyi6Ae1Vta9n/k6gmaoOTLXOGs860Z75LZ51Dpy1r37Amc7k6wJrfBI69ykJXHwviXmDHYsUdixS2LFIUUNVI7Je7Vy+vJmd3pnB2VXJm3VQ1THAGAARWXqhp095jR2LFHYsUtixSGHHIoWILL3QbX156SkauDTVfCRwdufpyeuISBBQBDiEMcYYv+HLQrEEqC4ilUWkANATmHbWOtOAuz3T3YCfNbc92GGMMXmczy49qWqCiAwEZgOBwDhVXSsiL+IM8j0NGAt8LiKbcc4kenqx6zG+ypwL2bFIYccihR2LFHYsUlzwsch1T2YbY4zJWbmzm3FjjDE5xgqFMcaYTPltofBV9x+5kRfH4jERWSciq0TkJxGp6EbOnJDVsUi1XjcRURHJs00jvTkWItLd87OxVkQm5HTGnOLF/5EKIjJPRFZ4/p90cCOnr4nIOBHZ53lGLb33RURGeY7TKhG53Ksdq6rffeHc/N4CVAEKAH8Btc9aZwDwoWe6JzDZ7dwuHotrgXDPdP/8fCw860UA84FFQJTbuV38uagOrACKeeZLu53bxWMxBujvma4NbHc7t4+OxdXA5cCaDN7vAMzCeYbtCuBPb/brr2cUPun+I5fK8lio6jxVPemZXYTzzEpe5M3PBcBw4HUgL/cJ7s2xuA8YraqHAVR1Xw5nzCneHAsFCnumi3DuM3e42+kAAAYuSURBVF15gqrOJ/Nn0boCn6ljEVBURMpltV9/LRSXADtTzUd7lqW7jqomADFAXhyQwptjkVofnL8Y8qIsj4WINAIuVdUZORnMBd78XFwGXCYiC0RkkYi0z7F0OcubY/E8cIeIRAPfAw/lTDS/c76/TwD/HY8i27r/yAO8/j5F5A4gCrjGp4nck+mxEJEA4G3gnpwK5CJvfi6CcC4/tcI5y/xNROqq6hEfZ8tp3hyL24DxqvqWiDTHeX6rrqom+T6eX7mg35v+ekZh3X+k8OZYICLXAc8AXVT1VA5ly2lZHYsInE4jfxGR7f/f3v2FWFVFcRz//grLMUGQwYfoYUjL0jJJDcmHMsuyUgwGpmGyJrIoishiBCkswofMXhQRTR/0wcQ/mP+KVCItarQRUWeS/mpYIGUFImUQ0+phr8GT3Tn3aI7embs+MMzcO+fPPhvmrLvXnrM2KQe7pY9OaBf9G9lsZn+Z2VHgK1Lg6GuK9MUTwDoAM2sF+pMKBlabQveTs1VqoIjyH2eU7QtPtywjBYm+moeGMn1hZifNrNbM6sysjjRfM83MzrsYWgUr8jeyifSPDkiqJaWijlzUVl4cRfriGDAJQNKNpEBx4qK2sjJsAR71/34aD5w0s+PldqrI1JP1XPmPXqdgXywABgLrfT7/mJlNu2SN7iEF+6IqFOyL7cBkSYeBTqDFzH69dK3uGQX74iVguaRZpFRLc1/8YClpDSnVWOvzMa8C/QDMbClpfuZ+4FvgD+DxQsftg30VQgjhAqrU1FMIIYQKEYEihBBCrggUIYQQckWgCCGEkCsCRQghhFwRKELFkdQp6UDmqy5n27ruKmWe4zl3efXRg17yYvh5HGO6pBGZ16/7g5D/t20rJR31vjgoaVKBfZolXZ15vSLbthDORUU+RxGq3mkzG30JzttkZvskPUV6NuVcn0WZDmwDDgOY2dwL2LYWM9sgaSKpEmq5J6ybgQ78qVszm3kB2xKqTIwoQq/gI4dPJO33r9tLbDNS0uf+yfuQpOv8/Ucy7y+TdHmZ030MDPN9J/kaBu1e6/9Kf/8NnVkD5C1vzzRggZ9nqI8E6iVNkbQu0847JW31nydLavVrWi9pYJm2tZIp4iZprqQ2SR2S3vYnbutJNb9We1tqfMQ01vdp9OvpkDS/zPlCiEARKlJNJu30rr/3M3CPmd0KNACLSuz3NLDQRyNjgR+9XEMDMMHf7wSaypx/KtAuqT+wEmgws5tJI/BnJA0GHgJGmtkoYJ6ZfUYqj9BiZqPN7LvM8XYC4yVd5a8bgLVeVuMV4G6/rn3Ai2Xadh+pNEeXxWY2zsxuAmqAB81sgx+rydtyumtjT0fNB+4CRgPjJE0vc85Q5SL1FCpRqdRTP2CxpK6b/fUl9msFXpZ0DbDRzL7xfP4YoM3Lm9SQgk4pqyWdBr4nlaEeDhw1s6/996uAZ4HFpLUuVkh6j5Ru6paXmPgAmCppA/AAMJtU5XcE8Km37Qq/hlIWSHoTGEIqdthloqTZwABgMPAFsDWnOeOAXWZ2AkDSatJiN5ty9glVLgJF6C1mAT8Bt5BGwv9ZlMjM3pG0l3Qj3i5pJqms8iozm1PgHE3ZAoKSSq5v4jf+20hF5h4GniN9Qs+zlhRkfgPazOyUUnTYaWaNBdrWAmwEnicFrDE+4llCWsXvB0mvkYrd5emLi3uFHhapp9BbDAKO+/oBM0jF3/5F0rXAETNbREoDjQI+BOolDfFtBqv4muJfAnWShvnrGcBun0cYZGbvAy+QUjgAp0ilzkvZRVqi8klS0IBU3XZC1/ElDZBUaqQEgF/7QuAySfdyJij84m2qz2zeXVv2AndIqvW5mkZgd3fnDAEiUITeYwnwmKQ9pLTT7yW2aQA6JB0AbiAt+XiYNA+wQ9Ih0nxB2aUfAczsT1J1zfWS2oG/gaWkG/A2P95u0mgH0hKcLT75PfSsY3WSUlRT/Due/mkG1vix9ni789pkwDxgti9AtBxoJ6WO2jKbrgSWdk1mZ/Y/DswBPiKtLb3fzDYX6Y9QvaJ6bAghhFwxogghhJArAkUIIYRcEShCCCHkikARQgghVwSKEEIIuSJQhBBCyBWBIoQQQq5/ACldIpmh6XpHAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1c208c6780>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure()\n",
    "lw = 2\n",
    "plt.plot(fpr, tpr, color='darkorange', lw=lw, label='ROC curve (area = %0.4f)' % roc_auc)\n",
    "plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--')\n",
    "plt.xlim(0, 1)\n",
    "plt.ylim(0, 1)\n",
    "plt.xlabel('False Postive Ratio')\n",
    "plt.ylabel('True Postive Ratio(Recall)')\n",
    "plt.title('ROC curve')\n",
    "plt.legend(loc='lower right')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 6. &#44208;&#51221;&#44221;&#44228;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 253,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Threshold 0.0 Train accuracy: 0.3012\n",
      "Threshold 0.1 Train accuracy: 0.4625\n",
      "Threshold 0.2 Train accuracy: 0.63\n",
      "Threshold 0.3 Train accuracy: 0.6975\n",
      "Threshold 0.4 Train accuracy: 0.735\n",
      "Threshold 0.5 Train accuracy: 0.7438\n",
      "Threshold 0.6 Train accuracy: 0.74\n",
      "Threshold 0.7 Train accuracy: 0.7275\n",
      "Threshold 0.8 Train accuracy: 0.715\n",
      "Threshold 0.9 Train accuracy: 0.7\n"
     ]
    }
   ],
   "source": [
    "for i in np.arange(0, 1, 0.1):\n",
    "    train_pred['y_pred'] = 0\n",
    "    train_pred.loc[train_pred['probs'] > i, 'y_pred'] = 1\n",
    "    acc = round(accuracy_score(train_pred['class'], train_pred['y_pred']), 4)\n",
    "    print(\"Threshold\", round(i,1), \"Train accuracy:\",acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 254,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Threshold 0.0 Test accuracy: 0.295\n",
      "Threshold 0.1 Test accuracy: 0.455\n",
      "Threshold 0.2 Test accuracy: 0.625\n",
      "Threshold 0.3 Test accuracy: 0.675\n",
      "Threshold 0.4 Test accuracy: 0.725\n",
      "Threshold 0.5 Test accuracy: 0.735\n",
      "Threshold 0.6 Test accuracy: 0.735\n",
      "Threshold 0.7 Test accuracy: 0.72\n",
      "Threshold 0.8 Test accuracy: 0.705\n",
      "Threshold 0.9 Test accuracy: 0.71\n"
     ]
    }
   ],
   "source": [
    "for i in np.arange(0, 1, 0.1):\n",
    "    test_pred['y_pred'] = 0\n",
    "    test_pred.loc[test_pred['probs'] > i, 'y_pred'] = 1\n",
    "    acc = round(accuracy_score(test_pred['class'], test_pred['y_pred']), 4)\n",
    "    print(\"Threshold\", round(i,1), \"Test accuracy:\",acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 261,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Test prediction\n",
    "test_pred = pd.DataFrame({\n",
    "    'probs': drop_logistic_model.predict(sm.add_constant(test_x.drop(unnecesarries, axis=1))),\n",
    "    'class': test_y\n",
    "})\n",
    "\n",
    "test_pred['y_pred'] = 0\n",
    "test_pred.loc[test_pred['probs'] > 0.4, 'y_pred'] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 262,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Confusion matrix:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>Actual</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>All</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Predict</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>106</td>\n",
       "      <td>20</td>\n",
       "      <td>126</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>35</td>\n",
       "      <td>39</td>\n",
       "      <td>74</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>All</th>\n",
       "      <td>141</td>\n",
       "      <td>59</td>\n",
       "      <td>200</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Actual     0   1  All\n",
       "Predict              \n",
       "0        106  20  126\n",
       "1         35  39   74\n",
       "All      141  59  200"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "print('Test Confusion matrix:')\n",
    "display(pd.crosstab(test_pred['y_pred'], test_pred['class'], rownames=['Predict'], colnames=['Actual'], margins=True))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}