Lecture17_Creating_Random_Forest.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Simple version Random Forest &#49373;&#49457;"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "(1) *Sklearn moons* &#54984;&#47144; &#45936;&#51060;&#53552;&#47484; 1,000&#44060; &#49373;&#49457;&#54633;&#45768;&#45796;. &#44033;&#44033;&#51032; &#54984;&#47144; &#45936;&#51060;&#53552;&#45716; &#47924;&#51089;&#50948;&#47196; &#49440;&#53469;&#46108; 10,000&#44060;&#51032; &#49368;&#54540;&#51012; &#45812;&#44256; &#51080;&#46020;&#47197; &#54633;&#45768;&#45796;."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "(2) &#47676;&#51200; &#53944;&#47532;&#48516;&#47448;&#44592;&#47484; &#54617;&#49845;&#54620; &#54980; &#53580;&#49828;&#53944; &#49483;&#50640;&#49436; &#52572;&#51333; &#49457;&#45733;&#51012; &#54869;&#51064;&#54633;&#45768;&#45796;."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "(3) &#44033; &#53580;&#49828;&#53944; &#49368;&#54540;&#50640; &#45824;&#54644; 1,000&#44060;&#51032; &#44208;&#51221; &#53944;&#47532; &#50696;&#52769;&#51012; &#47564;&#46308;&#44256; &#45796;&#49688;&#47196; &#45208;&#50728; &#50696;&#52769;&#47564; &#52712;&#54633;&#45768;&#45796;. &#44536;&#47084;&#47732; &#53580;&#49828;&#53944; &#49464;&#53944;&#50640; &#45824;&#54620; **&#45796;&#49688;&#44208; &#50696;&#52769;(majority vote prediction)** &#51060; &#49373;&#49457;&#46121;&#45768;&#45796;."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "(4) &#53580;&#49828;&#53944; &#49464;&#53944;&#50640;&#49436; &#51060; &#50696;&#52769;&#51012; &#54217;&#44032;&#54633;&#45768;&#45796;."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. &#45936;&#51060;&#53552; &#47196;&#46377; "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import make_moons\n",
    "\n",
    "X, y = make_moons(n_samples=10000, noise=0.4, random_state=42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. &#54617;&#49845; &#48143; &#53580;&#49828;&#53944; &#49464;&#53944; &#44396;&#48516;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X_train, X_test, y_train, y_test = \\\n",
    "    train_test_split(X, y, test_size=0.3, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "&#54617;&#49845; &#49483; : (7000, 2)\n",
      "&#53580;&#49828;&#53944; &#49483; : (3000, 2)\n"
     ]
    }
   ],
   "source": [
    "print('&#54617;&#49845; &#49483; :', X_train.shape)\n",
    "print('&#53580;&#49828;&#53944; &#49483; :', X_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.84441684,  1.2423668 ],\n",
       "       [ 0.16320378,  0.82374035],\n",
       "       [ 1.24805333,  0.05579093],\n",
       "       [ 0.35703881, -0.01696228],\n",
       "       [ 0.69022909, -0.25945021]])"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 1, 1, 1])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Hyperparameter &#44160;&#49353;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "params = {\n",
    "    'max_leaf_nodes': [2, 3, 4, 5, 6, 7], \n",
    "    'min_samples_split': [2, 3, 4],\n",
    "    'max_depth': [3, 5, 10, 15, 20]\n",
    "}\n",
    "grid_search_cv = GridSearchCV(\n",
    "    DecisionTreeClassifier(random_state=42), \n",
    "    params, \n",
    "    n_jobs=-1, \n",
    "    verbose=1, \n",
    "    cv=3\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 90 candidates, totalling 270 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done 270 out of 270 | elapsed:    0.7s finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=3, error_score='raise',\n",
       "       estimator=DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=None,\n",
       "            max_features=None, max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=1, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, presort=False, random_state=42,\n",
       "            splitter='best'),\n",
       "       fit_params=None, iid=True, n_jobs=-1,\n",
       "       param_grid={'max_leaf_nodes': [2, 3, 4, 5, 6, 7], 'min_samples_split': [2, 3, 4], 'max_depth': [3, 5, 10, 15, 20]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring=None, verbose=1)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_search_cv.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=3,\n",
       "            max_features=None, max_leaf_nodes=4, min_impurity_decrease=0.0,\n",
       "            min_impurity_split=None, min_samples_leaf=1,\n",
       "            min_samples_split=2, min_weight_fraction_leaf=0.0,\n",
       "            presort=False, random_state=42, splitter='best')"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# &#49457;&#45733;&#51060; &#51339;&#51008; &#54616;&#51060;&#54140;&#54028;&#46972;&#48120;&#53552;&#47484; &#52286;&#45716;&#45796;.\n",
    "grid_search_cv.best_estimator_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. &#45800;&#51068; &#53944;&#47532; &#49457;&#45733; &#52404;&#53356;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.856"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "y_pred = grid_search_cv.predict(X_test)\n",
    "accuracy_score(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5. &#47004;&#45924; &#54252;&#47112;&#49828;&#53944; &#47784;&#45944;&#51012; &#49373;&#49457;&#54616;&#44592; &#50948;&#54616;&#50668; &#54617;&#49845;&#49483; &#49368;&#54540;&#46308; &#49373;&#49457;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import ShuffleSplit\n",
    "\n",
    "# &#52509; 1000&#44060;&#51032; tree\n",
    "n_trees = 1000\n",
    "n_instances = 100\n",
    "\n",
    "mini_sets = []\n",
    "\n",
    "rs = ShuffleSplit(\n",
    "    n_splits=n_trees, \n",
    "    test_size=len(X_train) - n_instances, \n",
    "    random_state=42\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1000"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# &#49368;&#54540; &#44060;&#49688; &#54869;&#51064;\n",
    "len(list(rs.split(X_train)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "&#54617;&#49845; &#49464;&#53944; : 100\n",
      "&#53580;&#49828;&#53944; &#49464;&#53944; : 6900\n"
     ]
    }
   ],
   "source": [
    "# &#54617;&#49845; &#49464;&#53944;&#45716; &#49324;&#50857;&#54620;&#45796;.\n",
    "print('&#54617;&#49845; &#49464;&#53944; :', len(list(rs.split(X_train))[0][0]))\n",
    "# &#53580;&#49828;&#53944; &#49464;&#53944;&#45716; &#49324;&#50857;&#54616;&#51648; &#50506;&#51012; &#44163;&#51060;&#45796;.\n",
    "print('&#53580;&#49828;&#53944; &#49464;&#53944; :', len(list(rs.split(X_train))[0][1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "for mini_train_index, _ in rs.split(X_train):\n",
    "    X_mini_train = X_train[mini_train_index]\n",
    "    y_mini_train = y_train[mini_train_index]\n",
    "    mini_sets.append((X_mini_train, y_mini_train))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 6. 1000&#44060;&#51032; &#44060;&#48324; &#47784;&#45944; &#54617;&#49845;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8281106666666667"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "from sklearn.base import clone\n",
    "\n",
    "# 1000&#44060;&#51032; &#54617;&#49845;&#53944;&#47532;\n",
    "forest = [clone(grid_search_cv.best_estimator_) \\\n",
    "          for _ in range(n_trees)]\n",
    "\n",
    "accuracy_scores = []\n",
    "\n",
    "# 1000&#44060;&#51032; &#53944;&#47532;&#50640; &#45824;&#54644;&#49436; &#54617;&#49845;&#54620;&#45796;.\n",
    "for tree, (X_mini_train, y_mini_train) in zip(forest, mini_sets):\n",
    "    # &#44060;&#48324; &#47784;&#45944;&#51012; &#54617;&#49845;&#54620;&#45796;.\n",
    "    tree.fit(X_mini_train, y_mini_train)\n",
    "    # &#54617;&#49845;&#54620; &#44060;&#48324; &#47784;&#45944;&#51032; &#50696;&#52769;&#51012; &#44396;&#54620;&#45796;.\n",
    "    y_pred = tree.predict(X_test)\n",
    "    accuracy_scores.append(accuracy_score(y_test, y_pred))\n",
    "\n",
    "np.mean(accuracy_scores)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "&#45800;&#51068; &#53944;&#47532;&#47484; &#51060;&#50857;&#54664;&#51012; &#46412;, 7000&#44060;&#51032; &#49368;&#54540;&#50640;&#49436; &#54617;&#49845;&#51012; &#54616;&#50688;&#44592; &#46412;&#47928;&#50640;, 100&#44060;&#51032; &#49368;&#54540;&#50640;&#49436; &#54617;&#49845;&#51012; &#54664;&#51012; &#46412;&#48372;&#45796; &#51221;&#54869;&#46020;&#44032; &#45458;&#45796;.\n",
    "\n",
    "100&#44060;&#51032; &#49368;&#54540;&#50640;&#49436; &#54617;&#49845;&#54620;, &#44060;&#48324; &#54617;&#49845;&#44592; 1000&#44060;&#51032; &#49457;&#45733; &#54217;&#44512;&#44050;&#51060; &#45230;&#51008; &#44163;&#51012; &#54869;&#51064; &#54624; &#49688; &#51080;&#45796;."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 7. 1000&#44060;&#51032; &#44060;&#48324; &#47784;&#45944;&#50640;&#49436; &#50696;&#52769;&#44050;&#51012; &#50619;&#44256; &#50521;&#49345;&#48660;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "Y_pred = np.empty([n_trees, len(X_test)], dtype=np.uint8)\n",
    "\n",
    "# 1000&#44060;&#51032; &#44060;&#48324; &#47784;&#45944;&#50640;&#49436; &#50696;&#52769;&#44050;&#51012; &#44033;&#44033; &#44396;&#54620;&#45796;.\n",
    "for tree_index, tree in enumerate(forest):\n",
    "    Y_pred[tree_index] = tree.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.stats import mode\n",
    "\n",
    "# Majority vote\n",
    "y_pred_majority_votes, n_votes = mode(Y_pred, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1, 1, 0, ..., 0, 1, 1]], dtype=uint8)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred_majority_votes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8613333333333333"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(y_test, y_pred_majority_votes.reshape([-1]))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}