07-ensembles/code/07-04_gradient-boosting.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# STAT451: Machine Learning -- L07: Ensemble Methods Part 2/3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "STAT 451: Intro to Machine Learning (Fall 2021)  \n",
    "Instructor: Sebastian Raschka (sraschka@wisc.edu)  \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "scikit-learn: 1.0\n",
      "\n"
     ]
    }
   ],
   "source": [
    "%load_ext watermark\n",
    "%watermark -p scikit-learn"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Gradient Boosting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train/Valid/Test sizes: 318 80 171\n"
     ]
    }
   ],
   "source": [
    "from sklearn import model_selection\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn import datasets\n",
    "\n",
    "\n",
    "data = datasets.load_breast_cancer()\n",
    "X, y = data.data, data.target\n",
    "\n",
    "X_temp, X_test, y_temp, y_test = \\\n",
    "    train_test_split(X, y, test_size=0.3, random_state=123, stratify=y)\n",
    "\n",
    "X_train, X_valid, y_train, y_valid = \\\n",
    "    train_test_split(X_temp, y_temp, test_size=0.2, random_state=123, stratify=y_temp)\n",
    "\n",
    "print('Train/Valid/Test sizes:', y_train.shape[0], y_valid.shape[0], y_test.shape[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Original gradient boosting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Accuracy: 1.00\n",
      "Validation Accuracy: 0.90\n",
      "Test Accuracy: 0.92\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "\n",
    "\n",
    "boost = GradientBoostingClassifier(\n",
    "    learning_rate=0.1,\n",
    "    n_estimators=100,\n",
    "    max_depth=8,\n",
    "    random_state=1)\n",
    "\n",
    "boost.fit(X_train, y_train)\n",
    "    \n",
    "    \n",
    "print(\"Training Accuracy: %0.2f\" % boost.score(X_train, y_train))\n",
    "print(\"Validation Accuracy: %0.2f\" % boost.score(X_valid, y_valid))\n",
    "print(\"Test Accuracy: %0.2f\" % boost.score(X_test, y_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## HistGradientBoostingClassifier (inspired by LightGBM)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Accuracy: 1.00\n",
      "Validation Accuracy: 0.96\n",
      "Test Accuracy: 0.97\n"
     ]
    }
   ],
   "source": [
    "#from sklearn.experimental import enable_hist_gradient_boosting\n",
    "from sklearn.ensemble import HistGradientBoostingClassifier\n",
    "\n",
    "\n",
    "boost = HistGradientBoostingClassifier(\n",
    "    learning_rate=0.1,\n",
    "    #n_estimators=100,\n",
    "    #max_depth=8,\n",
    "    random_state=1)\n",
    "\n",
    "boost.fit(X_train, y_train)\n",
    "    \n",
    "print(\"Training Accuracy: %0.2f\" % boost.score(X_train, y_train))\n",
    "print(\"Validation Accuracy: %0.2f\" % boost.score(X_valid, y_valid))\n",
    "print(\"Test Accuracy: %0.2f\" % boost.score(X_test, y_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```\n",
    "import numpy as np\n",
    "import xgboost as xgb\n",
    "\n",
    "\n",
    "dtrain = xgb.DMatrix(X_train, label=y_train)\n",
    "dtest = xgb.DMatrix(X_test, label=y_test)\n",
    "\n",
    "\n",
    "param = {\n",
    "    'max_depth': 8,\n",
    "    'eta': 0.1,  # learning rate\n",
    "    'objective': 'multi:softprob',  # loss function for multiclass\n",
    "    'num_class': 3}  # number of classes\n",
    "\n",
    "boost = xgb.train(param, dtrain, num_boost_round=100)\n",
    "\n",
    "y_pred = boost.predict(dtest)\n",
    "y_labels = np.argmax(y_pred, axis=1)\n",
    "\n",
    "\n",
    "print(\"Test Accuracy: %0.2f\" % (y_labels == y_test).mean())\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## XGBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://xgboost.readthedocs.io/en/latest/build.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!pip install xgboost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[07:41:34] WARNING: ../src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "Training Accuracy: 1.00\n",
      "Validation Accuracy: 0.95\n",
      "Test Accuracy: 0.98\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/raschka/miniforge3/lib/python3.8/site-packages/xgboost/sklearn.py:888: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import xgboost as xgb\n",
    "\n",
    "\n",
    "boost = xgb.XGBClassifier()\n",
    "\n",
    "boost.fit(X_train, y_train)\n",
    "    \n",
    "print(\"Training Accuracy: %0.2f\" % boost.score(X_train, y_train))\n",
    "print(\"Validation Accuracy: %0.2f\" % boost.score(X_valid, y_valid))\n",
    "print(\"Test Accuracy: %0.2f\" % boost.score(X_test, y_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LightGBM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://lightgbm.readthedocs.io/en/latest/Installation-Guide.html\n",
    "# conda install -c conda-forge lightgbm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Accuracy: 1.00\n",
      "Validation Accuracy: 0.96\n",
      "Test Accuracy: 0.98\n"
     ]
    }
   ],
   "source": [
    "import lightgbm as lgb\n",
    "\n",
    "\n",
    "boost = lgb.LGBMClassifier()\n",
    "\n",
    "boost.fit(X_train, y_train)\n",
    "\n",
    "\n",
    "print(\"Training Accuracy: %0.2f\" % boost.score(X_train, y_train))\n",
    "print(\"Validation Accuracy: %0.2f\" % boost.score(X_valid, y_valid))\n",
    "print(\"Test Accuracy: %0.2f\" % boost.score(X_test, y_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## CatBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://catboost.ai\n",
    "# conda install -c conda-forge catboost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Accuracy: 1.00\n",
      "Validation Accuracy: 0.97\n",
      "Test Accuracy: 0.98\n"
     ]
    }
   ],
   "source": [
    "from catboost import CatBoostClassifier\n",
    "\n",
    "\n",
    "boost = CatBoostClassifier(verbose=0)\n",
    "\n",
    "boost.fit(X_train, y_train)\n",
    "\n",
    "\n",
    "print(\"Training Accuracy: %0.2f\" % boost.score(X_train, y_train))\n",
    "print(\"Validation Accuracy: %0.2f\" % boost.score(X_valid, y_valid))\n",
    "print(\"Test Accuracy: %0.2f\" % boost.score(X_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}