{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "%matplotlib inline"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "\n# Plotting Calibration curve\n\nCompare mimic and isotonic calibration.\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "import matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nfrom sklearn.datasets import make_classification\nfrom sklearn.naive_bayes import MultinomialNB, GaussianNB\nfrom sklearn.calibration import calibration_curve\nfrom sklearn.isotonic import IsotonicRegression\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.metrics import (brier_score_loss,\n                             precision_score,\n                             recall_score,\n                             f1_score)\nfrom sklearn.svm import LinearSVC\nfrom copy import copy\nimport sys\n# sys.path.append('../')\nfrom mimic.mimic_calibration import _MimicCalibration\n# plt.rcParams[\"figure.figsize\"] = (20,10)\n\n\ndef calibration_comparison(base_estimator,\n                           n_samples,\n                           weights=None,\n                           n_bins=10,\n                           detail=False):\n\n    X, y = make_classification(n_samples=3*n_samples,\n                               n_features=6,\n                               random_state=42,\n                               weights=weights)\n    base_estimator_dict = {\n        \"MultinomialNB\": MultinomialNB(),\n        \"GaussianNB\": GaussianNB(),\n        \"SVC\": LinearSVC()\n    }\n\n    if (base_estimator == \"MultinomialNB\"):\n        X -= X.min()\n    # Train data: train binary model.\n    X_train, y_train = X[:n_samples], y[:n_samples]\n    print(\"Positive Rate: {x}\".format(x=y_train.mean()))\n    # calibrate data.\n    X_calib, y_calib = X[n_samples:2 * n_samples], y[n_samples:2 * n_samples]\n    # test data.\n    X_test, y_test = X[2 * n_samples:], y[2 * n_samples:]\n    # train the base estimator\n    clf = base_estimator_dict[base_estimator].fit(X_train, y_train)\n\n    if (base_estimator == \"SVC\"):\n        # y_calib_score: training in the calibration model.\n        y_calib_score = clf.decision_function(X_calib)\n        y_calib_score = (y_calib_score - y_calib_score.min()) /\\\n                        (y_calib_score.max() - y_calib_score.min())\n        # y_test_score: evaluation in the calibration model.\n        y_test_score = clf.decision_function(X_test)\n        y_test_score = (y_test_score - y_test_score.min()) /\\\n                       (y_test_score.max() - y_test_score.min())\n    else:\n        # y_calib_score: training in the calibration model.\n        y_calib_score = clf.predict_proba(X_calib)\n        y_calib_score = np.array([score[1] for score in y_calib_score])\n\n        # y_test_score: evaluation in the calibration model.\n        y_test_score = clf.predict_proba(X_test)\n        y_test_score = np.array([score[1] for score in y_test_score])\n\n    calibrate_model_dict = {\n        \"mimic\": _MimicCalibration(threshold_pos=5, record_history=False),\n        \"isotonic\": IsotonicRegression(y_min=0.0,\n                                       y_max=1.0,\n                                       out_of_bounds='clip'),\n        # \"platt\": LogisticRegression()\n    }\n\n    result = {}\n    result[base_estimator] = {}\n    for cal_name, cal_object in calibrate_model_dict.items():\n        # import pdb; pdb.set_trace()\n        print(cal_name)\n        cal_object.fit(copy(y_calib_score), copy(y_calib))\n        if cal_name in [\"mimic\", \"isotonic\"]:\n            y_output_score = cal_object.predict(copy(y_test_score))\n        else:\n            raise \"Please specify probability prediction function.\"\n\n        frac_pos, predicted_value = calibration_curve(\n            y_test,\n            y_output_score,\n            n_bins=n_bins)\n        b_score = brier_score_loss(y_test, y_output_score, pos_label=1)\n        # precsion = precision_score(y_test, y_output_score)\n        # recall = recall_score(y_test, y_output_score)\n        # f1 = f1_score(y_test, y_output_score)\n\n        result[base_estimator][cal_name] = {\n            \"calibration_curve\": [frac_pos, predicted_value],\n            # \"eval_score\" : [b_score, precsion, recall, f1]\n            \"eval_score\": [b_score]\n        }\n\n        if (detail):\n            result[base_estimator][cal_name][\"detail\"] = {\n                \"y_test\": y_test,\n                \"y_test_calibrate_score\": y_output_score\n            }\n\n    return result\n\n\ndef show_comparison_plots(base_estimator,\n                          n_samples,\n                          weights=None,\n                          n_bins=10,\n                          detail=False):\n    res = calibration_comparison(base_estimator,\n                                 n_samples,\n                                 weights,\n                                 n_bins,\n                                 detail)\n    all_calibration_methods = res[base_estimator]\n    all_calibration_methods_names = list(all_calibration_methods.keys())\n    color_map = {\n        \"isotonic\": 'orangered',\n        \"mimic\": 'limegreen'}\n\n    eval_df = []\n    fig = plt.figure()\n    for i, calib_name in enumerate(all_calibration_methods_names):\n        calib_model = all_calibration_methods[calib_name]\n        frac_pos, predicted_value = calib_model[\"calibration_curve\"]\n        b_score = all_calibration_methods[calib_name][\"eval_score\"][0]\n        if (i == 0):\n            plt.plot(frac_pos,\n                     frac_pos,\n                     color='grey',\n                     label=\"perfect-calibration\",\n                     alpha=0.3,\n                     linewidth=5)\n\n        plt.plot(predicted_value,\n                 frac_pos,\n                 color=color_map[calib_name],\n                 label=\"%s: %1.4f\" % (calib_name, b_score),\n                 alpha=0.7, linewidth=5)\n\n    plt.legend(fontsize=20)\n    plt.xlabel(\"calibrated probability\", fontsize=18)\n    plt.ylabel(\"fraction_of_positives\", fontsize=18)\n    plt.show()\n\n\nshow_comparison_plots(\"GaussianNB\", 10000, None, 10)"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.3"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}