LR_Loan.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "LR_Loan.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Sprivacy/DP-Utility-in-ML/blob/main/LR_Loan.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pNGkd76DabFu"
      },
      "source": [
        "#Mount Data on your google Drive\n",
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hN0p99H62MGY",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8286226c-3a97-4e1d-8ba4-26034c5be89c"
      },
      "source": [
        "#Neccessary Libraries for DP-SGD\n",
        "try:\n",
        "  # %tensorflow_version only exists in Colab.\n",
        "  %tensorflow_version 1.x\n",
        "except Exception:\n",
        "  pass\n",
        "\n",
        "import tensorflow as tf\n",
        "\n",
        "import numpy as np\n",
        "\n",
        "tf.compat.v1.logging.set_verbosity(tf.logging.ERROR)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "TensorFlow 1.x selected.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aqZhsEF42RuP"
      },
      "source": [
        "#importing libraries\n",
        "from tensorflow.keras.datasets import cifar10\n",
        "from tensorflow.keras.models import Sequential\n",
        "from tensorflow.keras.layers import Dense, Flatten, Conv2D, MaxPooling2D\n",
        "from tensorflow.keras.losses import sparse_categorical_crossentropy\n",
        "from tensorflow.keras.optimizers import Adam\n",
        "import matplotlib.pyplot as plt\n",
        "import tensorflow as tf\n",
        "from tensorflow import keras\n",
        "from tensorflow.keras import datasets, layers, models\n",
        "import matplotlib.pyplot as plt\n",
        "import sys\n",
        "from matplotlib import pyplot\n",
        "from keras.datasets import cifar10\n",
        "from keras.utils import to_categorical\n",
        "from keras.models import Sequential\n",
        "from keras.layers import Conv2D\n",
        "from keras.layers import MaxPooling2D\n",
        "from keras.layers import Dense\n",
        "from keras.layers import Flatten\n",
        "from keras.layers import Dropout\n",
        "from keras.optimizers import SGD\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import seaborn as sns\n",
        "import tensorflow as tf\n",
        "import matplotlib.pyplot as plt\n",
        "from sklearn.preprocessing import LabelEncoder, StandardScaler\n",
        "from sklearn.model_selection import train_test_split\n",
        "import tensorflow as tf\n",
        "#importing libraries\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import seaborn as sns\n",
        "import tensorflow as tf\n",
        "import matplotlib.pyplot as plt\n",
        "from sklearn.preprocessing import LabelEncoder, StandardScaler\n",
        "from sklearn.model_selection import train_test_split\n",
        "import tensorflow as tf"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "27N_XyNIvqAv"
      },
      "source": [
        "#install tensorflow privacy\n",
        "!pip install tensorflow_privacy"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KIvbP-Bjvu5s"
      },
      "source": [
        "#tensorflow Privacy Library\n",
        "\n",
        "from tensorflow_privacy.privacy.analysis import compute_dp_sgd_privacy\n",
        "from tensorflow_privacy.privacy.optimizers.dp_optimizer import DPGradientDescentGaussianOptimizer,DPAdamGaussianOptimizer"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "58TVTB36zQbr"
      },
      "source": [
        "#call the dataset\n",
        "dataset = pd.read_csv('...../accepted_2007_to_2018Q4.csv.gz', low_memory=False)\n",
        "data=dataset"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x9xzweQ1ulZE"
      },
      "source": [
        "#Drop null data\n",
        "data = data.sample(frac=0.4, axis=0, random_state=42).reset_index(drop=True)\n",
        "data = data.drop(data.loc[:, data.isna().mean().sort_values() > 0.3].columns, axis=1)\n",
        "data = data.dropna(axis=0).reset_index(drop=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZYQS1FZDwcsR"
      },
      "source": [
        "#Drop irrevalent column\n",
        "unneeded_columns = ['id', 'sub_grade', 'emp_title', 'url', 'title']\n",
        "{column: list(data[column].unique()) for column in data.drop(unneeded_columns, axis=1).columns if data.dtypes[column] == 'object'}\n",
        "data = data.drop(unneeded_columns, axis=1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mOakIJcB6IYI"
      },
      "source": [
        "#dealing with date feature\n",
        "date_columns = ['issue_d', 'earliest_cr_line', 'last_pymnt_d', 'last_credit_pull_d']\n",
        "for column in date_columns:\n",
        "    data[column + '_month'] = data[column].apply(lambda x: x[0:3])\n",
        "    data[column + '_year'] = data[column].apply(lambda x: x[-4:])\n",
        "\n",
        "data = data.drop(date_columns, axis=1)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sI_9vJToBGhH"
      },
      "source": [
        "month_ordering = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']\n",
        "for column in date_columns:\n",
        "    data[column + '_month'] = data[column + '_month'].apply(lambda x: month_ordering.index(x))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fLuMrFOEDwKi"
      },
      "source": [
        "Encoding"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6iazpPz4DOad"
      },
      "source": [
        "#dealing with dummy variables\n",
        "{column: list(data[column].unique()) for column in data.columns if data.dtypes[column] == 'object'}\n",
        "target = 'grade'\n",
        "\n",
        "binary_features = ['term', 'pymnt_plan', 'initial_list_status', 'application_type', 'hardship_flag', 'disbursement_method', 'debt_settlement_flag']\n",
        "binary_positives = [' 60 months', 'y', 'w', 'Individual', 'Y', 'Cash', 'Y']\n",
        "ordinal_features = ['emp_length']\n",
        "emp_ordering = [\n",
        "    '< 1 year',\n",
        "    '1 year',\n",
        "    '2 years',\n",
        "    '3 years',\n",
        "    '4 years',\n",
        "    '5 years',\n",
        "    '6 years',\n",
        "    '7 years',\n",
        "    '8 years',\n",
        "    '9 years',\n",
        "    '10+ years'\n",
        "]\n",
        "\n",
        "nominal_features = ['home_ownership', 'verification_status', 'loan_status', 'purpose', 'addr_state']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2aEzi7IVEY_h"
      },
      "source": [
        "# Encoding functions\n",
        "\n",
        "def binary_encode(df, column, positive_value):\n",
        "    df[column] = df[column].apply(lambda x: 1 if x == positive_value else 0)\n",
        "\n",
        "def ordinal_encode(df, column, ordering):\n",
        "    df[column] = df[column].apply(lambda x: ordering.index(x))\n",
        "\n",
        "def onehot_encode(df, column):\n",
        "    dummies = pd.get_dummies(df[column])\n",
        "    df_new = pd.concat([df, dummies], axis=1)\n",
        "    df_new = df_new.drop(column, axis=1)\n",
        "    return df_new"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "orVWUMOoEz0_"
      },
      "source": [
        "# Perform encoding\n",
        "for feature, positive_value in zip(binary_features, binary_positives):\n",
        "    binary_encode(data, feature, positive_value)\n",
        "\n",
        "ordinal_encode(data, 'emp_length', emp_ordering)\n",
        "\n",
        "for feature in nominal_features:\n",
        "    data = onehot_encode(data, feature)\n",
        "\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5yW4Vu9dG-XI",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "bfbde1ee-fc07-4039-b389-329634d08026"
      },
      "source": [
        "# Encoding label column\n",
        "\n",
        "label_encoder = LabelEncoder()\n",
        "\n",
        "data[target] = label_encoder.fit_transform(data[target])\n",
        "\n",
        "target_mappings = {index: label for index, label in enumerate(label_encoder.classes_)}\n",
        "target_mappings"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{0: 'A', 1: 'B', 2: 'C', 3: 'D', 4: 'E', 5: 'F', 6: 'G'}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "V2611nM0Nc7n"
      },
      "source": [
        "#remove string from zipcode/convert it into integers\n",
        "data['zip_code'] = data['zip_code'].str.replace(r'\\D', '').astype(int)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "khsu-aUGH87s"
      },
      "source": [
        "Spliting and Scaling"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RUE3uVwkH_Ae"
      },
      "source": [
        "y = data['grade']\n",
        "X = data.drop('grade', axis=1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JmLTv7CAIELK"
      },
      "source": [
        "scaler = StandardScaler()\n",
        "X = scaler.fit_transform(X)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Fb0JVERRJQej"
      },
      "source": [
        "X_train, X_test, y_train, y_test = train_test_split(X[0:100000], y[0:100000], train_size=0.5, random_state=0)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "M8Cgbrfy6d7G"
      },
      "source": [
        "#dp to numpy\n",
        "y_train=y_train.to_numpy()\n",
        "y_test=y_test.to_numpy()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dbdNm7n_3z-b"
      },
      "source": [
        "#convert categorical data\n",
        "from keras.utils import to_categorical\n",
        "trainY = to_categorical(y_train)\n",
        "testY = to_categorical(y_test)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jgduWsL4lvXe"
      },
      "source": [
        "Logistic Regression:Input Perturbation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9oqmn5odlzJU"
      },
      "source": [
        "n=25000\n",
        "xtrain=X_train[0:n]\n",
        "ytrain=y_train[0:n]\n",
        "xtest=X_test[0:n]\n",
        "ytest=y_test[0:n]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jISLP_cPl1Y1"
      },
      "source": [
        "from sklearn.linear_model import LogisticRegression\n",
        "model = LogisticRegression(penalty='l2',max_iter=100,C=10000,multi_class='ovr',solver='lbfgs').fit(xtrain, ytrain)\n",
        "np_in=model.score(xtest, ytest)\n",
        "np_in"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_CkKRV6ql3lZ"
      },
      "source": [
        "ep=[0.01, 0.1,1,10,100,1000,10000]\n",
        "ni=len(ep)\n",
        "acc_in=np.zeros(ni)\n",
        "for i in range(ni):\n",
        "  epi=ep[i]\n",
        "  delta=.00001\n",
        "  C=1\n",
        "  z=3*np.log(25000)\n",
        "  z1=z**2\n",
        "  sigma=z1*(8*np.log(2/(delta/2))+4*epi)/(epi**2)\n",
        "  #sigma=calc_sigma(ep[i],delta,18,1/C)\n",
        "  nsi=np.random.normal(loc=0.0, scale=sigma/n)\n",
        "  print(nsi)\n",
        "  X_in=xtrain-nsi\n",
        "  model_in = LogisticRegression(penalty='l2',max_iter=100,C=10000,multi_class='ovr',solver='lbfgs').fit(X_in, ytrain)\n",
        "  acc_in[i]=model_in.score(xtest, ytest)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iwffcCUh2Tu9"
      },
      "source": [
        "Logistic Regression:Objective Perturbation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Wr8u0lUbQQHf"
      },
      "source": [
        "!pip install diffprivlib"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wrrnKgY72XbI"
      },
      "source": [
        "from diffprivlib.models import LogisticRegression"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BmPxHt1d0t4i"
      },
      "source": [
        "ep=[0.01, 0.1,1,10,100,1000,10000]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9HVcbyiyxzhi"
      },
      "source": [
        "#50% data for training target model\n",
        "n=25000\n",
        "xtrain=X_train[0:n]\n",
        "ytrain=y_train[0:n]\n",
        "xtest=X_test[0:n]\n",
        "ytest=y_test[0:n]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bWkNdbkm2s6p"
      },
      "source": [
        "x_train=X_train.reshape(X_train.shape[0],len(X_train[0]),1)\n",
        "x_test=X_test.reshape(X_test.shape[0],len(X_test[0]),1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pYN9rPke2gZs"
      },
      "source": [
        "###training and testing data\n",
        "n=25000\n",
        "xtrain_agg=X_train[0:n]\n",
        "ytrain_agg=trainY[0:n]\n",
        "#ytrain_agg=ytrain_agg.to_numpy()\n",
        "#ytrain_pred=model_np.predict_proba(xtrain_agg)\n",
        "xtest_agg=X_test[0:n]\n",
        "ytest_agg=testY[0:n]\n",
        "#ytest_agg=ytest_agg.to_numpy()\n",
        "#ytest_pred=model_np.predict_proba(xtest_agg)\n",
        "target_train_agg = (xtrain_agg,ytrain_agg)\n",
        "target_test_agg = (xtest_agg,ytest_agg)\n",
        "#target_train_data_agg, target_test_data_agg = sample_data(target_train_agg, target_test_agg, NUM_TARGET)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "av6E9EX2kQAw"
      },
      "source": [
        "#define labels\n",
        "def label(my_list):\n",
        "  import numpy as np\n",
        "  my_array=np.array(my_list)\n",
        "  p=np.zeros(my_array.shape)\n",
        "  b=my_array.max(-1)\n",
        "  condition = my_array == b[..., np.newaxis]\n",
        "  c = np.where(condition, 1, 0)\n",
        "  final=np.multiply(c, my_array)\n",
        "  #my_sum=np.sum(final,axis=0)\n",
        "  labels=np.argmax(final, axis=1)\n",
        "  return labels"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pTojIuEp8cgT"
      },
      "source": [
        "#objective Perturbation\n",
        "n=25000\n",
        "n1=len(ep)\n",
        "acc_obj=np.zeros(n1)\n",
        "for i in range(len(ep)):\n",
        "     obj_model=LogisticRegression(data_norm=1, epsilon=ep[i],penalty='l2',max_iter=100,C=10000).fit(xtrain, ytrain)\n",
        "     acc_obj[i]=obj_model.score(xtest,ytest)    #take model accuracy for different privacy budget"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7-jBA0dCaOUK"
      },
      "source": [
        "#simple logistic regression\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "\n",
        "model = LogisticRegression(penalty='l2',max_iter=100,C=10000,multi_class='ovr',solver='lbfgs').fit(xtrain, ytrain)\n",
        "non_p=model.score(xtest, ytest)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2AdsZkSuDs1f"
      },
      "source": [
        "model.score(xtrain,ytrain)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8GSCYeUamTmu"
      },
      "source": [
        "output Perturbation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uuxkz9CemVyI"
      },
      "source": [
        "#non-private accuracy\n",
        "from keras.regularizers import l2\n",
        "def build_lg_model():\n",
        "    import keras\n",
        "    from keras.models import Sequential\n",
        "    from keras.layers import Conv2D, MaxPooling2D, Flatten, Dense, Dropout\n",
        "    # build the model\n",
        "    model = Sequential()\n",
        "    model.add(Dense(units=7, activation='sigmoid',kernel_regularizer=l2(.0001)))\n",
        "    model.compile(optimizer='Adam', loss='categorical_crossentropy', metrics=['accuracy'])\n",
        "    return model\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rEuYg1BT3KoG"
      },
      "source": [
        "#model=SGD_lg(1,x_train[0:15000],trainY[0:15000],x_test,testY)\n",
        "def calc_labelAcc(Model,xtest,ytest):\n",
        "  from sklearn.metrics import accuracy_score\n",
        "  ypred=Model.predict_classes(xtest)\n",
        "  acc=accuracy_score(label(ytest), ypred)\n",
        "  return acc\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8PuXjOtU9Lwy"
      },
      "source": [
        "model_np=build_lg_model()\n",
        "model_np.fit(xtrain,ytrain_agg,batch_size=250, epochs=100, verbose=0)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pr2C2CXfm0u1"
      },
      "source": [
        "model_np.save(\"my_model\")\n",
        "weight=model_np.get_weights()\n",
        "np_out=calc_labelAcc(model_np,xtest,testY[0:n])\n",
        "print(np_out)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_hn6Ow5HnrSv"
      },
      "source": [
        "#output perturbation\n",
        "ep=[.01,.1,1,10,100,1000,10000] \n",
        "delta=.00001   #privacy budget\n",
        "n=25000\n",
        "acc_out=np.zeros(len(ep))\n",
        "from tensorflow import keras\n",
        "for i in range(len(ep)):\n",
        "  #add noise to the weights\n",
        "  #noise=(2/(n*ep[i]*.0001)) \n",
        "  s=2/(n*.0001)  #lambda=.0001\n",
        "  model1=keras.models.load_model(\"my_model\")\n",
        "  w=model1.get_weights()\n",
        "  #print(w[0])\n",
        "  W=w\n",
        "  sigma = np.sqrt(2 * np.log(1.25 / delta)) * (s / ep[i])\n",
        "  #ns=np.random.laplace(loc=0.0, scale=noise)\n",
        "  ns=np.random.normal(loc=0.0, scale=sigma)\n",
        "  print(ns)\n",
        "  #print(ns)\n",
        "  for j in range(2):\n",
        "       W[j]=W[j]+ns\n",
        "  #print(W[0])\n",
        "  model2=keras.models.load_model(\"my_model\")\n",
        "  model2.set_weights(W)\n",
        "  acc_out[i]=calc_labelAcc(model2,xtest,testY[0:n])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QcECec2iPr41"
      },
      "source": [
        "Gradient Perturbation with logistic Regression"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Cr0dseQ-P55E"
      },
      "source": [
        "def SGD_lg(noise_multiplier,x_train,y_train,x_test,y_test):\n",
        "   epochs = 100\n",
        "   batch_size = 250\n",
        "   l2_norm_clip = 1.5\n",
        "   #noise_multiplier = .88\n",
        "   num_microbatches = 125\n",
        "   learning_rate = 0.01\n",
        "   x_train_s=x_train\n",
        "   y_train_s=y_train\n",
        "   x_test=x_test\n",
        "   y_test_s=y_test\n",
        "   n=len(y_train)\n",
        "   #noise_multiplier=.5\n",
        "   if batch_size % num_microbatches != 0:\n",
        "     raise ValueError('Batch size should be an integer multiple of the number of microbatches')\n",
        "   \n",
        "   eps=compute_dp_sgd_privacy.compute_dp_sgd_privacy(n=n, batch_size=250, noise_multiplier=noise_multiplier, epochs=epochs, delta=1e-5)\n",
        "   print(eps)\n",
        "\n",
        "   model = tf.keras.models.Sequential([tf.keras.layers.Flatten(), \n",
        "                                    tf.keras.layers.Dense(166, activation = tf.nn.relu), \n",
        "                                    tf.keras.layers.Dense(7, activation = tf.nn.softmax)])\n",
        "   print(\"compplete\")\n",
        "   optimizer = DPAdamGaussianOptimizer(\n",
        "            l2_norm_clip=l2_norm_clip,\n",
        "            noise_multiplier=noise_multiplier,\n",
        "            num_microbatches=num_microbatches,\n",
        "            learning_rate=learning_rate)\n",
        "   \n",
        "   loss = tf.keras.losses.CategoricalCrossentropy(\n",
        "    from_logits=True, reduction=tf.losses.Reduction.NONE)\n",
        "   model.compile(optimizer=optimizer, loss=loss, metrics=['accuracy'])\n",
        "   model.fit(x_train_s, y_train_s,\n",
        "          epochs=epochs,\n",
        "          validation_data=(x_test[0:1000], y_test_s[0:1000]),\n",
        "          batch_size=batch_size)\n",
        "   return model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A7o3Hl5J0uRw"
      },
      "source": [
        "#non-private model\n",
        "model_np = tf.keras.models.Sequential([tf.keras.layers.Flatten(), \n",
        "                                    tf.keras.layers.Dense(166, activation = tf.nn.relu), \n",
        "                                    tf.keras.layers.Dense(7, activation = tf.nn.softmax)])\n",
        "loss = tf.keras.losses.CategoricalCrossentropy(\n",
        "    from_logits=True, reduction=tf.losses.Reduction.NONE)\n",
        "opt=keras.optimizers.Adam(learning_rate=0.01,decay=.2)\n",
        "model_np.compile(optimizer = opt,\n",
        "              loss = loss,\n",
        "              metrics =['accuracy'])\n",
        "model_np.fit(xtrain, ytrain_agg, epochs = 100,batch_size=250,validation_data=(xtest[0:3000],ytest_agg[0:3000]))\n",
        "np_sgd=calc_labelAcc(model_np,xtest,ytest_agg)\n",
        "np_sgd"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EKU1lVu1P4kc"
      },
      "source": [
        "noise_mul=[350,33,4,.831,.41,.25515,.1856]  #noise multiplier\n",
        "ep=[.01,.1,1,10,100,1000,10000] "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B2f7bzbq9-7r"
      },
      "source": [
        "#train model locally\n",
        "acc_sgd= np.zeros(len(noise_mul))\n",
        "model_index =0\n",
        "for i in range(0,7):\n",
        "  model=SGD_lg(noise_mul[i],xtrain_agg,ytrain_agg,xtest_agg,ytest_agg)\n",
        "  acc_sgd[i]=calc_labelAcc(model,xtest_agg,ytest_agg)\n",
        "\n",
        "  globals()['model_sgd%s' % i]=model\n",
        "\n",
        "  \n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "otIf2JJm35cx"
      },
      "source": [
        "acc_sgd= np.zeros(len(noise_mul))\n",
        "#model_index =0\n",
        "for i in range(len(noise_mul)):\n",
        "  model=globals()['model_sgd%s' % i]\n",
        "  acc_sgd[i]=calc_labelAcc(model,xtest_agg,ytest_agg)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "c-aKYXu8vI2w"
      },
      "source": [
        "PATE"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f4NTRBmNvKPG"
      },
      "source": [
        "#split datasets into multiple teacher\n",
        "#Divide the images into 5 \n",
        "# split x_train to 10 disjoint datasets\n",
        "#store each dataset variable name is the list Xtrain\n",
        "import numpy as np\n",
        "M=40\n",
        "j=0\n",
        "k=len(xtrain_agg)/M\n",
        "for x in range(0,M):\n",
        "             globals()['x_train_split%s' % x]=xtrain_agg[int(j):int(k+j)]\n",
        "             globals()['y_train_split%s' % x]=ytrain[int(j):int(k+j)]\n",
        "             j=k+j\n",
        "             #print(j)\n",
        "             \n",
        "print(x_train_split29.shape)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "o8pWAbO-vnrU"
      },
      "source": [
        "for iter in range(M):\n",
        "     from sklearn.linear_model import LogisticRegression\n",
        "     model = LogisticRegression(random_state=0,penalty='l2',max_iter=100,C=10000,multi_class='ovr',solver='lbfgs')\n",
        "     model.fit(globals()['x_train_split%s' % iter], globals()['y_train_split%s' % iter])\n",
        "     globals()['model_pate%s' % iter]=model\n",
        "     "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XU5JAhl-_anK"
      },
      "source": [
        "#function for vote counting\n",
        "def vote(my_list):\n",
        "  import numpy as np\n",
        "  my_array=np.array(my_list)\n",
        "  p=np.zeros(my_array.shape)\n",
        "  b=my_array.max(-1)\n",
        "  condition = my_array == b[..., np.newaxis]\n",
        "  c = np.where(condition, 1, 0)\n",
        "  final=np.multiply(c, my_array)\n",
        "  #my_sum=np.sum(final,axis=0)\n",
        "  labels=np.argmax(final, axis=1)\n",
        "  return c"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5rzFofMwFvz2"
      },
      "source": [
        "#compute label\n",
        "def label(my_list):\n",
        "  import numpy as np\n",
        "  my_array=np.array(my_list)\n",
        "  p=np.zeros(my_array.shape)\n",
        "  b=my_array.max(-1)\n",
        "  condition = my_array == b[..., np.newaxis]\n",
        "  c = np.where(condition, 1, 0)\n",
        "  final=np.multiply(c, my_array)\n",
        "  #my_sum=np.sum(final,axis=0)\n",
        "  labels=np.argmax(final, axis=1)\n",
        "  return labels"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "v4i0X-j__LNy"
      },
      "source": [
        "sum=np.zeros([len(ytest),7])\n",
        "for iter in range(M):\n",
        "  New_model=globals()['model_pate%s' % iter]\n",
        "  y=np.zeros([len(ytest),7])\n",
        "  y=New_model.predict_proba(xtest)\n",
        "  #print(y.shape)\n",
        "  y=vote(y)\n",
        "  sum=y+sum"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WtCFtLLyFnAd"
      },
      "source": [
        "#create the labels\n",
        "from sklearn.metrics import accuracy_score\n",
        "lab=label(sum)\n",
        "#clear accuracy\n",
        "#y_true=label(ytest)\n",
        "accnp=accuracy_score(ytest,lab)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cpZOS0dMF7ir"
      },
      "source": [
        "#add Laplace noise \n",
        "def add_noise_sum(noise,sum,experiment,ypred):\n",
        "  sum_s=sum\n",
        "  predt=np.zeros(experiment)\n",
        "  for i1 in range(experiment):\n",
        "    sum_s=np.zeros(sum.shape)\n",
        "    sum_f=sum+np.random.laplace(loc=0.0, scale=1/noise)\n",
        "    sum2=label(sum_f)\n",
        "    predt[i1]=accuracy_score(ypred,sum2)\n",
        "    #print(i)\n",
        "  pred=np.average(predt)\n",
        "  #print(predt)\n",
        "  #print(pred)\n",
        "  return pred"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PiAOVddpGLJw"
      },
      "source": [
        "acc_pate=np.zeros(len(ep))\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DugAYowXGCdy"
      },
      "source": [
        "from sklearn.metrics import accuracy_score\n",
        "from sklearn.metrics import average_precision_score\n",
        "#Noise after Aggregation Method\n",
        "for i in range(0,len(ep)):\n",
        "                          sum_s=np.zeros(sum.shape)\n",
        "                          sum_n=sum\n",
        "                          #sum_s=sum_n+np.random.laplace(loc=0.0, scale=1/noise[i])\n",
        "                          predF=add_noise_sum(ep[i],sum_n,100,ytest)\n",
        "                          acc_pate[i]=predF\n",
        "print(acc_pate)\n",
        "print(ep)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7nBgrgFvsIB7"
      },
      "source": [
        "Utility Loss Graph"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZmAkML4asGa4"
      },
      "source": [
        "import pandas as pd \n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns #grafikleştirme için\n",
        "import matplotlib.pyplot as plt \n",
        "from google.colab import files\n",
        "test1 = plt.figure()\n",
        "plt.semilogx(ep,non_p-acc_obj,color=\"black\",marker='D',label='Objective Perturbation',linewidth=1.5)\n",
        "plt.semilogx(ep,np_sgd-acc_sgd,color=\"black\",marker='*',label='Gradient Perturbation',linewidth=1.5)\n",
        "plt.semilogx(ep,np_out-acc_out,color=\"black\",marker='<',label='Output Perturbation',linewidth=1.5)\n",
        "plt.semilogx(ep,accnp-acc_pate,color=\"black\",marker='o',label='Prediction Perturbation',linewidth=1.5)\n",
        "#plt.plot(ep,non_p,color=\"red\",marker='*',linestyle=\"--\",label='Non-Private Model',linewidth=2)\n",
        "plt.legend(loc=1,fontsize=12)\n",
        "plt.xlabel(\"Privacy Budget($\\epsilon$)\",fontsize=13)\n",
        "plt.ylabel(\"Utility Loss\",fontsize=15)\n",
        "#plt.xscale('symlog', linthreshy=0.1)\n",
        "#plt.ylim([-.1,1])\n",
        "plt.xticks(size = 10)\n",
        "plt.yticks(size = 8)\n",
        "plt.ylim([-.05,1])\n",
        "#y.set_color(\"black\")\n",
        "plt.rcParams[\"axes.edgecolor\"] = \"black\"\n",
        "plt.rcParams[\"axes.linewidth\"] = 1\n",
        "plt.rcParams['axes.facecolor'] = 'white'\n",
        "#test1.set_facecolor('white')\n",
        "test1.show()\n",
        "test1.savefig('logistic_perturb_acc.pdf')\n",
        "#files.download('logistic_perturb_acc.pdf')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xZ4yY4VLP5JP"
      },
      "source": [
        "print(non_p-acc_obj)\n",
        "print(np_sgd-acc_sgd)\n",
        "print(np_out-acc_out)\n",
        "print(accnp-acc_pate)\n",
        "#obj,gradient,output,pate\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0DJccKmoQOWx"
      },
      "source": [
        "[0.55016 0.5572  0.49068 0.34488 0.24408 0.06888 0.01664]\n",
        "[0.5446  0.4876  0.37716 0.10808 0.14236 0.01392 0.002  ]\n",
        "[6.6584e-01 6.3656e-01 3.9260e-01 3.2280e-02 1.2800e-03 8.0000e-05\n",
        " 1.6000e-04]\n",
        "[ 2.17168000e-01  7.43524000e-02 -2.22044605e-16 -2.22044605e-16\n",
        " -2.22044605e-16 -2.22044605e-16 -2.22044605e-16]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bBEWpAuNL4Gj"
      },
      "source": [
        "Attack"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nMlwaXa5L5VQ"
      },
      "source": [
        "#Assign necessary variables for attacking the model\n",
        "import argparse\n",
        "import csv\n",
        "import numpy as np\n",
        "from sklearn.utils import resample, shuffle\n",
        "\n",
        "import keras\n",
        "import numpy as np\n",
        "from sklearn.utils import resample\n",
        "\n",
        "LEARNING_RATE = 0.01\n",
        "EPOCH = 100\n",
        "#EPOCH = 30\n",
        "DATA_SIZE = 50000\n",
        "TRAINING_SIZE = 25000\n",
        "TEST_SIZE = 25000\n",
        "NUM_TARGET = 1\n",
        "#NUM_SHADOW = 100\n",
        "NUM_SHADOW = 10\n",
        "IN = 1\n",
        "OUT = 0\n",
        "VERBOSE = 0"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zGjbecV6WgGF"
      },
      "source": [
        "#call required libraries\n",
        "from tensorflow.keras.datasets import cifar10\n",
        "from tensorflow.keras.models import Sequential\n",
        "from tensorflow.keras.layers import Dense, Flatten, Conv2D, MaxPooling2D\n",
        "from tensorflow.keras.losses import sparse_categorical_crossentropy\n",
        "from tensorflow.keras.optimizers import Adam\n",
        "import matplotlib.pyplot as plt\n",
        "import tensorflow as tf\n",
        "from tensorflow.keras import datasets, layers, models\n",
        "import matplotlib.pyplot as plt\n",
        "from keras.utils import to_categorical \n",
        "import sys\n",
        "from matplotlib import pyplot\n",
        "from keras.datasets import cifar10\n",
        "from keras.utils import to_categorical\n",
        "from keras.models import Sequential\n",
        "from keras.layers import Conv2D\n",
        "from keras.layers import MaxPooling2D\n",
        "from keras.layers import Dense\n",
        "from keras.layers import Flatten\n",
        "from keras.layers import Dropout\n",
        "from keras.optimizers import SGD"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "z51_bHesWw7g"
      },
      "source": [
        "#Define model configuaration\n",
        "# Model configuration\n",
        "batch_size = 250\n",
        "#img_width, img_height, img_num_channels = 32, 32, 3\n",
        "#loss_function = sparse_categorical_crossentropy\n",
        "no_classes = 2\n",
        "no_epochs = 250\n",
        "optimizer = Adam()\n",
        "validation_split = 0.2\n",
        "verbosity = 0"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HIRISU4nWyXv"
      },
      "source": [
        "#building a logistic regression model with keras\n",
        "def build_lg_model(num_class=7):\n",
        "    import keras\n",
        "    from keras.models import Sequential\n",
        "    from keras.layers import Conv2D, MaxPooling2D, Flatten, Dense, Dropout\n",
        "    # build the model\n",
        "    model = Sequential()\n",
        "    model.add(Dense(units=7,kernel_initializer='glorot_uniform', activation='sigmoid',kernel_regularizer=l2(.0001)))\n",
        "    return model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LY_6Nb9oW2VU"
      },
      "source": [
        "#data sampling\n",
        "def sample_data(train_data,test_data,num_sets):\n",
        "    (x_train, y_train), (x_test, y_test) = train_data, test_data\n",
        "    new_x_train, new_y_train = [], []\n",
        "    new_x_test, new_y_test = [], []\n",
        "    for i in range(num_sets):\n",
        "        x_temp, y_temp = resample(x_train, y_train, n_samples=n, random_state=0)\n",
        "        new_x_train.append(x_temp)\n",
        "        new_y_train.append(y_temp)\n",
        "        x_temp, y_temp = resample(x_test, y_test, n_samples=n, random_state=0)\n",
        "        new_x_test.append(x_temp)\n",
        "        new_y_test.append(y_temp)\n",
        "    return (new_x_train, new_y_train), (new_x_test, new_y_test)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pNeS9YS9W534"
      },
      "source": [
        "def get_trained_keras_models(keras_model, train_data, test_data, num_models):\n",
        "    (x_train, y_train), (x_test, y_test) = train_data, test_data\n",
        "    models = []\n",
        "    for i in range(num_models):\n",
        "        models.append(keras.models.clone_model(keras_model))\n",
        "        models[i].compile(optimizer='sgd', loss='binary_crossentropy', metrics=['binary_accuracy'])\n",
        "        models[i].fit(x_train[i], y_train[i], batch_size=32, epochs=EPOCH, verbose=VERBOSE, shuffle=True)\n",
        "        score = models[i].evaluate(x_test[i], y_test[i], verbose=VERBOSE)\n",
        "        print('\\n', 'Model ', i, ' test accuracy:', score[1])\n",
        "    return models"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nVBmQjM1YJqv"
      },
      "source": [
        "from sklearn.linear_model import LogisticRegression\n",
        "def get_trained_keras_models_lg(train_data, test_data, num_models):\n",
        "    (x_train, y_train), (x_test, y_test) = train_data, test_data\n",
        "    models = []\n",
        "    for i in range(num_models):\n",
        "        #models.append(keras.models.clone_model(keras_model))\n",
        "        #models[i].compile(optimizer='sgd', loss='binary_crossentropy', metrics=['binary_accuracy'])\n",
        "        #models[i].fit(x_train[i], y_train[i], batch_size=32, epochs=EPOCH, verbose=VERBOSE, shuffle=True)\n",
        "        models.append(LogisticRegression(random_state=0))\n",
        "        models[i].fit(x_train[i], y_train[i])\n",
        "        score = models[i].score(x_test[i], y_test[i])\n",
        "        print('\\n', 'Model ', i, ' test accuracy:', score)\n",
        "    return models"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "P_YbjpSiYP18"
      },
      "source": [
        "def get_trained_keras_models_target(keras_model, train_data, test_data, num_models):\n",
        "    (x_train, y_train), (x_test, y_test) = train_data, test_data\n",
        "    models = []\n",
        "    for i in range(num_models):\n",
        "        models.append(keras.models.clone_model(keras_model))\n",
        "        models[i].compile(optimizer='sgd', loss='binary_crossentropy', metrics=['binary_accuracy'])\n",
        "        models[i].fit(x_train[i], y_train[i], batch_size=32, epochs=EPOCH, verbose=VERBOSE, shuffle=True)\n",
        "        score = models[i].evaluate(x_test[i], y_test[i], verbose=VERBOSE)\n",
        "        print('\\n', 'Model ', i, ' test accuracy:', score[1])\n",
        "    return models"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S1U4DBvTYTTT"
      },
      "source": [
        "#collect the attack dataset from shadow models\n",
        "def get_attack_dataset(models, train_data, test_data, num_models, data_size):\n",
        "    # generate dataset for the attack model\n",
        "    (x_train, y_train), (x_test, y_test) = train_data, test_data\n",
        "    num_classes = len(y_train[0][0])\n",
        "    x_data, y_data = [[] for i in range(num_classes)], [[] for i in range(num_classes)]\n",
        "    for i in range(num_models):\n",
        "        # IN data\n",
        "        x_temp, y_temp = resample(x_train[i], y_train[i], n_samples=data_size, random_state=0)\n",
        "        for j in range(data_size):\n",
        "            y_idx = np.argmax(y_temp[j])\n",
        "            x_data[y_idx].append(models[i].predict(x_temp[j:j+1])[0])\n",
        "            #print(y_idx)\n",
        "            y_data[y_idx].append(IN)\n",
        "            print(\"starts1\",j)\n",
        "        # OUT data\n",
        "        x_temp, y_temp = resample(x_test[i], y_test[i], n_samples=data_size, random_state=0)\n",
        "        for j in range(data_size):\n",
        "            y_idx = np.argmax(y_temp[j])\n",
        "            p=models[i].predict(x_temp[j:j+1])[0]\n",
        "            x_data[y_idx].append(p)\n",
        "            y_data[y_idx].append(OUT)\n",
        "            print(\"starts2\",j)\n",
        "    return x_data, y_data"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sm15CzhEYXKr"
      },
      "source": [
        "#collect the attack dataset from shadow models: Logistic Regression\n",
        "def get_attack_dataset_lg(models, train_data, test_data, num_models, data_size):\n",
        "    # generate dataset for the attack model\n",
        "    (x_train, y_train), (x_test, y_test) = train_data, test_data\n",
        "    num_classes = 7\n",
        "    x_data, y_data = [[] for i in range(num_classes)], [[] for i in range(num_classes)]\n",
        "    for i in range(num_models):\n",
        "        # IN data\n",
        "        x_temp, y_temp = resample(x_train[i], y_train[i], n_samples=data_size, random_state=0)\n",
        "        for j in range(data_size):\n",
        "            y_idx = y_temp[j]\n",
        "            x_data[y_idx].append(models[i].predict_proba(x_temp[j:j+1])[0])\n",
        "            #print(y_idx)\n",
        "            y_data[y_idx].append(IN)\n",
        "            print(\"starts1\",j)\n",
        "        # OUT data\n",
        "        x_temp, y_temp = resample(x_test[i], y_test[i], n_samples=data_size, random_state=0)\n",
        "        for j in range(data_size):\n",
        "            y_idx = y_temp[j]\n",
        "            p=models[i].predict_proba(x_temp[j:j+1])[0]\n",
        "            x_data[y_idx].append(p)\n",
        "            y_data[y_idx].append(OUT)\n",
        "            print(\"starts2\",j)\n",
        "    return x_data, y_data"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cayQ3VsAYbFc"
      },
      "source": [
        "#generate the report\n",
        "def get_leakage(models, test_data):\n",
        "    from sklearn.metrics import accuracy_score, precision_score, recall_score\n",
        "    from sklearn.metrics import average_precision_score\n",
        "    from sklearn import metrics\n",
        "    from sklearn.metrics import confusion_matrix\n",
        "    (x_test, y_true) = test_data\n",
        "    acc_scores = []\n",
        "    pre_scores = []\n",
        "    rec_scores = []\n",
        "    fp=np.zeros(len(models))\n",
        "    tp=np.zeros(len(models))\n",
        "    tn=np.zeros(len(models))\n",
        "    fn=np.zeros(len(models))\n",
        "    avg=len(models)\n",
        "    for i in range(len(models)):\n",
        "        y_pred = models[i].predict(x_test[i])\n",
        "        #print(len(models))\n",
        "        # _LOG_PRINT(y_pred)\n",
        "        acc_scores.append(accuracy_score(y_true[i], y_pred))\n",
        "        pre_scores.append(average_precision_score(y_true[i], y_pred))\n",
        "        rec_scores.append(recall_score(y_true[i], y_pred))\n",
        "        tn[i], fp[i], fn[i], tp[i] = confusion_matrix(y_true[i], y_pred).ravel()\n",
        "    return np.sum(tn),np.sum(tp),np.sum(fn),np.sum(fp)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_8oaAxCfYexb"
      },
      "source": [
        "#attack classifier:SVM\n",
        "def get_trained_svm_models(train_data, test_data, num_models):\n",
        "    from sklearn import svm\n",
        "    (x_train, y_train), (x_test, y_test) = train_data, test_data\n",
        "    models = []\n",
        "    for i in range(num_models):\n",
        "        print('Training svm model : ', i)\n",
        "        models.append(svm.SVC(gamma='scale',kernel='linear',verbose=VERBOSE))\n",
        "        models[i].fit(x_train[i], y_train[i])\n",
        "        score = models[i].score(x_test[i],y_test[i])\n",
        "        #print(i)\n",
        "        print('SVM model ', i, 'score : ',score)\n",
        "    return models,score"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dhMo1itZYiqz"
      },
      "source": [
        "shadow_train = (X_train[n:n*2],y_train[n:n*2])\n",
        "shadow_test = (X_test[n:n*2],y_test[n:n*2])\n",
        "shadow_train_data, shadow_test_data = sample_data(shadow_train, shadow_test, NUM_SHADOW)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GSQ1e071YuC7"
      },
      "source": [
        "#when we use logistic regression model from sklearn\n",
        "shadow_models = get_trained_keras_models_lg(shadow_train_data, shadow_test_data, NUM_SHADOW)\n",
        "attack_train = get_attack_dataset_lg(shadow_models, shadow_train_data, shadow_test_data, NUM_SHADOW, TEST_SIZE)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BSl9UC5sgyL-"
      },
      "source": [
        "def lk(tn,tp,fn,fp):\n",
        "  tpr=(tp/(tp+fn))\n",
        "  fpr=(fp/(fp+tn))\n",
        "  return tpr-fpr"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pI3ZUWv1gz53"
      },
      "source": [
        "##attack classifier:RF\n",
        "def get_trained_RF_models(train_data, test_data, num_models):\n",
        "    from sklearn.ensemble import RandomForestClassifier\n",
        "    (x_train, y_train), (x_test, y_test) = train_data, test_data\n",
        "    models = []\n",
        "    score=np.zeros(num_models)\n",
        "    #RF=RandomForestClassifier(random_state=0)\n",
        "    for i in range(num_models):\n",
        "        print('Training RF model : ', i)\n",
        "        models.append(RandomForestClassifier(random_state=0))\n",
        "        models[i].fit(x_train[i], y_train[i])\n",
        "        score[i] = models[i].score(x_test[i],y_test[i])\n",
        "        print('Random Forest model ', i, 'score : ',score)\n",
        "    return models, score"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lEvDXs_Xg7AY"
      },
      "source": [
        "n=25000\n",
        "xtrain=X_train[0:n]\n",
        "ytrain=y_train[0:n]\n",
        "#ytrain_agg=ytrain_agg.to_numpy()\n",
        "#ytrain_pred=model_np.predict_proba(xtrain_agg)\n",
        "xtest=X_test[0:n]\n",
        "ytest=y_test[0:n]\n",
        "#ytest_agg=ytest_agg.to_numpy()\n",
        "#ytest_pred=model_np.predict_proba(xtest_agg)\n",
        "target_train_agg = (xtrain,ytrain)\n",
        "target_test_agg = (xtest,ytest)\n",
        "target_train_data, target_test_data = sample_data(target_train_agg, target_test_agg, 1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cybaci9hg-yP"
      },
      "source": [
        "from sklearn.linear_model import LogisticRegression\n",
        "target_model=[]\n",
        "model= LogisticRegression(penalty='l2',max_iter=100,C=10000).fit(xtrain, ytrain)\n",
        "target_model.append(model)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-ir1yUXIhD9z"
      },
      "source": [
        "#logistic regression\n",
        "#non_private_model\n",
        "attack_test_agg=get_attack_dataset_lg(target_model, target_train_data, target_test_data, 1, TEST_SIZE)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Lf9kTveOhJTD"
      },
      "source": [
        "#non_private\n",
        "attack_model,scores = get_trained_RF_models(attack_train,attack_test_agg, 7)\n",
        "#scores=get_score_svm_models(attack_model,attack_train)\n",
        "tn,tp,fn,fp=get_leakage(attack_model, attack_test_agg)\n",
        "l1=lk(tn,tp,fn,fp)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ImtEsl0GmB7y"
      },
      "source": [
        "Privacy Leakage For Input Perturbation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FTKRFtremE1R"
      },
      "source": [
        "def Input_pert_train(epi,xtrain,ytrain,xtest,ytest):\n",
        "  delta=.00001\n",
        "  C=1\n",
        "  z=3*np.log(25000)\n",
        "  z1=z**2\n",
        "  sigma=z1*(8*np.log(2/(delta/2))+4*epi)/(epi**2)\n",
        "  #sigma=calc_sigma(ep[i],delta,18,1/C)\n",
        "  nsi=np.random.normal(loc=0.0, scale=sigma/n)\n",
        "  print(nsi)\n",
        "  X_in=xtrain-nsi\n",
        "  target_train_aggin = (X_in,ytrain)\n",
        "  target_test_aggin = (xtest,ytest)\n",
        "  target_train_datain, target_test_datain = sample_data(target_train_aggin, target_test_aggin, 1)\n",
        "  return X_in,target_train_datain,target_test_datain\n",
        "  "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Wm3yVuacmZu8"
      },
      "source": [
        "p=len(ep)\n",
        "lin=np.zeros(p)\n",
        "tn0=np.zeros(p)\n",
        "tp0=np.zeros(p)\n",
        "fn0=np.zeros(p)\n",
        "fp0=np.zeros(p)\n",
        "\n",
        "for i in range(len(ep)):\n",
        "     target_model=[]\n",
        "     X_in,target_train_datain,target_test_datain=Input_pert_train(ep[i],xtrain,ytrain,xtest,ytest)\n",
        "     in_model=LogisticRegression(penalty='l2',max_iter=100,C=10000).fit(X_in, ytrain)\n",
        "     target_model.append(in_model)\n",
        "     attack_test_agg=get_attack_dataset_lg(target_model, target_train_datain, target_test_datain, 1, TEST_SIZE)\n",
        "     tn0[i],tp0[i],fn0[i],fp0[i]=get_leakage(attack_model, attack_test_agg)\n",
        "     lin[i]=lk(tn0[i],tp0[i],fn0[i],fp0[i])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pf9G-8dymcrK"
      },
      "source": [
        "print(lin)\n",
        "print(tp0)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_O2IaDIwi0Mu"
      },
      "source": [
        "Privacy Leakage for Objective Perturbation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u17ecfmyixks"
      },
      "source": [
        "from diffprivlib.models import LogisticRegression\n",
        "p=len(ep)\n",
        "lobj=np.zeros(p)\n",
        "tn1=np.zeros(p)\n",
        "tp1=np.zeros(p)\n",
        "fn1=np.zeros(p)\n",
        "fp1=np.zeros(p)\n",
        "for i in range(len(ep)):\n",
        "     target_model=[]\n",
        "     obj_model=LogisticRegression(data_norm=12, epsilon=ep[i],penalty='l2',max_iter=100,C=10000).fit(xtrain, ytrain)\n",
        "     target_model.append(obj_model)\n",
        "     attack_test_agg=get_attack_dataset_lg(target_model, target_train_data, target_test_data, 1, TEST_SIZE)\n",
        "     tn1[i],tp1[i],fn1[i],fp1[i]=get_leakage(attack_model, attack_test_agg)\n",
        "     lobj[i]=lk(tn1[i],tp1[i],fn1[i],fp1[i])\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "anG1-DnoE5h_"
      },
      "source": [
        "print(lobj)\n",
        "print(tp1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RoTBaqi-pyPT"
      },
      "source": [
        "Privacy Leakage: Gradient Perturbation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Gol75-cFp3n8"
      },
      "source": [
        "p=len(ep)\n",
        "#lgd=np.zeros(p)\n",
        "#tn2=np.zeros(p)\n",
        "#tp2=np.zeros(p)\n",
        "#fn2=np.zeros(p)\n",
        "#fp2=np.zeros(p)\n",
        "for i in range(2,len(ep)):\n",
        "     target_model=[]\n",
        "     modelSGD=globals()['model_sgd%s' % i]\n",
        "     target_model.append(modelSGD)\n",
        "     attack_test_agg=get_attack_dataset_lg(target_model, target_train_data, target_test_data, 1, TEST_SIZE)\n",
        "     tn2[i],tp2[i],fn2[i],fp2[i]=get_leakage(attack_model, attack_test_agg)\n",
        "     lgd[i]=lk(tn2[i],tp2[i],fn2[i],fp2[i])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8WP1vbytpXGG"
      },
      "source": [
        "print(lgd)\n",
        "print(tp2)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NyOdK2dKqTXv"
      },
      "source": [
        "Output Perturbation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f9my0A7cqWQ5"
      },
      "source": [
        "from tensorflow import keras\n",
        "def output_pert(ep):\n",
        "  #add noise to the weights\n",
        "  #noise=(2/(n1*ep[i]*.0001))\n",
        "  n1=25000 \n",
        "  delta=.00001\n",
        "  s=2/(n1*.0001)  #lambda=.0001\n",
        "  model1=keras.models.load_model(\"my_model\")\n",
        "  w=model1.get_weights()\n",
        "  #print(w[0])\n",
        "  W=w\n",
        "  sigma = np.sqrt(2 * np.log(1.25 / delta)) * (s / ep)\n",
        "  #ns=np.random.laplace(loc=0.0, scale=noise)\n",
        "  ns=np.random.normal(loc=0.0, scale=sigma)\n",
        "  print(ns)\n",
        "  #print(ns)\n",
        "  for j in range(2):\n",
        "       W[j]=W[j]+ns\n",
        "  #print(W[0])\n",
        "  model2=keras.models.load_model(\"my_model\")\n",
        "  model2.set_weights(W)\n",
        "  #acc_out[i]=calc_labelAcc(model2,xtest,testY[0:n])\n",
        "  return model2\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vh-JHl85YJH6"
      },
      "source": [
        "p=len(ep)\n",
        "#lout=np.zeros(p)\n",
        "#tn3=np.zeros(p)\n",
        "#tp3=np.zeros(p)\n",
        "#fn3=np.zeros(p)\n",
        "#fp3=np.zeros(p)\n",
        "for i in range(3,p):\n",
        "     target_model=[]\n",
        "     model_out=output_pert(ep[i])\n",
        "     target_model.append(model_out)\n",
        "     attack_test_agg=get_attack_dataset_lg(target_model, target_train_data, target_test_data, 1, TEST_SIZE)\n",
        "     tn3[i],tp3[i],fn3[i],fp3[i]=get_leakage(attack_model, attack_test_agg)\n",
        "     lout[i]=lk(tn3[i],tp3[i],fn3[i],fp3[i])\n",
        "     print(\"complete\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fVOdfDk9pC0-"
      },
      "source": [
        "print(lout)\n",
        "print(tp3)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HzT5zjb-jMYh"
      },
      "source": [
        "PATE_attack"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IqmeDBG1jOgl"
      },
      "source": [
        "def PATE(ep,xtest1,ytest1):\n",
        "            sum_t=np.zeros([len(ytest1),7])\n",
        "            for k1 in range(100):\n",
        "               sum=PATE_sum(xtest1,ytest1)\n",
        "               sum_s=np.zeros(sum.shape)\n",
        "               sum_n=sum\n",
        "               sum_s=sum_n+np.random.laplace(loc=0.0, scale=1/ep)\n",
        "               sum_t=sum_s+sum_t\n",
        "\n",
        "            return sum_t/100\n",
        "          \n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HceI_rDdjSv9"
      },
      "source": [
        "def PATE_sum(xtest1,ytest1):\n",
        "  sum=np.zeros([len(ytest1),7])\n",
        "  for iter in range(M):\n",
        "    New_model=globals()['model_pate%s' % iter]\n",
        "    y=np.zeros([len(ytest1),7])\n",
        "    y=New_model.predict_proba(xtest1)\n",
        "  #print(y.shape)\n",
        "    y=vote(y)\n",
        "    sum=y+sum\n",
        "  return np.average(sum)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XrTgO6IkjWl5"
      },
      "source": [
        "#Get the prediction vectors from the DP model\n",
        "def get_attack_dataset_combined_lg(x_train, x_test, train_pred, y_train, y_test, test_pred):\n",
        "    # generate dataset for the attack model\n",
        "    #(x_train, y_train), (x_test, y_test) = train_data, test_data\n",
        "    data_size=len(y_train)\n",
        "    num_class = 7\n",
        "    x_data, y_data = [[] for _ in range(num_class)], [[] for _ in range(num_class)]\n",
        "    #for i in range(num_models):\n",
        "        # IN data\n",
        "    #x_temp, y_temp = resample(x_train, y_train, n_samples=data_size, random_state=0)\n",
        "    for j in range(data_size):\n",
        "            y_idx = y_train[j]\n",
        "            x_data[y_idx].append(train_pred[j])\n",
        "            #print(train_pred[j])\n",
        "            #x_data[y_idx].append(models.predict(x_temp[j:j+1])[0])\n",
        "            y_data[y_idx].append(IN)\n",
        "        # OUT data\n",
        "    #x_temp, y_temp = resample(x_test, y_test, n_samples=data_size, random_state=0)\n",
        "    for j in range(data_size):\n",
        "            y_idx = y_test[j]\n",
        "            #x_data[y_idx].append(models.predict(x_temp[j:j+1])[0])\n",
        "            x_data[y_idx].append(test_pred[j])\n",
        "            y_data[y_idx].append(OUT)\n",
        "    return x_data, y_data"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0aLaSiZojbrn"
      },
      "source": [
        "p=len(ep)\n",
        "lpate=np.zeros(p)\n",
        "tn4=np.zeros(p)\n",
        "tp4=np.zeros(p)\n",
        "fn4=np.zeros(p)\n",
        "fp4=np.zeros(p)\n",
        "for i in range(len(ep)):\n",
        "     ytrain_pred=PATE(ep[i],xtrain,ytrain)\n",
        "     ytest_pred=PATE(ep[i],xtest,ytest)\n",
        "     attack_test_agg=get_attack_dataset_combined_lg(xtrain,xtest,ytrain_pred,ytrain,ytest,ytest_pred)\n",
        "     tn4[i],tp4[i],fn4[i],fp4[i]=get_leakage(attack_model, attack_test_agg)\n",
        "     lpate[i]=lk(tn4[i],tp4[i],fn4[i],fp4[i])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hO-Smje9tm5z"
      },
      "source": [
        "print(lpate)\n",
        "print(tp4)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rROFt9xusaDF"
      },
      "source": [
        "[0.      0.00792 0.00792 0.00792 0.00792 0.00792 0.00792]\n",
        "[ 8143. 14715. 14715. 14715. 14715. 14715. 14715.]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LqvKWltGggBJ"
      },
      "source": [
        "Plot the graph"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "slw484suhLez"
      },
      "source": [
        "import pandas as pd \n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns #grafikleştirme için\n",
        "import matplotlib.pyplot as plt \n",
        "from google.colab import files\n",
        "test1 = plt.figure()\n",
        "plt.semilogx(ep,lobj,color=\"black\",marker='D',label='Objective Perturbation',linewidth=1.5)\n",
        "plt.semilogx(ep,lgd,color=\"black\",marker='*',label='Gradient Perturbation',linewidth=1.5)\n",
        "plt.semilogx(ep,lout,color=\"black\",marker='<',label='Output Perturbation',linewidth=1.5)\n",
        "plt.semilogx(ep,lpate,color=\"black\",marker='o',label='Prediction Perturbation',linewidth=1.5)\n",
        "#plt.semilogx(ep,non_p-acc_in,color=\"orange\",marker='.',linestyle=\"--\",label='Input',linewidth=1.5)\n",
        "#plt.plot(ep,non_p,color=\"red\",marker='*',linestyle=\"--\",label='Non-Private Model',linewidth=2)\n",
        "plt.legend(loc=1,fontsize=12)\n",
        "plt.xlabel(\"Privacy Budget($\\epsilon$)\",fontsize=13)\n",
        "plt.ylabel(\"Privacy Leakage\",fontsize=15)\n",
        "#plt.xscale('symlog', linthreshy=0.1)\n",
        "#plt.ylim([-.1,1])\n",
        "plt.xticks(size = 10)\n",
        "plt.yticks(size = 8)\n",
        "plt.ylim([-.02,.25])\n",
        "#y.set_color(\"black\")\n",
        "plt.rcParams[\"axes.edgecolor\"] = \"black\"\n",
        "plt.rcParams[\"axes.linewidth\"] = 1\n",
        "plt.rcParams['axes.facecolor'] = 'white'\n",
        "#test1.set_facecolor('white')\n",
        "test1.show()\n",
        "test1.savefig('logistic_perturb_leakage_loan2.pdf')\n",
        "files.download('logistic_perturb_leakage_loan2.pdf')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9YflmcKwpdlM"
      },
      "source": [
        "import pandas as pd \n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns #grafikleştirme için\n",
        "import matplotlib.pyplot as plt \n",
        "from google.colab import files\n",
        "test1 = plt.figure()\n",
        "plt.semilogx(ep,tp1,color=\"black\",marker='D',label='Objective Perturbation',linewidth=1.5)\n",
        "plt.semilogx(ep,tp2,color=\"black\",marker='*',label='Gradient Perturbation',linewidth=1.5)\n",
        "plt.semilogx(ep,tp3,color=\"black\",marker='<',label='Output Perturbation',linewidth=1.5)\n",
        "plt.semilogx(ep,tp4,color=\"black\",marker='o',label='Prediction Perturbation',linewidth=1.5)\n",
        "#plt.semilogx(ep,non_p-acc_in,color=\"orange\",marker='.',linestyle=\"--\",label='Input',linewidth=1.5)\n",
        "#plt.plot(ep,non_p,color=\"red\",marker='*',linestyle=\"--\",label='Non-Private Model',linewidth=2)\n",
        "plt.legend(loc=1,fontsize=10)\n",
        "plt.xlabel(\"Privacy Budget($\\epsilon$)\",fontsize=13)\n",
        "plt.ylabel(\"True Revealed Data (True Positive)\",fontsize=12)\n",
        "#plt.xscale('symlog', linthreshy=0.1)\n",
        "#plt.ylim([-.1,1])\n",
        "plt.xticks(size = 10)\n",
        "plt.yticks(size = 8)\n",
        "plt.ylim([0,30000])\n",
        "#y.set_color(\"black\")\n",
        "plt.rcParams[\"axes.edgecolor\"] = \"black\"\n",
        "plt.rcParams[\"axes.linewidth\"] = 1\n",
        "plt.rcParams['axes.facecolor'] = 'white'\n",
        "#test1.set_facecolor('white')\n",
        "test1.show()\n",
        "test1.savefig('logistic_perturb_tp_loan.pdf')\n",
        "files.download('logistic_perturb_tp_loan.pdf')"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}