Observer.2.0.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import sklearn.cluster as cluster\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.preprocessing import StandardScaler as SC\n",
    "from sklearn.neighbors import KNeighborsRegressor as KNN\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "\n",
    "big_data = pd.read_csv('airbnb_coded.2.0.csv')\n",
    "sc = SC()\n",
    "sc_y = SC()\n",
    "le = LabelEncoder()\n",
    "knn = KNN(n_neighbors=65, weights='distance')\n",
    "\n",
    "pd.options.mode.chained_assignment = None  # default='warn'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1334\n",
      "1482\n",
      "3780\n",
      "4827\n",
      "6359\n",
      "6854\n",
      "7987\n",
      "9016\n",
      "9768\n",
      "11343\n",
      "11715\n",
      "12215\n",
      "14009\n",
      "14971\n",
      "15301\n",
      "15662\n",
      "17309\n",
      "17373\n",
      "9000.0\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.compose import ColumnTransformer\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "\n",
    "ct = ColumnTransformer(transformers=[('encoder', OneHotEncoder(), [-1])], remainder='passthrough')\n",
    "var = 'bathrooms'\n",
    "var2 = 'accommodates'\n",
    "var3 = 'cleaning_fee'\n",
    "var4 = 'bedrooms'\n",
    "var5 = 'beds'\n",
    "var6 = 'minimum_nights_x'\n",
    "var7 = 'maximum_nights'\n",
    "var8 = 'number_of_reviews_y'\n",
    "var9 = 'listing_class'\n",
    "var10 = 'price_x'\n",
    "var11 = 'bed_code'\n",
    "var12 = 'cancellation_policy_mapped'\n",
    "var13 = 'room_code'\n",
    "var14 = 'superhost'\n",
    "var15 = 'instant_book'\n",
    "var16 = 'review_scores_location'\n",
    "var17 = 'review_scores_rating'\n",
    "var18 = 'host_total_listings_count'\n",
    "var19 = 'availability_90'\n",
    "var20 = 'room_code'\n",
    "var21 = 'extra_people'\n",
    "var22 = 'property_type'\n",
    "\n",
    "\n",
    "data = big_data[[var, var2, var3, var22, var10]]\n",
    "\n",
    "\n",
    "data[var] = data[var].fillna(data[var].median())\n",
    "data[var2] = data[var2].fillna(data[var2].median())\n",
    "data[var3] = data[var3].fillna(data[var3].median())\n",
    "# data[var4] = data[var4].fillna(data[var4].median())\n",
    "# data[var5] = data[var5].fillna(data[var5].median())\n",
    "\n",
    "# data[var6][17990] = data[var6].median()\n",
    "# data[var6][6215] = data[var6].median()\n",
    "# data[var6][11299] = data[var6].median()\n",
    "# data[var6][12737] = data[var6].median()\n",
    "# data[var6][14598] = data[var6].median()\n",
    "# data[var7][872] = 8000\n",
    "# data[var7][2372] = 8000\n",
    "# data[var7][4391] = 8000\n",
    "# data[var7][17952] = 8000\n",
    "# data[var7][17990] = data[var7].median()\n",
    "# data[var7][6351] = data[var7].median()\n",
    "# data[var7][8341] = data[var7].median()\n",
    "\n",
    "\n",
    "# data[var6] = data[var6].fillna(data[var6].median())\n",
    "# data[var7] = data[var7].fillna(data[var7].median())\n",
    "# data[var8] = data[var8].fillna(data[var8].median())\n",
    "data[var10] = data[var10].fillna(data[var10].median())\n",
    "# data[var11] = data[var11].fillna(data[var11].median())\n",
    "# data[var12] = data[var12].fillna(data[var12].median())\n",
    "# data[var13] = data[var13].fillna(data[var13].median())\n",
    "# data[var14] = data[var14].fillna(data[var14].median())\n",
    "# data[var15] = data[var15].fillna(data[var15].median())\n",
    "# data[var16] = data[var16].fillna(data[var16].median())\n",
    "# data[var17] = data[var17].fillna(data[var17].median())\n",
    "# data[var18] = data[var18].fillna(data[var18].median())\n",
    "# data[var19] = data[var19].fillna(data[var19].median())\n",
    "# data[var20] = data[var20].fillna(data[var20].median())\n",
    "# data[var21] = data[var21].fillna(data[var21].median())\n",
    "data['property_type'] = data['property_type'].fillna('Apartment')\n",
    "\n",
    "\n",
    "# data[var6][17990] = data[var6].median()\n",
    "\n",
    "X = data.iloc[:, :-1].values\n",
    "y = data.iloc[:, -1].values\n",
    "X = ct.fit_transform(X)\n",
    "X = X.todense()\n",
    "X = np.array(X)\n",
    "# X = ct.fit_transform(X)\n",
    "# # X = X.todense()\n",
    "# X = np.array(X)\n",
    "# X = ct.fit_transform(X)\n",
    "# # X = X.todense()\n",
    "# X = np.array(X)\n",
    "# X = ct.fit_transform(X)\n",
    "# # X = X.todense()\n",
    "# X = np.array(X)\n",
    "# # ct = ColumnTransformer(transformers=[('encoder', OneHotEncoder(), [-4])], remainder='passthrough')\n",
    "# X = ct.fit_transform(X)\n",
    "# # X = X.todense()\n",
    "# X = np.array(X)\n",
    "# ct = ColumnTransformer(transformers=[('encoder', OneHotEncoder(), [-5])], remainder='passthrough')\n",
    "# X = ct.fit_transform(X)\n",
    "# X = X.todense()\n",
    "# X = np.array(X)\n",
    "# ct = ColumnTransformer(transformers=[('encoder', OneHotEncoder(), [-6])], remainder='passthrough')\n",
    "# X = ct.fit_transform(X)\n",
    "# # X = X.todense()\n",
    "# X = np.array(X)\n",
    "\n",
    "# plt.scatter(data[var6].values, data[var7].values, s=100, c='red', alpha=0.5)\n",
    "# plt.show()\n",
    "\n",
    "# X[:,-8:] = sc.fit_transform(X[:,-8:])\n",
    "# X = sc.fit_transform(X)\n",
    "for i in range(len(y)):\n",
    "    if y[i]>=6000:\n",
    "        print(i)\n",
    "print(max(y.T))\n",
    "# X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.1)\n",
    "\n",
    "# data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.scatter(data[var22].values, data[var10].values, alpha=0.2)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from hdbscan import HDBSCAN as HDB\n",
    "# hdb = HDB(min_cluster_size=10, min_samples=16)\n",
    "# price_cluster = hdb.fit_predict(X)\n",
    "# price_cluster\n",
    "# data['price_class'] = price_cluster\n",
    "# big_data['price_class'] = price_cluster\n",
    "# data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# counter = 0\n",
    "# for index in price_cluster:\n",
    "#     if index == -1:\n",
    "#         counter +=1\n",
    "# counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clusterer = cluster.KMeans(n_clusters=6)\n",
    "price_cluster = clusterer.fit_predict(X)\n",
    "price_cluster\n",
    "data['price_class'] = price_cluster\n",
    "big_data['price_class'] = price_cluster\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.scatter(data[var3].values, data['price_class'].values, alpha=0.1, c='red')\n",
    "plt.scatter(data[var10].values, data['price_class'].values, alpha=0.1, c='blue')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "big_data.to_csv('airbnb_coded.3.0.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import sklearn.cluster as cluster\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.preprocessing import StandardScaler as SC\n",
    "from sklearn.neighbors import KNeighborsRegressor as KNN\n",
    "\n",
    "big_data = pd.read_csv('airbnb_coded.3.0.csv')\n",
    "sc = SC()\n",
    "sc_y = SC()\n",
    "knn = KNN(n_neighbors=65, weights='distance')\n",
    "\n",
    "pd.options.mode.chained_assignment = None  # default='warn'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'big_data' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-342637946f1d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     28\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     29\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 30\u001b[0;31m \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbig_data\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mvar2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvar3\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvar6\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvar18\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvar19\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvar21\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvar8\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvar\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvar29\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvar30\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvar9\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvar13\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvar31\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvar12\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvar20\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvar22\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvar10\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     31\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     32\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mvar\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mvar\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfillna\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mvar\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmedian\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'big_data' is not defined"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.compose import ColumnTransformer\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "\n",
    "ct = ColumnTransformer(transformers=[('encoder', OneHotEncoder(), [-1])], remainder='passthrough')\n",
    "var = 'bathrooms'\n",
    "var2 = 'accommodates'\n",
    "var3 = 'cleaning_fee'\n",
    "var4 = 'bedrooms'\n",
    "var5 = 'beds'\n",
    "var6 = 'minimum_nights_x'\n",
    "var7 = 'maximum_nights'\n",
    "var8 = 'number_of_reviews_y'\n",
    "var9 = 'listing_class'\n",
    "var10 = 'price_x'\n",
    "var11 = 'bed_code'\n",
    "var12 = 'cancellation_policy_mapped'\n",
    "var13 = 'room_code'\n",
    "var14 = 'superhost'\n",
    "var15 = 'instant_book'\n",
    "var16 = 'review_scores_location'\n",
    "var17 = 'review_scores_rating'\n",
    "var18 = 'host_total_listings_count'\n",
    "var19 = 'availability_90'\n",
    "var20 = 'room_code'\n",
    "var21 = 'extra_people'\n",
    "var22 = 'price_class'\n",
    "\n",
    "\n",
    "data = big_data[[var, var2, var3, var6, var8, var18, var17, var21, var19, var9, var22, var12, var20, var14, var10]]\n",
    "\n",
    "data[var] = data[var].fillna(data[var].median())\n",
    "data[var2] = data[var2].fillna(data[var2].median())\n",
    "data[var3] = data[var3].fillna(data[var3].median())\n",
    "# data[var4] = data[var4].fillna(data[var4].median())\n",
    "# data[var5] = data[var5].fillna(data[var5].median())\n",
    "\n",
    "data[var6][17990] = data[var6].median()\n",
    "data[var6][6215] = data[var6].median()\n",
    "data[var6][11299] = data[var6].median()\n",
    "data[var6][12737] = data[var6].median()\n",
    "data[var6][14598] = data[var6].median()\n",
    "# data[var7][872] = 8000\n",
    "# data[var7][2372] = 8000\n",
    "# data[var7][4391] = 8000\n",
    "# data[var7][17952] = 8000\n",
    "# data[var7][17990] = data[var7].median()\n",
    "# data[var7][6351] = data[var7].median()\n",
    "# data[var7][8341] = data[var7].median()\n",
    "\n",
    "\n",
    "data[var6] = data[var6].fillna(data[var6].median())\n",
    "# data[var7] = data[var7].fillna(data[var7].median())\n",
    "data[var8] = data[var8].fillna(data[var8].median())\n",
    "data[var10] = data[var10].fillna(data[var10].median())\n",
    "# data[var11] = data[var11].fillna(data[var11].median())\n",
    "data[var12] = data[var12].fillna(data[var12].median())\n",
    "# data[var13] = data[var13].fillna(data[var13].median())\n",
    "data[var14] = data[var14].fillna(data[var14].median())\n",
    "# data[var15] = data[var15].fillna(data[var15].median())\n",
    "# data[var16] = data[var16].fillna(data[var16].median())\n",
    "data[var17] = data[var17].fillna(data[var17].median())\n",
    "data[var18] = data[var18].fillna(data[var18].median())\n",
    "data[var19] = data[var19].fillna(data[var19].median())\n",
    "data[var20] = data[var20].fillna(data[var20].median())\n",
    "data[var21] = data[var21].fillna(data[var21].median())\n",
    "\n",
    "\n",
    "data[var6][17990] = data[var6].median()\n",
    "\n",
    "data = data.drop(data.index[17990])\n",
    "data = data.drop(data.index[14598])\n",
    "\n",
    "X = data.iloc[:, :-1].values\n",
    "y = data.iloc[:, -1].values\n",
    "X = ct.fit_transform(X)\n",
    "# X = X.todense()\n",
    "X = np.array(X)\n",
    "X = ct.fit_transform(X)\n",
    "# X = X.todense()\n",
    "X = np.array(X)\n",
    "X = ct.fit_transform(X)\n",
    "# X = X.todense()\n",
    "X = np.array(X)\n",
    "X = ct.fit_transform(X)\n",
    "# X = X.todense()\n",
    "X = np.array(X)\n",
    "# ct = ColumnTransformer(transformers=[('encoder', OneHotEncoder(), [-4])], remainder='passthrough')\n",
    "X = ct.fit_transform(X)\n",
    "# X = X.todense()\n",
    "X = np.array(X)\n",
    "# ct = ColumnTransformer(transformers=[('encoder', OneHotEncoder(), [-5])], remainder='passthrough')\n",
    "# X = ct.fit_transform(X)\n",
    "# X = X.todense()\n",
    "# X = np.array(X)\n",
    "# ct = ColumnTransformer(transformers=[('encoder', OneHotEncoder(), [-6])], remainder='passthrough')\n",
    "# X = ct.fit_transform(X)\n",
    "# # X = X.todense()\n",
    "# X = np.array(X)\n",
    "\n",
    "# plt.scatter(data[var6].values, data[var7].values, s=100, c='red', alpha=0.5)\n",
    "# plt.show()\n",
    "\n",
    "print(X)\n",
    "# X[:,-8:] = sc.fit_transform(X[:,-8:])\n",
    "# X = sc.fit_transform(X)\n",
    "print(len(X[0]))\n",
    "# X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.1)\n",
    "\n",
    "# data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# X = sc.fit_transform(X)\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.3)\n",
    "# X_train = sc.fit_transform(X_train)\n",
    "# X_test = sc.fit_transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestRegressor as RFR\n",
    "rf = RFR(n_estimators=500, min_samples_leaf=1)\n",
    "rf = rf.fit(X_train, y_train)\n",
    "rf.score(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rf.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# rf = RFR(n_estimators=500, min_samples_leaf=10)\n",
    "# rf = rf.fit(X_train, y_train)\n",
    "# rf.score(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# rf.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import cross_val_score\n",
    "scores = cross_val_score(rf, X, \n",
    "                          y, scoring='r2', cv=10)\n",
    "print(scores)\n",
    "print(np.mean(scores))\n",
    "np.std(scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import cross_val_score\n",
    "scores = cross_val_score(rf, X, \n",
    "                          y, scoring='r2', cv=15)\n",
    "print(scores)\n",
    "print(np.mean(scores))\n",
    "np.std(scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "array = data[var10].isna()\n",
    "array = array.values\n",
    "for i in range(len(array)):\n",
    "    if array[i]:\n",
    "        print(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "knn.fit(X_train, y_train)\n",
    "knn.score(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "knn.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import cross_val_score\n",
    "scores = -cross_val_score(knn, X, \n",
    "                          y, scoring='neg_mean_absolute_error', cv=5)\n",
    "print(scores)\n",
    "np.mean(scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.tree import DecisionTreeRegressor as DTR\n",
    "dt = DTR(min_samples_leaf=10)\n",
    "dt = dt.fit(X_train, y_train)\n",
    "dt.score(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dt.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "scores = -cross_val_score(dt, X, \n",
    "                          y, scoring='neg_mean_absolute_error', cv=5)\n",
    "print(scores)\n",
    "np.mean(scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}