notebooks/Data_Upload.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d80c5185-241c-4d84-8aed-421608657038",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import subprocess\n",
    "import argparse\n",
    "import datetime\n",
    "import requests\n",
    "from requests.structures import CaseInsensitiveDict"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8e8de8b8-7ce2-45ee-aade-406cf447f577",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "API-key and Domain-ID which need to be copy/pasted from the Uptrendz platform:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5cae4b21-2634-40e1-a075-08f0a5a71c1e",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "API_KEY = '<INSERT YOUR API KEY>'\n",
    "DOMAIN_ID = '<INSERT YOUR DOMAIN ID>'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bf9a7322-7ee8-449b-a25e-486b26cbe010",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### Download MovieLens user data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8fbcffab-3e39-4cfe-ba73-e03fc53ea60d",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "user_url = \"http://files.grouplens.org/datasets/movielens/ml-100k/u.user\"\n",
    "\n",
    "colnames=['id', 'age', 'gender', 'occupation', 'zip'] \n",
    "user_df = pd.read_csv(user_url, sep='|', header=None, names=colnames)\n",
    "user_df.head(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8dc7e6b8-d71f-4fae-b776-1d8338744c8d",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### Prepare user data to be in the appropriate format\n",
    "\n",
    "Example request:\n",
    "```json\n",
    " curl --location --request POST 'https://app.v2.uptrendz.ai/di/users/user'\n",
    "--header 'x-api-key: <INSERT YOUR API KEY>'\n",
    "--header 'x-domain-id: <INSERT YOUR DOMAIN ID>'\n",
    "--header 'Content-Type: application/json'\n",
    "--data-raw\n",
    "\n",
    "'[{\n",
    "    \"zip\": \"some-zip-value\",\n",
    "    \"occupation\": \"some-occupation-value\",\n",
    "    \"gender\": \"some-gender-value\",\n",
    "    \"id\": \"some-id-value\",\n",
    "    \"age\": 1\n",
    "}]'\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a6c83017-79ef-4faf-aa7f-1f1d54d382a1",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "user_data = []\n",
    "\n",
    "for idx, row in user_df.iterrows():\n",
    "    # create json object to send\n",
    "    user_data.append({\n",
    "        \"id\": str(row.id),\n",
    "        \"age\": row.age,\n",
    "        \"gender\": row.gender,\n",
    "        \"occupation\": row.occupation,\n",
    "        \"zip\": row.zip\n",
    "    })\n",
    "print(\"Prepared \" + str(len(user_data)) + \" users\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c698a2e7-2aa8-4da8-88a6-573f078736c9",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### Send user data to the Uptrendz platform\n",
    "\n",
    "Use the appropriate API key and Domain Id which is displayed in the platform"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e401d21d-617c-4ea3-bfed-de637ce5068b",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "headers = {\n",
    "    'x-api-key': API_KEY,\n",
    "    'x-domain-id': DOMAIN_ID,\n",
    "    'Content-Type': 'application/json'\n",
    "}\n",
    "\n",
    "response = requests.post('https://app.v2.uptrendz.ai/di/users/user', headers=headers, json=user_data)\n",
    "response.text"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "16de5e50-267e-49c8-9292-e632e1f21c9a",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    },
    "tags": []
   },
   "source": [
    "### Download MovieLens item data and clean it up\n",
    "\n",
    "Aggregate genres to have a list of genres for each movie <br/>\n",
    "Remove unused columns (i.e., which have not been defined in the Uptrendz platform)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2ab37f5b-b908-447d-aaf9-cbb242a7ac59",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "item_url = \"http://files.grouplens.org/datasets/movielens/ml-100k/u.item\"\n",
    "\n",
    "genres = [\"unknown\", \"action\", \"adventure\", \"animation\", \"children\", \"comedy\", \"crime\", \"documentary\", \"drama\", \n",
    "          \"fantasy\", \"noir\", \"horror\", \"musical\", \"mystery\", \"romance\", \"scifi\", \"thriller\", \"war\", \"western\"] \n",
    "colnames=['id', 'title', 'release_date', 'video_release_date', 'imdb_url'] \n",
    "colnames.extend(genres)\n",
    "\n",
    "item_df = pd.read_csv(item_url, sep='|', header=None, names=colnames, encoding='latin-1')\n",
    "\n",
    "\n",
    "# merge genres into a list within 1 column\n",
    "for col in item_df[genres]:\n",
    "    item_df[col]=item_df[col].replace(1, item_df[col].name)\n",
    "    item_df['genres'] = [[e for e in row if e!=0] for row in item_df[genres].values.tolist()]\n",
    "\n",
    "# let's just keep the item id, title, release date and newly created list of genres\n",
    "item_df = item_df.drop(genres, axis=1)\n",
    "item_df = item_df.drop([\"video_release_date\",\"imdb_url\"], axis=1)\n",
    "\n",
    "item_df.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b317300e-59fb-41a6-ad2e-9c1620610b44",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### Prepare item data to be in the appropriate format\n",
    "\n",
    "Date values need to be in the *%Y-%m-%dT%H:%M:%SZ* <br/>\n",
    "Id needs to be of type string\n",
    "\n",
    "Example request:\n",
    "```json\n",
    " curl --location --request POST 'https://app.v2.uptrendz.ai/di/items/movie'\n",
    "--header 'x-api-key: <INSERT YOUR API KEY>'\n",
    "--header 'x-domain-id: <INSERT YOUR DOMAIN ID>'\n",
    "--header 'Content-Type: application/json'\n",
    "--data-raw\n",
    "\n",
    "'[{\n",
    "    \"release_date\": \"2022-10-09T15:41:38Z\",\n",
    "    \"genres\": [\"some-genres-value\"],\n",
    "    \"id\": \"some-id-value\",\n",
    "    \"title\": \"some-title-value\"\n",
    "}]'\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "471ec556-1ecc-4f99-89b5-fb2a8cd2ce91",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "default_date = \"01-Jan-1995\"\n",
    "item_data = []\n",
    "\n",
    "for idx, row in item_df.iterrows():\n",
    "    # handle NaN values\n",
    "    if row.release_date is np.nan:\n",
    "        row.release_date = default_date\n",
    "    # create json object to send\n",
    "    item_data.append({\n",
    "        # convert data string to timestamp and convert to appropriate format for the API\n",
    "        \"release_date\": datetime.datetime.strptime(row.release_date, \"%d-%b-%Y\").strftime(\"%Y-%m-%dT%H:%M:%SZ\"),\n",
    "        \"genres\": row.genres,\n",
    "        \"id\": str(row.id),\n",
    "        \"title\": row.title\n",
    "    })\n",
    "print(\"Prepared \" + str(len(item_data)) + \" items\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "80861cd5-3b62-4fa4-b120-8c689d2baa80",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### Send item data to the Uptrendz platform\n",
    "\n",
    "Use the appropriate API key and Domain Id which is displayed in the platform"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e988abcf-ca67-46b6-8572-421d6fef77b8",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "headers = {\n",
    "    'x-api-key': API_KEY,\n",
    "    'x-domain-id': DOMAIN_ID,\n",
    "    'Content-Type': 'application/json'\n",
    "}\n",
    "\n",
    "response = requests.post('https://app.v2.uptrendz.ai/di/items/movie', headers=headers, json=item_data)\n",
    "response.text"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "37994241-3c7b-4cda-8078-1d7a096c7c27",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### Download MovieLens interaction data\n",
    "\n",
    "Convert timestamp to date"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bedba96e-9336-44c3-a000-92927a8a7bec",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "interaction_url = \"http://files.grouplens.org/datasets/movielens/ml-100k/u.data\"\n",
    "\n",
    "colnames=['u_id', 'i_id', 'rating', 'timestamp'] \n",
    "interaction_df = pd.read_csv(interaction_url, sep='\\t', header=None, names=colnames)\n",
    "    \n",
    "interaction_df['timestamp'] = interaction_df['timestamp'].apply(lambda x: pd.to_datetime(x, unit='s', origin='unix') )\n",
    "\n",
    "interaction_df.head(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0dfe240a-d828-4a19-80b6-0d1cf26ca441",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### Prepare interaction data to be in the appropriate format\n",
    "\n",
    "Date values need to be in the *%Y-%m-%dT%H:%M:%SZ* <br/>\n",
    "User and item ids needs to be of type string\n",
    "\n",
    "Example request:\n",
    "```json\n",
    " curl --location --request POST 'https://app.v2.uptrendz.ai/di/interactions'\n",
    "--header 'x-api-key: <INSERT YOUR API KEY>'\n",
    "--header 'x-domain-id: <INSERT YOUR DOMAIN ID>'\n",
    "--header 'Content-Type: application/json'\n",
    "--data-raw\n",
    "\n",
    "'[{\n",
    "    \"item_id\": \"some-item-id\",\n",
    "    \"user_id\": \"some-user-id\",\n",
    "    \"type\": \"rating\",\n",
    "    \"value\": 1,\n",
    "    \"timestamp\": \"2022-10-09T15:41:38Z\"\n",
    "}]'\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7bfe30b2-3b63-422e-8e3c-a6833bc349f7",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "interaction_data = []\n",
    "\n",
    "for idx, row in interaction_df.iterrows():\n",
    "    # create json object to send\n",
    "    interaction_data.append({\n",
    "        # convert data string to timestamp and convert to appropriate format for the API\n",
    "        \"timestamp\": row.timestamp.strftime(\"%Y-%m-%dT%H:%M:%SZ\"),\n",
    "        \"user_id\": str(row.u_id),\n",
    "        \"item_id\": str(row.i_id),\n",
    "        \"type\": \"rating\", # needs to have the value as defined in the dashboard\n",
    "        \"value\": row.rating\n",
    "    })\n",
    "print(\"Prepared \" + str(len(interaction_data)) + \" interactions\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3d8c9256-820f-4fa4-8716-a388b92cd41d",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    },
    "tags": []
   },
   "source": [
    "### Send interaction data to the Uptrendz platform\n",
    "\n",
    "Use the appropriate API key and Domain Id which is displayed in the platform\n",
    "\n",
    "* Note, we can submit a maximum of 40,000 interactions per request"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6232140b-7042-4724-a660-3dfdf19513a2",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "headers = {\n",
    "    'x-api-key': API_KEY,\n",
    "    'x-domain-id': DOMAIN_ID,\n",
    "    'Content-Type': 'application/json'\n",
    "}\n",
    "\n",
    "# let's divide interaction data into chunks of 5,000 interactions\n",
    "batch_size = 5000\n",
    "chunks = [interaction_data[x:x+batch_size] for x in range(0, len(interaction_data), batch_size)]\n",
    "\n",
    "for chunk in chunks:\n",
    "    response = requests.post('https://app.v2.uptrendz.ai/di/interactions', headers=headers, json=chunk)\n",
    "    print(response.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a8b92018-0fcc-43ba-a2ea-c9701d1d1b52",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}