s3_inspect.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Inspecting a AWS S3 Bucket\n",
    "First import necessary modules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import boto3\n",
    "import configparser"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Get configuration from dwh.cfg file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "config = configparser.ConfigParser()\n",
    "config.read_file(open(\"dwh.cfg\"))\n",
    "\n",
    "KEY = config.get('AWS','KEY')\n",
    "SECRET = config.get('AWS','SECRET')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create a S3 bucket object. In my case it is in the us west 2 region and is called udacity-dend."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "s3 = boto3.client(\"s3\", region_name=\"us-west-2\", aws_access_key_id=KEY,\n",
    "                    aws_secret_access_key=SECRET)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Search only for \"song-data\" objects"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "searchobj = \"song_data\"\n",
    "lcobj = list(s3.list_objects_v2(Bucket=\"udacity-dend\", \n",
    "                                Prefix=searchobj + \"/\").values())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "#print first 10 objects\n",
    "i = 1\n",
    "sdlist = list()\n",
    "prefix = 's3://udacity-dend/'\n",
    "for o in lcobj[2]:\n",
    "    if i <= 20:\n",
    "        #print(o[\"Key\"])\n",
    "        sdlist.append(prefix+o[\"Key\"])\n",
    "        i += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "sdlist = [x for x in sdlist if x.find(\"json\") > -1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['s3://udacity-dend/song_data/A/A/A/TRAAAAK128F9318786.json',\n",
       " 's3://udacity-dend/song_data/A/A/A/TRAAAAV128F421A322.json',\n",
       " 's3://udacity-dend/song_data/A/A/A/TRAAABD128F429CF47.json',\n",
       " 's3://udacity-dend/song_data/A/A/A/TRAAACN128F9355673.json',\n",
       " 's3://udacity-dend/song_data/A/A/A/TRAAAEA128F935A30D.json',\n",
       " 's3://udacity-dend/song_data/A/A/A/TRAAAED128E0783FAB.json',\n",
       " 's3://udacity-dend/song_data/A/A/A/TRAAAEM128F93347B9.json',\n",
       " 's3://udacity-dend/song_data/A/A/A/TRAAAEW128F42930C0.json',\n",
       " 's3://udacity-dend/song_data/A/A/A/TRAAAFD128F92F423A.json',\n",
       " 's3://udacity-dend/song_data/A/A/A/TRAAAGR128F425B14B.json',\n",
       " 's3://udacity-dend/song_data/A/A/A/TRAAAHD128F42635A5.json',\n",
       " 's3://udacity-dend/song_data/A/A/A/TRAAAHJ128F931194C.json',\n",
       " 's3://udacity-dend/song_data/A/A/A/TRAAAHZ128E0799171.json',\n",
       " 's3://udacity-dend/song_data/A/A/A/TRAAAIR128F1480971.json',\n",
       " 's3://udacity-dend/song_data/A/A/A/TRAAAJN128F428E437.json',\n",
       " 's3://udacity-dend/song_data/A/A/A/TRAAAND12903CD1F1B.json',\n",
       " 's3://udacity-dend/song_data/A/A/A/TRAAANK128F428B515.json',\n",
       " 's3://udacity-dend/song_data/A/A/A/TRAAAOF128F429C156.json',\n",
       " 's3://udacity-dend/song_data/A/A/A/TRAAAPK128E0786D96.json']"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sdlist"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check the content of some json files for song data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "testdf = pd.read_json(sdlist[0], typ=\"series\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "artist_id           ARJNIUY12298900C91\n",
       "artist_latitude                   None\n",
       "artist_location                       \n",
       "artist_longitude                  None\n",
       "artist_name               Adelitas Way\n",
       "duration                       213.942\n",
       "num_songs                            1\n",
       "song_id             SOBLFFE12AF72AA5BA\n",
       "title                           Scream\n",
       "year                              2009\n",
       "dtype: object"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "testdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}