qualtrics_preprocess.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "source_dir = os.path.join(os.path.abspath('..'),'sourcedata','qualtrics')\n",
    "derivs_dir = os.path.join(os.path.abspath('..'),'derivatives','qualtrics')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "frames={}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for f in os.listdir(source_dir):\n",
    "    if f.startswith('180711'):\n",
    "        keyname = f.strip('180711_.csv')\n",
    "        fname = os.path.join(source_dir, f)\n",
    "        frames[keyname] = pd.read_csv(fname)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['econdec-post', 'eyetrack-pre', 'econdec-pre', 'eyetrack-post']"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "frames.keys()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Break out individual scales"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Econdec-Pre"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "econdec_pre_frame=frames['econdec-pre']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Demographics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "econdec_demographics_columns = ['Finished','SSID','Q1','Q2','Q3','Q4','Q5','Q5_6_TEXT','Q6','Q7_1',\n",
    "                 'Q8','Q9','Q10','Q11_1','Q12_1','Q13','Q14','Q14_4_TEXT','Q15']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "econdec_demographics_frame = econdec_pre_frame[econdec_demographics_columns]\n",
    "econdec_demographics_frame.columns = econdec_demographics_frame.iloc[0]\n",
    "econdec_demographics_frame = econdec_demographics_frame[econdec_demographics_frame['Finished'] == '1']\n",
    "econdec_demographics_frame.to_csv(os.path.join(derivs_dir,'0.indiv','econdec_demog.csv'),index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### PANAS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "econdec_panas_columns = ['Finished','SSID','Q42','Q43','Q44','Q45','Q46','Q47','Q48','Q49','Q50','Q51',\n",
    "                 'Q52','Q53','Q54','Q55','Q56','Q57','Q58','Q59','Q60','Q61']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "econdec_panas_frame = econdec_pre_frame[econdec_panas_columns]\n",
    "econdec_panas_frame.columns = econdec_panas_frame.iloc[0]\n",
    "econdec_panas_frame = econdec_panas_frame[econdec_panas_frame['Finished'] == '1']\n",
    "econdec_panas_frame.to_csv(os.path.join(derivs_dir,'0.indiv','econdec_panas.csv'),index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### BISBAS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "econdec_BISBAS_columns = ['Finished','SSID','Q17','Q18','Q19','Q20','Q21','Q22','Q23','Q24',\n",
    "                          'Q25','Q26','Q27','Q28','Q29','Q30','Q31','Q32','Q33',\n",
    "                          'Q34','Q35','Q36','Q37','Q38','Q39','Q40']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "econdec_BISBAS_frame = econdec_pre_frame[econdec_BISBAS_columns]\n",
    "econdec_BISBAS_frame.columns = econdec_BISBAS_frame.iloc[0]\n",
    "econdec_BISBAS_frame = econdec_BISBAS_frame[econdec_BISBAS_frame['Finished'] == '1']\n",
    "econdec_BISBAS_frame.to_csv(os.path.join(derivs_dir,'0.indiv','econdec_bisbas.csv'),index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### GDMS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "econdec_dms_columns = ['Finished','SSID','Q63','Q64','Q65','Q66','Q67','Q68','Q69','Q70','Q71',\n",
    "                       'Q72','Q73','Q74','Q75','Q76','Q77','Q78','Q79','Q80','Q81',\n",
    "                       'Q82','Q83','Q84','Q85','Q86','Q87']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "econdec_dms_frame = econdec_pre_frame[econdec_dms_columns]\n",
    "econdec_dms_frame.columns = econdec_dms_frame.iloc[0]\n",
    "econdec_dms_frame = econdec_dms_frame[econdec_dms_frame['Finished'] == '1']\n",
    "econdec_dms_frame.to_csv(os.path.join(derivs_dir,'0.indiv','econdec_gdms.csv'),index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Econdec-Post"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "econdec_post_frame=frames['econdec-post']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Financial Literacy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "econdec_finlit_columns = ['Finished','SSID','Q21','Q22']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "econdec_finlit_frame = econdec_post_frame[econdec_finlit_columns]\n",
    "econdec_finlit_frame.columns = econdec_finlit_frame.iloc[0]\n",
    "econdec_finlit_frame = econdec_finlit_frame[econdec_finlit_frame['Finished'] == '1']\n",
    "econdec_finlit_frame.to_csv(os.path.join(derivs_dir,'0.indiv','econdec_financ.csv'),index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Eyetrack-Pre"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "eyetrack_pre_frame=frames['eyetrack-pre']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Demographics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "eyetrack_demog_columns = ['Finished','SSID','Q4','Q6','Q8','Q10','Q12','Q12_6_TEXT','Q14',\n",
    "                          'Q16_1','Q18','Q20','Q22','Q24_1','Q26_1','Q28','Q30','Q30_4_TEXT','Q32']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "eyetrack_demog_frame = eyetrack_pre_frame[eyetrack_demog_columns]\n",
    "eyetrack_demog_frame.columns = eyetrack_demog_frame.iloc[0]\n",
    "eyetrack_demog_frame = eyetrack_demog_frame[eyetrack_demog_frame['Finished'] == '1']\n",
    "eyetrack_demog_frame.to_csv(os.path.join(derivs_dir,'0.indiv','eyetrack_demog.csv'),index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### PANAS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "eyetrack_pinas_columns = ['Finished','SSID','Q86','Q88','Q90','Q92','Q94','Q96','Q98','Q100',\n",
    "                          'Q102','Q104','Q106','Q108','Q110','Q112','Q114','Q116',\n",
    "                          'Q118','Q120','Q122','Q124']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "eyetrack_pinas_frame = eyetrack_pre_frame[eyetrack_pinas_columns]\n",
    "eyetrack_pinas_frame.columns = eyetrack_pinas_frame.iloc[0]\n",
    "eyetrack_pinas_frame = eyetrack_pinas_frame[eyetrack_pinas_frame['Finished'] == '1']\n",
    "eyetrack_pinas_frame.to_csv(os.path.join(derivs_dir,'0.indiv','eyetrack_panas.csv'),index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### BISBAS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "eyetrack_BISBAS_columns = ['Finished','SSID','Q36','Q38','Q40','Q42','Q44','Q46','Q48',\n",
    "                            'Q50','Q52','Q54','Q56','Q58','Q60','Q62','Q64','Q66',\n",
    "                            'Q68','Q70','Q72','Q74','Q76','Q78','Q80','Q82']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "eyetrack_BISBAS_frame = eyetrack_pre_frame[eyetrack_BISBAS_columns]\n",
    "eyetrack_BISBAS_frame.columns = eyetrack_BISBAS_frame.iloc[0]\n",
    "eyetrack_BISBAS_frame = eyetrack_BISBAS_frame[eyetrack_BISBAS_frame['Finished'] == '1']\n",
    "eyetrack_BISBAS_frame.to_csv(os.path.join(derivs_dir,'0.indiv','eyetrack_bisbas.csv'),index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### GDMS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "eyetrack_dms_columns = ['Finished','SSID','Q128','Q130','Q132','Q134','Q136','Q138','Q140',\n",
    "                        'Q142','Q144','Q146','Q148','Q150','Q152','Q154','Q156',\n",
    "                        'Q158','Q160','Q162','Q164','Q166','Q168','Q170','Q172',\n",
    "                        'Q174','Q176']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "eyetrack_dms_frame = eyetrack_pre_frame[eyetrack_dms_columns]\n",
    "eyetrack_dms_frame.columns = eyetrack_dms_frame.iloc[0]\n",
    "eyetrack_dms_frame = eyetrack_dms_frame[eyetrack_dms_frame['Finished'] == '1']\n",
    "eyetrack_dms_frame.to_csv(os.path.join(derivs_dir,'0.indiv','eyetrack_gdms.csv'),index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Eyetrack-Post"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "eyetrack_post_frame=frames['eyetrack-post']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Financial Literacy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "eyetrack_finlit_columns = ['Finished','SSID','Q14','Q15']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "eyetrack_finlit_frame = eyetrack_post_frame[eyetrack_finlit_columns]\n",
    "eyetrack_finlit_frame.columns = eyetrack_finlit_frame.iloc[0]\n",
    "eyetrack_finlit_frame = eyetrack_finlit_frame[eyetrack_finlit_frame['Finished'] == '1']\n",
    "eyetrack_finlit_frame.to_csv(os.path.join(derivs_dir,'0.indiv','eyetrack_financ.csv'),index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Combine eye-tracking and behavioral study"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "frames={}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "econdec_bisbas.csv\n",
      "econdec_demog.csv\n",
      "econdec_financ.csv\n",
      "econdec_gdms.csv\n",
      "econdec_panas.csv\n",
      "eyetrack_bisbas.csv\n",
      "eyetrack_demog.csv\n",
      "eyetrack_financ.csv\n",
      "eyetrack_gdms.csv\n",
      "eyetrack_panas.csv\n"
     ]
    }
   ],
   "source": [
    "for f in os.listdir(os.path.join(derivs_dir,'0.indiv')):\n",
    "    print(f)\n",
    "    fpath=os.path.join(derivs_dir,'0.indiv',f)\n",
    "    frames[f]=pd.read_csv(fpath)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "demog_frame=pd.merge(frames['econdec_demog.csv'],frames['eyetrack_demog.csv'],'outer')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "panas_frame=pd.merge(frames['econdec_panas.csv'],frames['eyetrack_panas.csv'],'outer')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "gdms_frame=pd.merge(frames['econdec_gdms.csv'],frames['eyetrack_gdms.csv'],'outer')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "financ_frame=pd.merge(frames['econdec_financ.csv'],frames['eyetrack_financ.csv'],'outer')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "bisbas_frame=pd.merge(frames['econdec_bisbas.csv'],frames['eyetrack_bisbas.csv'],'outer')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "demog_frame.to_csv(os.path.join(derivs_dir,'1.merge','demog.csv'),index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "panas_frame.to_csv(os.path.join(derivs_dir,'1.merge','panas.csv'),index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "gdms_frame.to_csv(os.path.join(derivs_dir,'1.merge','gdms.csv'),index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "financ_frame.to_csv(os.path.join(derivs_dir,'1.merge','financ.csv'),index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "bisbas_frame.to_csv(os.path.join(derivs_dir,'1.merge','bisbas.csv'),index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}