CRIM_Metadata_Migration_2022.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Various Routines to Harvest CRIM Metadata from Production Server"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load the data from CRIM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import pandas as pd\n",
    "import json\n",
    "# old_rel_data = requests.get('http://crimproject.org/data/relationships-old/').json()\n",
    "# old_obs_data = requests.get('https://crimproject.org/data/observations-old/').json()\n",
    "rel_data = requests.get('http://crimproject.org/data/relationships/').json()\n",
    "people = requests.get('https://crimproject.org/data/people/').json()\n",
    "pieces = requests.get('https://crimproject.org/data/pieces/').json()\n",
    "voices = requests.get('https://crimproject.org/data/voices/').json()\n",
    "\n",
    "\n",
    "# rels = pd.json_normalize(rel_data)\n",
    "# rels.head(7)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'url': 'https://crimproject.org/data/relationships/1/',\n",
       " 'id': 1,\n",
       " 'observer': {'url': 'https://crimproject.org/data/people/CRIM_Person_1012/',\n",
       "  'name': 'Ian Lorenz'},\n",
       " 'model_observation': {'url': 'https://crimproject.org/data/observations/1/',\n",
       "  'id': 1,\n",
       "  'piece': {'url': 'https://crimproject.org/data/pieces/CRIM_Model_0011/',\n",
       "   'piece_id': 'CRIM_Model_0011',\n",
       "   'full_title': 'Tota pulchra es'},\n",
       "  'ema': '1-6/1,1,1-2,1-2,2,2/@1,@1-3,@1-3+@1,@1-3+@1-3,@1-3,@1'},\n",
       " 'derivative_observation': {'url': 'https://crimproject.org/data/observations/2/',\n",
       "  'id': 2,\n",
       "  'piece': {'url': 'https://crimproject.org/data/pieces/CRIM_Mass_0009_3/',\n",
       "   'piece_id': 'CRIM_Mass_0009_3',\n",
       "   'full_title': 'Missa Tota pulchra es: Credo'},\n",
       "  'ema': '1-6/1,1,1+3,1+3,3,3/@1-4,@1-3,@1-3+@1-4,@1-3+@1-3,@1-3,@1'},\n",
       " 'relationship_type': 'quotation',\n",
       " 'musical_type': 'fuga',\n",
       " 'definition': {'url': 'https://crimproject.org/data/definition/15/',\n",
       "  'id': 15,\n",
       "  'relationship_definition': [{'name': 'quotation',\n",
       "    'subtypes': [{'form': 'radio',\n",
       "      'name': 'type',\n",
       "      'options': ['exact', 'monnayage'],\n",
       "      'required': True},\n",
       "     {'form': 'radio',\n",
       "      'name': 'self',\n",
       "      'default': 'none',\n",
       "      'options': ['none', 'enchainment', 'repetition', 'return']}]},\n",
       "   {'name': 'mechanical transformation',\n",
       "    'subtypes': [{'form': 'boolean', 'name': 'sounding in different voices'},\n",
       "     {'form': 'boolean', 'name': 'melodically inverted'},\n",
       "     {'form': 'boolean', 'name': 'retrograde'},\n",
       "     {'form': 'boolean', 'name': 'metrically shifted'},\n",
       "     {'form': 'radio',\n",
       "      'name': 'transposition',\n",
       "      'options': ['not transposed',\n",
       "       'transposed',\n",
       "       'transposed different amounts'],\n",
       "      'required': True},\n",
       "     {'form': 'boolean', 'name': 'double or invertible counterpoint'},\n",
       "     {'form': 'boolean', 'name': 'systematic diminution'},\n",
       "     {'form': 'boolean', 'name': 'systematic augmentation'},\n",
       "     {'form': 'radio',\n",
       "      'name': 'self',\n",
       "      'default': 'none',\n",
       "      'options': ['none', 'enchainment', 'repetition', 'return']}]},\n",
       "   {'name': 'non-mechanical transformation',\n",
       "    'subtypes': [{'form': 'radio',\n",
       "      'name': 'extent',\n",
       "      'default': 'not changed',\n",
       "      'options': ['not changed', 'amplified', 'truncated']},\n",
       "     {'form': 'radio',\n",
       "      'name': 'activity',\n",
       "      'default': 'not changed',\n",
       "      'options': ['not changed', 'embellished', 'reduced']},\n",
       "     {'form': 'boolean', 'name': 'sounding in different voices'},\n",
       "     {'form': 'boolean', 'name': 'whole passage transposed'},\n",
       "     {'form': 'boolean', 'name': 'whole passage metrically shifted'},\n",
       "     {'form': 'boolean', 'name': 'melodically inverted'},\n",
       "     {'form': 'boolean', 'name': 'retrograde'},\n",
       "     {'form': 'boolean', 'name': 'new counter subject'},\n",
       "     {'form': 'boolean', 'name': 'old counter subject shifted metrically'},\n",
       "     {'form': 'boolean', 'name': 'old counter subject transposed'},\n",
       "     {'form': 'boolean', 'name': 'double or invertible counterpoint'},\n",
       "     {'form': 'boolean', 'name': 'new combination'},\n",
       "     {'form': 'radio',\n",
       "      'name': 'self',\n",
       "      'default': 'none',\n",
       "      'options': ['none', 'enchainment', 'repetition', 'return']}]},\n",
       "   {'form': 'boolean', 'name': 'omission'},\n",
       "   {'form': 'boolean', 'name': 'new material'}]},\n",
       " 'details': {'self': None, 'type': 'exact'},\n",
       " 'remarks': '',\n",
       " 'created': '2017-06-19T19:22:54-04:00',\n",
       " 'updated': '2022-02-03T18:01:55.836123-05:00',\n",
       " 'curated': True}"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rel_data[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Look at any of the data loaded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'url': 'https://crimproject.org/data/observations-old/2/',\n",
       " 'id': 2,\n",
       " 'observer': {'url': 'https://crimproject.org/data/people/CRIM_Person_1012/',\n",
       "  'name': 'Ian Lorenz'},\n",
       " 'piece': {'url': 'https://crimproject.org/data/pieces/CRIM_Mass_0009_3/',\n",
       "  'piece_id': 'CRIM_Mass_0009_3',\n",
       "  'full_title': 'Missa Tota pulchra es: Credo',\n",
       "  'mass': 'CRIM_Mass_0009'},\n",
       " 'ema': '1-6/1,1,1+3,1+3,3,3/@1-4,@1-3,@1-3+@1-4,@1-3+@1-3,@1-3,@1',\n",
       " 'musical_type': 'Fuga',\n",
       " 'mt_cf': False,\n",
       " 'mt_cf_voices': '',\n",
       " 'mt_cf_dur': False,\n",
       " 'mt_cf_mel': False,\n",
       " 'mt_sog': False,\n",
       " 'mt_sog_voices': '',\n",
       " 'mt_sog_dur': False,\n",
       " 'mt_sog_mel': False,\n",
       " 'mt_sog_ostinato': False,\n",
       " 'mt_sog_periodic': False,\n",
       " 'mt_csog': False,\n",
       " 'mt_csog_voices': '',\n",
       " 'mt_csog_dur': False,\n",
       " 'mt_csog_mel': False,\n",
       " 'mt_cd': False,\n",
       " 'mt_cd_voices': '',\n",
       " 'mt_fg': True,\n",
       " 'mt_fg_voices': '[Tenor 1]\\r\\n[Tenor 3]',\n",
       " 'mt_fg_int': '4-',\n",
       " 'mt_fg_tint': 'B2',\n",
       " 'mt_fg_periodic': False,\n",
       " 'mt_fg_strict': False,\n",
       " 'mt_fg_flexed': False,\n",
       " 'mt_fg_sequential': False,\n",
       " 'mt_fg_inverted': False,\n",
       " 'mt_fg_retrograde': False,\n",
       " 'mt_pe': False,\n",
       " 'mt_pe_voices': '',\n",
       " 'mt_pe_int': '',\n",
       " 'mt_pe_tint': '',\n",
       " 'mt_pe_strict': False,\n",
       " 'mt_pe_flexed': False,\n",
       " 'mt_pe_flt': False,\n",
       " 'mt_pe_sequential': False,\n",
       " 'mt_pe_added': False,\n",
       " 'mt_pe_invertible': False,\n",
       " 'mt_id': False,\n",
       " 'mt_id_voices': '',\n",
       " 'mt_id_int': '',\n",
       " 'mt_id_tint': '',\n",
       " 'mt_id_strict': False,\n",
       " 'mt_id_flexed': False,\n",
       " 'mt_id_flt': False,\n",
       " 'mt_id_invertible': False,\n",
       " 'mt_nid': False,\n",
       " 'mt_nid_voices': '',\n",
       " 'mt_nid_int': '',\n",
       " 'mt_nid_tint': '',\n",
       " 'mt_nid_strict': False,\n",
       " 'mt_nid_flexed': False,\n",
       " 'mt_nid_flt': False,\n",
       " 'mt_nid_sequential': False,\n",
       " 'mt_nid_invertible': False,\n",
       " 'mt_hr': False,\n",
       " 'mt_hr_voices': '',\n",
       " 'mt_hr_simple': False,\n",
       " 'mt_hr_staggered': False,\n",
       " 'mt_hr_sequential': False,\n",
       " 'mt_hr_fauxbourdon': False,\n",
       " 'mt_cad': False,\n",
       " 'mt_cad_cantizans': '',\n",
       " 'mt_cad_tenorizans': '',\n",
       " 'mt_cad_type': '',\n",
       " 'mt_cad_tone': '',\n",
       " 'mt_cad_dtv': '',\n",
       " 'mt_cad_dti': '',\n",
       " 'mt_int': False,\n",
       " 'mt_int_voices': '',\n",
       " 'mt_int_p6': False,\n",
       " 'mt_int_p3': False,\n",
       " 'mt_int_c35': False,\n",
       " 'mt_int_c83': False,\n",
       " 'mt_int_c65': False,\n",
       " 'mt_fp': False,\n",
       " 'mt_fp_ir': False,\n",
       " 'mt_fp_range': '',\n",
       " 'mt_fp_comment': '',\n",
       " 'remarks': 'Makes up longer ID',\n",
       " 'created': '2017-06-19T19:22:54-04:00',\n",
       " 'updated': '2020-07-15T11:37:01.876533-04:00',\n",
       " 'curated': True}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "obs_data[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Which Fields do we Need?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here is where to find the fields we need:\n",
    "\n",
    "**Observer (which will be CRIM_Person_0000):  **\n",
    "\n",
    "obs_pers_id = observation['observer']\n",
    "rel_pers_id = relationship['observer']\n",
    "\n",
    "**Observation_ID or Relationship_ID:**\n",
    "\n",
    "obs_id = observation['id']\n",
    "rel_id = relationship['id']\n",
    "\n",
    "**EMA**\n",
    "\n",
    "ema_data = observation['ema']\n",
    "\n",
    "**PIECE**\n",
    "\n",
    "obs_piece_id = observation['piece']\n",
    "\n",
    "**VOICES** \n",
    "\n",
    "crim_voices https://crimproject.org/data/voices/\n",
    "\n",
    "voice_name = voices['original_name']\n",
    "voice_obj = voices['order']\n",
    "piece_id_voices_file = voices[\n",
    "These are dependent on the musical type, so we need to have searches dependent on other searches!  And then we might be able to translate the strings, like  'Superius\\r\\nContratenor', into sets of integers.  \n",
    "\n",
    "Micah reports:  We want voices to be a JSON list: {'voices': [2, 4, 1, 2]}.\n",
    "\n",
    "**MUSICAL TYPE**\n",
    "\n",
    "This is a boolean, as well as a string in another field.  Since the \"Musical Type\" field can have more than one value, better to send the IP type values to comments?  Will that suppress the current values?  Or deal with these via Curate?\n",
    "\n",
    "if observation['mt_cf'] is True\n",
    "    new_json[{'musical_type': 'Cantus Firmus'}\n",
    "    new_json.update({\n",
    "\n",
    "**MODEL of SUBROUTINE**\n",
    "\n",
    "for observation in observation_data:\n",
    "    obs_pers_id = observation['observer']   \n",
    "    obs_id = observation['id']\n",
    "    obs_piece_id = observation['piece'] \n",
    "    obs_ema = observation['ema']\n",
    "    if observation['mt_cf'] is True:\n",
    "        cf_voice = observation['voice']\n",
    "        cf_dur = observation['mt_cad_type']\n",
    "        cf_mel = observation['mt_cad_type']\n",
    "        cf_data = {'details': {'durations only': cf_dur, 'pitches only': cf_mel}}\n",
    "        new_json.update(cf_data)\n",
    "        new_json.update({'observer': obs_pers_id, 'id': obs_id, 'piece': obs_piece_id, 'ema': obs_ema})\n",
    "    elif observation['mtsog'] is True:\n",
    "    \n",
    "    \n",
    "if example_json['mt_cad'] is True:\n",
    "    tone_val = example_json['mt_cad_tone']\n",
    "    type_val = example_json['mt_cad_type']\n",
    "    cad = {'cadence': {'tone': tone_val, 'type': type_val}}\n",
    "    example_json.update(cad)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load the Voices Data\n",
    "\n",
    "'CRIM_Voices_for_Migration' is a table that assembles voice names and object numbers, with forms of voices found in CRIM Observation data, along with Regularized names."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>work_title</th>\n",
       "      <th>piece.piece_id</th>\n",
       "      <th>piece__title</th>\n",
       "      <th>voice_id</th>\n",
       "      <th>order</th>\n",
       "      <th>original_name</th>\n",
       "      <th>regularized_name</th>\n",
       "      <th>CRIM_Obs_Name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Missa Confitemini</td>\n",
       "      <td>CRIM_Mass_0001_1</td>\n",
       "      <td>Kyrie</td>\n",
       "      <td>CRIM_Mass_0001_1(1)</td>\n",
       "      <td>1</td>\n",
       "      <td>[Superius]</td>\n",
       "      <td>Superius</td>\n",
       "      <td>Superius</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Missa Confitemini</td>\n",
       "      <td>CRIM_Mass_0001_1</td>\n",
       "      <td>Kyrie</td>\n",
       "      <td>CRIM_Mass_0001_1(2)</td>\n",
       "      <td>2</td>\n",
       "      <td>[Contratenor]</td>\n",
       "      <td>Contratenor</td>\n",
       "      <td>Contratenor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Missa Confitemini</td>\n",
       "      <td>CRIM_Mass_0001_1</td>\n",
       "      <td>Kyrie</td>\n",
       "      <td>CRIM_Mass_0001_1(3)</td>\n",
       "      <td>3</td>\n",
       "      <td>[Tenor]</td>\n",
       "      <td>Tenor</td>\n",
       "      <td>Tenor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Missa Confitemini</td>\n",
       "      <td>CRIM_Mass_0001_1</td>\n",
       "      <td>Kyrie</td>\n",
       "      <td>CRIM_Mass_0001_1(4)</td>\n",
       "      <td>4</td>\n",
       "      <td>[Bassus]</td>\n",
       "      <td>Bassus</td>\n",
       "      <td>Bassus</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Missa Confitemini</td>\n",
       "      <td>CRIM_Mass_0001_2</td>\n",
       "      <td>Gloria</td>\n",
       "      <td>CRIM_Mass_0001_2(1)</td>\n",
       "      <td>1</td>\n",
       "      <td>[Superius]</td>\n",
       "      <td>Superius</td>\n",
       "      <td>Superius</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Missa Confitemini</td>\n",
       "      <td>CRIM_Mass_0001_2</td>\n",
       "      <td>Gloria</td>\n",
       "      <td>CRIM_Mass_0001_2(2)</td>\n",
       "      <td>2</td>\n",
       "      <td>[Contratenor]</td>\n",
       "      <td>Contratenor</td>\n",
       "      <td>Contratenor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Missa Confitemini</td>\n",
       "      <td>CRIM_Mass_0001_2</td>\n",
       "      <td>Gloria</td>\n",
       "      <td>CRIM_Mass_0001_2(3)</td>\n",
       "      <td>3</td>\n",
       "      <td>[Tenor]</td>\n",
       "      <td>Tenor</td>\n",
       "      <td>Tenor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Missa Confitemini</td>\n",
       "      <td>CRIM_Mass_0001_2</td>\n",
       "      <td>Gloria</td>\n",
       "      <td>CRIM_Mass_0001_2(4)</td>\n",
       "      <td>4</td>\n",
       "      <td>[Bassus]</td>\n",
       "      <td>Bassus</td>\n",
       "      <td>Bassus</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Missa Confitemini</td>\n",
       "      <td>CRIM_Mass_0001_3</td>\n",
       "      <td>Credo</td>\n",
       "      <td>CRIM_Mass_0001_3(1)</td>\n",
       "      <td>1</td>\n",
       "      <td>[Superius]</td>\n",
       "      <td>Superius</td>\n",
       "      <td>Superius</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Missa Confitemini</td>\n",
       "      <td>CRIM_Mass_0001_3</td>\n",
       "      <td>Credo</td>\n",
       "      <td>CRIM_Mass_0001_3(2)</td>\n",
       "      <td>2</td>\n",
       "      <td>[Contratenor]</td>\n",
       "      <td>Contratenor</td>\n",
       "      <td>Contratenor</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          work_title    piece.piece_id piece__title             voice_id  \\\n",
       "0  Missa Confitemini  CRIM_Mass_0001_1        Kyrie  CRIM_Mass_0001_1(1)   \n",
       "1  Missa Confitemini  CRIM_Mass_0001_1        Kyrie  CRIM_Mass_0001_1(2)   \n",
       "2  Missa Confitemini  CRIM_Mass_0001_1        Kyrie  CRIM_Mass_0001_1(3)   \n",
       "3  Missa Confitemini  CRIM_Mass_0001_1        Kyrie  CRIM_Mass_0001_1(4)   \n",
       "4  Missa Confitemini  CRIM_Mass_0001_2       Gloria  CRIM_Mass_0001_2(1)   \n",
       "5  Missa Confitemini  CRIM_Mass_0001_2       Gloria  CRIM_Mass_0001_2(2)   \n",
       "6  Missa Confitemini  CRIM_Mass_0001_2       Gloria  CRIM_Mass_0001_2(3)   \n",
       "7  Missa Confitemini  CRIM_Mass_0001_2       Gloria  CRIM_Mass_0001_2(4)   \n",
       "8  Missa Confitemini  CRIM_Mass_0001_3        Credo  CRIM_Mass_0001_3(1)   \n",
       "9  Missa Confitemini  CRIM_Mass_0001_3        Credo  CRIM_Mass_0001_3(2)   \n",
       "\n",
       "   order  original_name regularized_name CRIM_Obs_Name  \n",
       "0      1     [Superius]         Superius      Superius  \n",
       "1      2  [Contratenor]      Contratenor   Contratenor  \n",
       "2      3        [Tenor]            Tenor         Tenor  \n",
       "3      4       [Bassus]           Bassus        Bassus  \n",
       "4      1     [Superius]         Superius      Superius  \n",
       "5      2  [Contratenor]      Contratenor   Contratenor  \n",
       "6      3        [Tenor]            Tenor         Tenor  \n",
       "7      4       [Bassus]           Bassus        Bassus  \n",
       "8      1     [Superius]         Superius      Superius  \n",
       "9      2  [Contratenor]      Contratenor   Contratenor  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "v_data = pd.read_csv('CRIM_Voices_for_Migration.csv')\n",
    "v_data.head(10) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load the Observation Migration \n",
    "\n",
    "* all functions for migration\n",
    "* gets details for each musical type\n",
    "*  gets person id for each person\n",
    "* migrates data to new format\n",
    "* updates voice number and regularized names for voices\n",
    "* functions for types with voice strings fixed and split and trailing/leading whitespace removed\n",
    "*  OK 1/12/22"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "\n",
    "def get_cf(observation):\n",
    "\n",
    "    features = None\n",
    "    cf_voice = observation['mt_cf_voices'].rstrip().strip()\n",
    "    if observation[\"mt_cf_dur\"] and observation[\"mt_cf_mel\"]:\n",
    "        features = \"both pitches and durations\"\n",
    "    elif observation[\"mt_cf_dur\"]:\n",
    "        features = \"durations only\"\n",
    "    elif observation[\"mt_cf_mel\"]:\n",
    "        features = \"pitches only\"\n",
    "    remarks = observation['remarks']\n",
    "    cf_data = {'details': {'voice name': cf_voice, \n",
    "                           'features': features}}\n",
    "    return cf_data\n",
    "\n",
    "def get_sg(observation):\n",
    "    \n",
    "    features = None\n",
    "    sg_voice = observation['mt_sog_voices'].rstrip().strip()\n",
    "    if observation[\"mt_sog_dur\"] and observation[\"mt_sog_mel\"]:\n",
    "        features = \"both pitches and durations\"\n",
    "    elif observation[\"mt_sog_dur\"]:\n",
    "        features = \"durations only\"\n",
    "    elif observation[\"mt_sog_mel\"]:\n",
    "        features = \"pitches only\"\n",
    "    sg_ostinato = observation['mt_sog_ostinato']\n",
    "    sg_periodic = observation['mt_sog_periodic']\n",
    "    remarks = observation['remarks']\n",
    "    sg_data = {'details': {'voice name': sg_voice, \n",
    "                           'features': features, \n",
    "                           'ostinato': sg_ostinato, \n",
    "                           'periodic': sg_periodic}}\n",
    "    return sg_data\n",
    "\n",
    "\n",
    "def get_csg(observation):\n",
    "    features = None\n",
    "    csg_voice = observation['mt_csog_voices'].rstrip().strip()\n",
    "    if observation[\"mt_csog_dur\"] and observation[\"mt_csog_mel\"]:\n",
    "        features = \"both pitches and durations\"\n",
    "    elif observation[\"mt_csog_dur\"]:\n",
    "        features = \"durations only\"\n",
    "    elif observation[\"mt_csog_mel\"]:\n",
    "        features = \"pitches only\"\n",
    "    features = \"pitches only\"\n",
    "    remarks = observation['remarks']\n",
    "    csg_data = {'details': {'voice name': csg_voice, \n",
    "                           'features': features,}}\n",
    "    return csg_data\n",
    "    \n",
    "def get_cd(observation):\n",
    "    cd_voices = observation['mt_cd_voices'].replace('\\r\\n', '\\n').split('\\n')\n",
    "    cd_voices = [v.strip() for v in cd_voices]\n",
    "    remarks = observation['remarks']\n",
    "    cd_data = {'details': {'voice names': cd_voices}}\n",
    "    return cd_data\n",
    "    \n",
    "def get_fg(observation):\n",
    "    \n",
    "    regularity = None\n",
    "    fg_voices = observation['mt_fg_voices'].replace('\\r\\n', '\\n').split('\\n')\n",
    "    fg_voices = [v.strip() for v in fg_voices]\n",
    "    fg_int = observation['mt_fg_int']\n",
    "    fg_tint = observation['mt_fg_tint']\n",
    "    fg_periodic = observation['mt_fg_periodic']\n",
    "    fg_strict = observation['mt_fg_strict']\n",
    "    fg_flexed = observation['mt_fg_flexed']\n",
    "    if observation[\"mt_fg_strict\"]:\n",
    "        regularity = \"strict\"\n",
    "    elif observation[\"mt_fg_flexed\"]:\n",
    "        regularity = \"flexed\"\n",
    "    fg_sequential = observation['mt_fg_sequential']\n",
    "    fg_inverted = observation['mt_fg_inverted']\n",
    "    fg_retrograde = observation['mt_fg_retrograde']\n",
    "    remarks = observation['remarks']\n",
    "    fg_data = {'details': {'voice names': fg_voices, \n",
    "                           'entry intervals': fg_int, \n",
    "                           'time intervals': fg_tint, \n",
    "                           'periodic': fg_periodic, \n",
    "                           'regularity': regularity, \n",
    "                           'sequential': fg_sequential, \n",
    "                           'inverted': fg_inverted, \n",
    "                           'retrograde': fg_retrograde}}\n",
    "        \n",
    "    return fg_data\n",
    "\n",
    "\n",
    "def get_pe(observation):\n",
    "    regularity = None\n",
    "    pe_voices = observation['mt_pe_voices'].replace('\\r\\n', '\\n').split('\\n')\n",
    "    pe_voices = [v.strip() for v in pe_voices]\n",
    "    pe_int = observation['mt_pe_int']\n",
    "    pe_tint = observation['mt_pe_tint']\n",
    "    if observation[\"mt_pe_strict\"]:\n",
    "        regularity = \"strict\"\n",
    "    elif observation[\"mt_pe_flexed\"]:\n",
    "        regularity = \"flexed\"\n",
    "    elif observation[\"mt_pe_flt\"]:\n",
    "        regularity = \"flexed tonal\"\n",
    "    pe_sequential = observation['mt_pe_sequential']\n",
    "    pe_added = observation['mt_pe_added']\n",
    "    pe_invt = observation['mt_pe_invertible']\n",
    "    remarks = observation['remarks']\n",
    "    pe_data = {'details': {'voice names': pe_voices, \n",
    "                           'entry intervals': pe_int, \n",
    "                           'time intervals': pe_tint, \n",
    "                           'regularity': regularity,\n",
    "                           'sequential': pe_sequential, \n",
    "                           'invertible counterpoint': pe_invt, \n",
    "                           'added entries': pe_added}}\n",
    "        \n",
    "    return pe_data\n",
    "\n",
    "\n",
    "def get_id(observation):\n",
    "    regularity = None\n",
    "    id_voices = observation['mt_id_voices'].replace('\\r\\n', '\\n').split('\\n')\n",
    "    id_voices = [v.strip() for v in id_voices]\n",
    "    id_int = observation['mt_id_int']\n",
    "    id_tint = observation['mt_id_tint']\n",
    "    if observation[\"mt_id_strict\"]:\n",
    "        regularity = \"strict\"\n",
    "    elif observation[\"mt_id_flexed\"]:\n",
    "        regularity = \"flexed\"\n",
    "    elif observation[\"mt_id_flt\"]:\n",
    "        regularity = \"flexed tonal\"\n",
    "    id_invt = observation['mt_id_invertible']\n",
    "    remarks = observation['remarks']\n",
    "    id_data = {'details': {'voice names': id_voices, \n",
    "                           'entry intervals': id_int, \n",
    "                           'time intervals': id_tint, \n",
    "                           'regularity': regularity,\n",
    "                           'invertible counterpoint': id_invt,\n",
    "                           'added entries': None}}\n",
    "    return id_data\n",
    "\n",
    "def get_nid(observation):\n",
    "    regularity = None\n",
    "    nid_voices = observation['mt_nid_voices'].replace('\\r\\n', '\\n').split('\\n')\n",
    "    nid_voices = [v.strip() for v in nid_voices]\n",
    "    nid_int = observation['mt_nid_int']\n",
    "    nid_tint = observation['mt_nid_tint']\n",
    "\n",
    "    if observation[\"mt_nid_strict\"]:\n",
    "        regularity = \"strict\"\n",
    "    elif observation[\"mt_nid_flexed\"]:\n",
    "        regularity = \"flexed\"\n",
    "    elif observation[\"mt_nid_flt\"]:\n",
    "        regularity = \"flexed tonal\"\n",
    "    nid_sequential = observation['mt_nid_sequential']\n",
    "    nid_invt = observation['mt_nid_invertible']\n",
    "    remarks = observation['remarks']\n",
    "    nid_data = {'details': {'voice names': nid_voices, \n",
    "                           'entry intervals': nid_int, \n",
    "                           'time intervals': nid_tint, \n",
    "                           'regularity': regularity,\n",
    "                           'sequential': nid_sequential, \n",
    "                           'invertible counterpoint': nid_invt,\n",
    "                           'added entries': None}}\n",
    "    return nid_data   \n",
    "\n",
    "def get_hr(observation):\n",
    "    type = None\n",
    "    hr_voices = observation['mt_hr_voices'].replace('\\r\\n', '\\n').split('\\n')\n",
    "    hr_voices = [v.strip() for v in hr_voices]\n",
    "    if observation[\"mt_hr_simple\"]:\n",
    "        type = \"simple\"\n",
    "    elif observation[\"mt_hr_staggered\"]:\n",
    "        type = \"staggered\"\n",
    "    elif observation[\"mt_hr_sequential\"]:\n",
    "        type = \"sequential\"\n",
    "    elif observation[\"mt_hr_fauxbourdon\"]:\n",
    "        type = \"fauxbourdon\"\n",
    "    remarks = observation['remarks']\n",
    "    hr_data = {'details': {'voice names': hr_voices, \n",
    "                           'type': type,\n",
    "                           'dialogue': None}}\n",
    "    return hr_data    \n",
    "\n",
    "\n",
    "        \n",
    "def get_cad(observation):\n",
    "    cad_dove_bool = None\n",
    "    cad_tone = observation['mt_cad_tone']\n",
    "    cad_type = observation['mt_cad_type']\n",
    "    cad_can = observation['mt_cad_cantizans'].rstrip().strip()\n",
    "    cad_ten = observation['mt_cad_tenorizans'].rstrip().strip()\n",
    "    cad_dove_v = observation['mt_cad_dtv'].rstrip().strip()\n",
    "    if not cad_dove_v:\n",
    "        cad_dove_bool = False\n",
    "    else: \n",
    "        cad_dove_bool = True  \n",
    "    # cad_dove_i = observation['mt_cad_dti']\n",
    "    remarks = observation['remarks']\n",
    "    cad_data = {'details': {'tone': cad_tone, \n",
    "                            'type': cad_type, \n",
    "                            'cantizans name': cad_can, \n",
    "                            'tenorizans name': cad_ten,\n",
    "                            'dovetail cadence':  cad_dove_bool,\n",
    "                            'dovetail voice name': cad_dove_v,\n",
    "                            'dovetail position': None,\n",
    "                            'irregular cadence': None,\n",
    "                            'irregular roles': None}}\n",
    "    return cad_data   \n",
    "\n",
    "def look_up_voice_order(v_data, piece_id, obs_voice):\n",
    "    mask = (v_data[\"piece.piece_id\"] == piece_id) & (v_data[\"CRIM_Obs_Name\"] == obs_voice)\n",
    "    v_order = v_data[mask].iloc[0].order\n",
    "    \n",
    "    return int(v_order)\n",
    "    \n",
    "def look_up_voice_reg_name(v_data, piece_id, obs_voice):\n",
    "    mask = (v_data[\"piece.piece_id\"] == piece_id) & (v_data[\"CRIM_Obs_Name\"] == obs_voice)\n",
    "    v_reg_name = v_data[mask].iloc[0].regularized_name\n",
    "\n",
    "    return (v_reg_name)\n",
    "\n",
    "def get_person_id(people: list, name: str):\n",
    "    obs_pers_id = 'unknown'\n",
    "    for record in people:\n",
    "            if record['name'] == name:\n",
    "                obs_pers_id = (record['person_id'])          \n",
    "    return obs_pers_id\n",
    "\n",
    "def migrate_obs_data(observation: dict, people: list, v_data):\n",
    "    \n",
    "    # adding fields = {}\n",
    "    fields = {}\n",
    "    migrated_json = {}\n",
    "    m_type = \"\"\n",
    "    details = {}\n",
    "    \n",
    "\n",
    "    # here we get the 'name' as recorded in the CRIM Observation, then look up their CRIM ID\n",
    "    \n",
    "    obs_pers_name = observation['observer']['name']\n",
    "    obs_pers_id = get_person_id(people, obs_pers_name)\n",
    "    # checks if the observer is \"unknown\".  If so:  replace with CRIM_Person_0000\n",
    "    if obs_pers_id == \"unknown\":\n",
    "        obs_pers_id = \"CRIM_Person_0000\"\n",
    "    # and now the rest of the id for the piece and obs, plus ema\n",
    "    obs_id = observation['id']\n",
    "    obs_piece_id = observation['piece']['piece_id'] \n",
    "    obs_ema = observation['ema']\n",
    "    remarks = observation['remarks']\n",
    "    curated = observation['curated']\n",
    "    created = observation['created']\n",
    "    updated = observation['updated']\n",
    "    # and how we test for each musical type and gather subfields\n",
    "    \n",
    "    # cantus firmus = data#9\n",
    "    if observation['mt_cf'] is True:\n",
    "        m_type = \"cantus firmus\"\n",
    "        details = get_cf(observation)\n",
    "        \n",
    "    # soggetto data#94\n",
    "    if observation['mt_sog'] is True:\n",
    "        m_type = \"soggetto\"\n",
    "        details = get_sg(observation)\n",
    "        \n",
    "    # counter soggetto data#53\n",
    "    if observation['mt_csog'] is True:\n",
    "        m_type = \"counter soggetto\"\n",
    "        details = get_csg(observation)\n",
    "       \n",
    "    # contrapuntal duo data#88\n",
    "    if observation['mt_cd'] is True:\n",
    "        m_type = \"contrapuntal duo\"\n",
    "        details = get_cd(observation)\n",
    "        \n",
    "    # fuga data #64\n",
    "    if observation['mt_fg'] is True:\n",
    "        m_type = \"fuga\"\n",
    "        details = get_fg(observation)\n",
    "        \n",
    "    # periodic entry data #93\n",
    "    if observation['mt_pe'] is True:\n",
    "        m_type = \"periodic entry\"\n",
    "        details = get_pe(observation)\n",
    "           \n",
    "    # imitative duo data #98\n",
    "    if observation['mt_id'] is True:\n",
    "        m_type = \"imitative duo\"\n",
    "        details = get_id(observation)\n",
    "               \n",
    "    # non-imitative duo data #89\n",
    "    if observation['mt_nid'] is True:\n",
    "        m_type = \"non-imitative duo\"\n",
    "        details = get_nid(observation)\n",
    "      \n",
    "    # homorhythm data #92\n",
    "    if observation['mt_hr'] is True:\n",
    "        m_type = \"homorhythm\"\n",
    "        details = get_hr(observation)\n",
    "         \n",
    "    # cadence data #159\n",
    "    # note that the details of the OLD cadence data don't fit the new type\n",
    "    if observation['mt_cad'] is True:\n",
    "        m_type = \"cadence\"\n",
    "        details = get_cad(observation)\n",
    "                   \n",
    "    # nested version\n",
    "    '''\n",
    "    migrated_json.update({'pk': obs_id, \n",
    "                        'model': 'crim.cjobservation',\n",
    "                        'fields': {'observer': obs_pers_id,\n",
    "                        'piece': obs_piece_id, \n",
    "                        'musical_type': m_type,\n",
    "                        'ema': obs_ema,\n",
    "                        'details': details,\n",
    "                        'remarks': remarks,\n",
    "                        'curated': curated,\n",
    "                        'created': created,\n",
    "                        'updated': updated}\n",
    "                        })\n",
    "    '''\n",
    "    \n",
    "    # Corrected 1/22/22 as per MW, with 'pk' 'model' and 'fields'+'details'\n",
    "\n",
    "    fields.update({'definition': 15,\n",
    "                   'observer': obs_pers_id, \n",
    "                        'piece': obs_piece_id, \n",
    "                        'musical_type': m_type,\n",
    "                        'ema': obs_ema,\n",
    "                        'remarks': remarks,\n",
    "                        'curated': curated,\n",
    "                        'created': created,\n",
    "                        'updated': updated})\n",
    "    fields.update(details)\n",
    "    migrated_json.update({'pk': obs_id, \n",
    "                            'model': 'crim.cjobservation',\n",
    "                            'fields': fields})\n",
    "                        \n",
    "                        \n",
    "    \n",
    "    #OLD VERSION HERE \n",
    "    #migrated_json.update({'id': obs_id, \n",
    "                       # 'observer': obs_pers_id, \n",
    "                        #'piece': obs_piece_id, \n",
    "                       # 'musical_type': m_type,\n",
    "                        #'ema': obs_ema,\n",
    "                        #'remarks': remarks,\n",
    "                        #'curated': curated,\n",
    "                        #'created': created,\n",
    "                        #'updated': updated})\n",
    "\n",
    "    \n",
    "    # migrated_json.update(details) \n",
    "    \n",
    "# now run voice order and name updates on the first stage of migrated data, then update\n",
    "\n",
    "# 1/11/21:  removed underscores from voice_name_reg, can_name_reg, ten_name_reg, voice_order in all types\n",
    "\n",
    "    v_order_list = []\n",
    "    v_reg_name_list = []\n",
    "    \n",
    "    type_1_list = [\"fuga\", \"contrapuntal duo\", \"periodic entry\", \"imitative duo\", \"non-imitative duo\", \"homorhythm\"]\n",
    "    type_2_list = [\"cantus firmus\", \"soggetto\", \"counter soggetto\"]\n",
    "    # replaced id with 'pk'\n",
    "    obs_id = migrated_json['pk']\n",
    "    # now add ['fields'] to these calls\n",
    "    piece_id = migrated_json['fields']['piece']\n",
    "    if migrated_json['fields'][\"musical_type\"] in type_1_list:\n",
    "        obs_voices = migrated_json['fields']['details']['voice names']\n",
    "        for obs_voice in obs_voices:\n",
    "            v_reg_name_list.append(look_up_voice_reg_name(v_data, piece_id, obs_voice))\n",
    "            v_order_list.append(look_up_voice_order(v_data, piece_id, obs_voice))\n",
    "            migrated_json['fields']['details']['voice names reg']=v_reg_name_list\n",
    "            migrated_json['fields']['details']['voices']=v_order_list\n",
    "\n",
    "    if migrated_json['fields'][\"musical_type\"] in type_2_list:\n",
    "        obs_voice = migrated_json['fields']['details']['voice name']\n",
    "        migrated_json['fields']['details']['voice name reg']=look_up_voice_reg_name(v_data, piece_id, obs_voice)\n",
    "        migrated_json['fields']['details']['voice']=look_up_voice_order(v_data, piece_id, obs_voice)\n",
    "\n",
    "# this is Micah's NEW code for cadences\n",
    "\n",
    "    if migrated_json['fields'][\"musical_type\"] == \"cadence\":\n",
    "        obs_can = migrated_json['fields']['details']['cantizans name']\n",
    "        if not obs_can:\n",
    "            migrated_json['fields']['details']['cantizans name reg'] = None\n",
    "            migrated_json['fields']['details']['cantizans'] = None\n",
    "        else:\n",
    "            migrated_json['fields']['details']['cantizans name reg'] = look_up_voice_reg_name(v_data, piece_id, obs_can)\n",
    "            migrated_json['fields']['details']['cantizans'] = look_up_voice_order(v_data, piece_id, obs_can)\n",
    "\n",
    "        obs_ten = migrated_json['fields']['details']['tenorizans name']\n",
    "        if not obs_ten:\n",
    "            migrated_json['fields']['details']['tenorizans name reg'] = None\n",
    "            migrated_json['fields']['details']['tenorizans'] = None\n",
    "        else:\n",
    "            migrated_json['fields']['details']['tenorizans name reg'] = look_up_voice_reg_name(v_data, piece_id, obs_ten)\n",
    "            migrated_json['fields']['details']['tenorizans'] = look_up_voice_order(v_data, piece_id, obs_ten)\n",
    " \n",
    "        obs_dovetail_voice = migrated_json['fields']['details']['dovetail voice name']\n",
    "        if not obs_dovetail_voice:\n",
    "            migrated_json['fields']['details']['dovetail voice name reg'] = None\n",
    "            migrated_json['fields']['details']['dovetail cadence voice'] = None\n",
    "        else:\n",
    "            migrated_json['fields']['details']['dovetail voice name reg'] = look_up_voice_reg_name(v_data, piece_id, obs_dovetail_voice)\n",
    "            migrated_json['fields']['details']['dovetail cadence voice'] = look_up_voice_order(v_data, piece_id, obs_dovetail_voice)\n",
    "\n",
    "\n",
    "    return migrated_json\n",
    "\n",
    "\n",
    "   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'pk': 5199,\n",
       " 'model': 'crim.cjobservation',\n",
       " 'fields': {'definition': 15,\n",
       "  'observer': 'CRIM_Person_0000',\n",
       "  'piece': 'CRIM_Model_0009',\n",
       "  'musical_type': 'soggetto',\n",
       "  'ema': '3-5/1,1,1/@1-4,@1-3,@1',\n",
       "  'remarks': '',\n",
       "  'curated': True,\n",
       "  'created': '2019-07-18T11:45:11.189000-04:00',\n",
       "  'updated': '2019-07-18T11:45:11.189000-04:00',\n",
       "  'details': {'voice name': 'Superius',\n",
       "   'features': 'both pitches and durations',\n",
       "   'ostinato': False,\n",
       "   'periodic': False,\n",
       "   'voice name reg': 'Superius',\n",
       "   'voice': 1}}}"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#  this tests on just ONE OBservation\n",
    "sampleobs = obs_data[5192]\n",
    "migrate_obs_data(sampleobs, people, v_data)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## The following are just to Check Data and Test Lists"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'url': 'https://crimproject.org/data/observations-old/1370/',\n",
       " 'id': 1370,\n",
       " 'observer': {'url': 'https://crimproject.org/data/people/CRIM_Person_1015/',\n",
       "  'name': 'Anne Nautré'},\n",
       " 'piece': {'url': 'https://crimproject.org/data/pieces/CRIM_Mass_0020_4/',\n",
       "  'piece_id': 'CRIM_Mass_0020_4',\n",
       "  'full_title': 'Missa Susanne un jour: Sanctus',\n",
       "  'mass': 'CRIM_Mass_0020'},\n",
       " 'ema': '29-33,35-37,39-40/2,2-3,3+5,1+5,1,4,4-5,5,5,5/@2-4,@1+@2-4,@1+@2-4,@2-4+@1,@1,@2-4,@1+@2-4,@1,@1-4,@1',\n",
       " 'musical_type': 'Fuga',\n",
       " 'mt_cf': False,\n",
       " 'mt_cf_voices': '',\n",
       " 'mt_cf_dur': False,\n",
       " 'mt_cf_mel': False,\n",
       " 'mt_sog': False,\n",
       " 'mt_sog_voices': '',\n",
       " 'mt_sog_dur': False,\n",
       " 'mt_sog_mel': False,\n",
       " 'mt_sog_ostinato': False,\n",
       " 'mt_sog_periodic': False,\n",
       " 'mt_csog': False,\n",
       " 'mt_csog_voices': '',\n",
       " 'mt_csog_dur': False,\n",
       " 'mt_csog_mel': False,\n",
       " 'mt_cd': False,\n",
       " 'mt_cd_voices': '',\n",
       " 'mt_fg': True,\n",
       " 'mt_fg_voices': 'Contratenor\\nTenor\\nBassus\\n[Superius]\\nTenorSecundus\\nBassus\\nBassus',\n",
       " 'mt_fg_int': '1+8-5+8-5-2-',\n",
       " 'mt_fg_tint': 'M4/4/4/12/4/11',\n",
       " 'mt_fg_periodic': True,\n",
       " 'mt_fg_strict': False,\n",
       " 'mt_fg_flexed': False,\n",
       " 'mt_fg_sequential': False,\n",
       " 'mt_fg_inverted': False,\n",
       " 'mt_fg_retrograde': False,\n",
       " 'mt_pe': False,\n",
       " 'mt_pe_voices': '',\n",
       " 'mt_pe_int': '',\n",
       " 'mt_pe_tint': '',\n",
       " 'mt_pe_strict': False,\n",
       " 'mt_pe_flexed': False,\n",
       " 'mt_pe_flt': False,\n",
       " 'mt_pe_sequential': False,\n",
       " 'mt_pe_added': False,\n",
       " 'mt_pe_invertible': False,\n",
       " 'mt_id': False,\n",
       " 'mt_id_voices': '',\n",
       " 'mt_id_int': '',\n",
       " 'mt_id_tint': '',\n",
       " 'mt_id_strict': False,\n",
       " 'mt_id_flexed': False,\n",
       " 'mt_id_flt': False,\n",
       " 'mt_id_invertible': False,\n",
       " 'mt_nid': False,\n",
       " 'mt_nid_voices': '',\n",
       " 'mt_nid_int': '',\n",
       " 'mt_nid_tint': '',\n",
       " 'mt_nid_strict': False,\n",
       " 'mt_nid_flexed': False,\n",
       " 'mt_nid_flt': False,\n",
       " 'mt_nid_sequential': False,\n",
       " 'mt_nid_invertible': False,\n",
       " 'mt_hr': False,\n",
       " 'mt_hr_voices': '',\n",
       " 'mt_hr_simple': False,\n",
       " 'mt_hr_staggered': False,\n",
       " 'mt_hr_sequential': False,\n",
       " 'mt_hr_fauxbourdon': False,\n",
       " 'mt_cad': False,\n",
       " 'mt_cad_cantizans': '',\n",
       " 'mt_cad_tenorizans': '',\n",
       " 'mt_cad_type': '',\n",
       " 'mt_cad_tone': '',\n",
       " 'mt_cad_dtv': '',\n",
       " 'mt_cad_dti': '',\n",
       " 'mt_int': False,\n",
       " 'mt_int_voices': '',\n",
       " 'mt_int_p6': False,\n",
       " 'mt_int_p3': False,\n",
       " 'mt_int_c35': False,\n",
       " 'mt_int_c83': False,\n",
       " 'mt_int_c65': False,\n",
       " 'mt_fp': False,\n",
       " 'mt_fp_ir': False,\n",
       " 'mt_fp_range': '',\n",
       " 'mt_fp_comment': '',\n",
       " 'remarks': \"Les 4 premières entrées sont périodiques\\nAprès deux mesures sans entrées, 5 et 6 entrent avec le même intervalle de temps que les 4 premières\\nLa dernière entrée à B n'est pas stricte\",\n",
       " 'created': '2017-07-22T08:43:57-04:00',\n",
       " 'updated': '2019-06-26T17:55:37.501000-04:00',\n",
       " 'curated': True}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "obs_data[1369]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Now Migrate the Observations--All of Them!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This runs the curation routine for all obs 11/2/21\n",
    "\n",
    "clean_obs = [migrate_obs_data(o, people, v_data) for o in obs_data]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'pk': 10, 'model': 'crim.cjobservation', 'fields': {'definition': 15, 'observer': 'CRIM_Person_1015', 'piece': 'CRIM_Mass_0017_4', 'musical_type': 'cantus firmus', 'ema': '1-41/5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5/@1,@1,@1,@1-3,@1,@1,@all,@1-4,@1-3,@1-4.5,@1,@1-4,@1-3,@1-4.5,@1-3,@all,@all,@1-3,@1,@1,@1-3,@1,@1,@all,@1-4,@1-3,@1-4.5,@1,@1-4,@1-3,@1-4.5,@1-3,@all,@all,@1,@1,@1,@1-3,@1,@1,@1', 'remarks': 'La voix de ténor primus cite intégralement le bassus du modèle', 'curated': True, 'created': '2017-06-20T13:13:54-04:00', 'updated': '2019-06-26T17:55:17.039000-04:00', 'details': {'voice name': 'Tenor[Primus]', 'features': 'both pitches and durations', 'voice name reg': 'Tenor 1 [resolutio]', 'voice': 3}}}\n",
      "{'pk': 95, 'model': 'crim.cjobservation', 'fields': {'definition': 15, 'observer': 'CRIM_Person_1016', 'piece': 'CRIM_Model_0017', 'musical_type': 'soggetto', 'ema': '99-107/1,1,1,1,1,1,1,1,1/@1,@1,@1,@1,@1,@1,@1,@1,@1', 'remarks': '', 'curated': True, 'created': '2017-07-01T19:09:24-04:00', 'updated': '2019-06-26T17:55:17.269000-04:00', 'details': {'voice name': '[Discantus]', 'features': 'both pitches and durations', 'ostinato': False, 'periodic': False, 'voice name reg': 'Discantus', 'voice': 1}}}\n",
      "{'pk': 54, 'model': 'crim.cjobservation', 'fields': {'definition': 15, 'observer': 'CRIM_Person_1016', 'piece': 'CRIM_Mass_0015_2', 'musical_type': 'counter soggetto', 'ema': '130-134/6,6,6,6,6/@1,@1,@1,@1,@1', 'remarks': '', 'curated': True, 'created': '2017-06-30T19:25:41-04:00', 'updated': '2021-07-13T18:47:10.916691-04:00', 'details': {'voice name': 'Bassus [2]', 'features': 'pitches only', 'voice name reg': 'Bassus 2', 'voice': 6}}}\n",
      "{'pk': 89, 'model': 'crim.cjobservation', 'fields': {'definition': 15, 'observer': 'CRIM_Person_1016', 'piece': 'CRIM_Model_0017', 'musical_type': 'contrapuntal duo', 'ema': '68-70/1+6,1+6,1+6/@2-4+@2-4,@1-4+@1-3,@1+@1', 'remarks': 'Sogetto in the bass is repeated, creating a module with CS in the contratenor, but the contrapuntal duo DB is not repeated.', 'curated': True, 'created': '2017-08-03T19:36:56-04:00', 'updated': '2019-06-26T17:55:17.254000-04:00', 'details': {'voice names': ['[Discantus]', '[Bassus]'], 'voice names reg': ['Discantus', 'Bassus'], 'voices': [1, 6]}}}\n",
      "{'pk': 65, 'model': 'crim.cjobservation', 'fields': {'definition': 15, 'observer': 'CRIM_Person_1016', 'piece': 'CRIM_Mass_0015_2', 'musical_type': 'fuga', 'ema': '152-155/2-3,2-3,2-3,2-3/@3-4.5+@4,@1-4+@1-4,@1-4+@1-4,@1+@1', 'remarks': '', 'curated': True, 'created': '2017-07-01T01:00:45-04:00', 'updated': '2020-07-20T14:59:12.118037-04:00', 'details': {'voice names': ['Contratenor [1]', 'Contratenor [2]'], 'entry intervals': '1+', 'time intervals': 'M1', 'periodic': False, 'regularity': None, 'sequential': False, 'inverted': False, 'retrograde': False, 'voice names reg': ['Contratenor 1', 'Contratenor 2'], 'voices': [2, 3]}}}\n",
      "{'pk': 94, 'model': 'crim.cjobservation', 'fields': {'definition': 15, 'observer': 'CRIM_Person_1016', 'piece': 'CRIM_Mass_0015_1', 'musical_type': 'periodic entry', 'ema': '86-89/2-3,1-3+5,1-3+5,1-3+5/@2-4+@4,@1-4+@1+@1-3+@2-4,@1-3+@4+@2.5-4+@1-4,@1+@1-3+@1+@1', 'remarks': 'NOT FLEXED', 'curated': True, 'created': '2017-08-03T19:38:24-04:00', 'updated': '2020-07-20T11:54:54.087316-04:00', 'details': {'voice names': ['[Contratenor 1]', '[Contratenor 2]', '[Bassus 1]', '[Discantus]', '[Contratenor 2]', '[Contratenor 1]'], 'entry intervals': '1+5-1+5+1+', 'time intervals': 'S1/1/1/1/1', 'regularity': None, 'sequential': False, 'invertible counterpoint': False, 'added entries': False, 'voice names reg': ['Contratenor 1', 'Contratenor 2', 'Bassus 1', 'Discantus', 'Contratenor 2', 'Contratenor 1'], 'voices': [2, 3, 5, 1, 3, 2]}}}\n",
      "{'pk': 99, 'model': 'crim.cjobservation', 'fields': {'definition': 15, 'observer': 'CRIM_Person_1011', 'piece': 'CRIM_Model_0001', 'musical_type': 'imitative duo', 'ema': '1-9/1,1-2,1-2,1-2,2-3,2-3+5,2-3+5,3+5,5/@1-3,@1-4+@3,@1-4+@1-4,@1+@1-4,@1-4+@1-3,@1-4+@1-4+@3,@1-3+@1-4+@1-4,@1+@1-4.5,@1-2', 'remarks': \"This ID is only the basis of a very free Fg, developed on 'Vidi speciosam sicut columbam'.\", 'curated': True, 'created': '2017-07-02T20:09:39-04:00', 'updated': '2020-08-30T14:47:35.839125-04:00', 'details': {'voice names': ['Superius', 'Contratenor', 'PrimusTenor', 'Bassus'], 'entry intervals': '4-5-4-', 'time intervals': 'S3/5/3', 'regularity': 'flexed tonal', 'invertible counterpoint': False, 'added entries': None, 'voice names reg': ['Superius', 'Contratenor', 'Tenor 1', 'Bassus'], 'voices': [1, 2, 3, 5]}}}\n",
      "{'pk': 90, 'model': 'crim.cjobservation', 'fields': {'definition': 15, 'observer': 'CRIM_Person_1016', 'piece': 'CRIM_Mass_0015_1', 'musical_type': 'non-imitative duo', 'ema': '40-49/2+5,2+4-6,2+4-6,4+6,1+2+4+6,1-2,1-2,1+6,1+6,1+6/@1-4+@1-4,@1-4.5+@3-4.5+@1-3+@3-4,@1+@1-4+@1+@1-4,@1-4+@1,@1-4+@2-4+@1-3+@1,@1-4.5+@1-3,@1-4+@1,@1-4+@1-4,@1-4+@1-3,@1+@1', 'remarks': 'The lower voice of the NIM is also used as fuga starting in m. 43.', 'curated': True, 'created': '2017-08-03T19:36:56-04:00', 'updated': '2020-08-30T14:39:08.143450-04:00', 'details': {'voice names': ['[Contratenor 1]', '[Bassus 1]', '[Tenor]', '[Bassus 2]', '[Discantus]', '[Contratenor 1]'], 'entry intervals': '1+8+', 'time intervals': 'S3/5', 'regularity': None, 'sequential': False, 'invertible counterpoint': False, 'added entries': None, 'voice names reg': ['Contratenor 1', 'Bassus 1', 'Tenor', 'Bassus 2', 'Discantus', 'Contratenor 1'], 'voices': [2, 5, 4, 6, 1, 2]}}}\n",
      "{'pk': 93, 'model': 'crim.cjobservation', 'fields': {'definition': 15, 'observer': 'CRIM_Person_1016', 'piece': 'CRIM_Model_0017', 'musical_type': 'homorhythm', 'ema': '94-98/1-3+5-6,1-6,1-6,1-6,1-2+4-6/@4+@4+@4+@4+@4,@1-3+@1-4.5+@1-3+@all+@1-3+@1-3,@1-3+@1-4+@1+@1-4+@1-4+@1-4,@1-3+@1-4.5+@all+@1-3+@1-3+@1-3,@1+@1+@1+@1+@1', 'remarks': '', 'curated': True, 'created': '2017-08-03T19:38:24-04:00', 'updated': '2021-11-02T16:00:02.112186-04:00', 'details': {'voice names': ['[Discantus]', '[Contratenor]', '[Tenor]', '[Quinta Vox]', '[Sexta Vox]', '[Bassus]'], 'type': 'staggered', 'dialogue': None, 'voice names reg': ['Discantus', 'Contratenor', 'Tenor', 'Quintus', 'Sextus', 'Bassus'], 'voices': [1, 2, 3, 4, 5, 6]}}}\n",
      "{'pk': 160, 'model': 'crim.cjobservation', 'fields': {'definition': 15, 'observer': 'CRIM_Person_1011', 'piece': 'CRIM_Mass_0002_1', 'musical_type': 'cadence', 'ema': '19-21/1+4,1+4,1+4/@2-4+@2-3,@1+@1,@1+@1', 'remarks': '', 'curated': True, 'created': '2017-07-06T10:13:20-04:00', 'updated': '2020-09-02T16:39:05.581675-04:00', 'details': {'tone': 'G', 'type': 'authentic', 'cantizans name': '[Superius]', 'tenorizans name': '[II.Tenor]', 'dovetail cadence': False, 'dovetail voice name': '', 'dovetail position': None, 'irregular cadence': None, 'irregular roles': None, 'cantizans name reg': 'Superius', 'cantizans': 1, 'tenorizans name reg': 'Tenor 2', 'tenorizans': 4, 'dovetail voice name reg': None, 'dovetail cadence voice': None}}}\n",
      "{'pk': 5206, 'model': 'crim.cjobservation', 'fields': {'definition': 15, 'observer': 'unknown', 'piece': 'CRIM_Mass_0003_1', 'musical_type': 'soggetto', 'ema': '1/1/@1-4', 'remarks': '', 'curated': True, 'created': '2019-10-05T13:33:17.057000-04:00', 'updated': '2019-10-05T13:33:17.057000-04:00', 'details': {'voice name': '[Superius]', 'features': 'both pitches and durations', 'ostinato': False, 'periodic': False, 'voice name reg': 'Superius', 'voice': 1}}}\n"
     ]
    }
   ],
   "source": [
    "# test list represents one of each basic musical type\n",
    "test_list = [9, 94, 53, 88, 64, 93, 98, 89, 92, 159, 5199]\n",
    "for item in test_list:\n",
    "    print(clean_obs[item])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'pk': 5199,\n",
       " 'model': 'crim.cjobservation',\n",
       " 'fields': {'definition': 15,\n",
       "  'observer': 'unknown',\n",
       "  'piece': 'CRIM_Model_0009',\n",
       "  'musical_type': 'soggetto',\n",
       "  'ema': '3-5/1,1,1/@1-4,@1-3,@1',\n",
       "  'remarks': '',\n",
       "  'curated': True,\n",
       "  'created': '2019-07-18T11:45:11.189000-04:00',\n",
       "  'updated': '2019-07-18T11:45:11.189000-04:00',\n",
       "  'details': {'voice name': 'Superius',\n",
       "   'features': 'both pitches and durations',\n",
       "   'ostinato': False,\n",
       "   'periodic': False,\n",
       "   'voice name reg': 'Superius',\n",
       "   'voice': 1}}}"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clean_obs[5192]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'list' object has no attribute 'columns'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m/var/folders/k5/k4j_550s2yq5776vgmw6r_xr0000gp/T/ipykernel_15840/2393695401.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mclean_obs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m: 'list' object has no attribute 'columns'"
     ]
    }
   ],
   "source": [
    "clean_obs.columns()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Export Migrated Observations as JSON"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Sends curated_obs to JSON\n",
    "\n",
    "with open(\"cleaned_observations.json\", \"w\") as f:\n",
    "    json.dump(clean_obs, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Same as list comprehension above\n",
    "curated_obs_2 = []\n",
    "for o in obs_data:\n",
    "    curated_obs_2.append(migrate_obs_data(o, people))\n",
    "# curated_obs   "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Various Filters and Tests to Check Pieces and Voices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mask = (v_data[\"piece.piece_id\"] == \"CRIM_Mass_0001_1\") & (v_data[\"CRIM_Obs_Name\"] == \"Superius\")\n",
    "# mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "v_order = v_data[mask].order\n",
    "v_reg_name = v_data[mask].regularized_name\n",
    "v_reg_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "search = v_data[v_data['piece.piece_id'].str.contains(\"0015_2\")]\n",
    "search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "v_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### The Following is Used to Check if Any Observations with Voices Spelled Incorrectly"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'curated_obs' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m/var/folders/k5/k4j_550s2yq5776vgmw6r_xr0000gp/T/ipykernel_15840/1273189937.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;31m# Finds wrong voices and reports  WORKS 11/2/21\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0;32mfor\u001b[0m \u001b[0mobs\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mcurated_obs\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m     \u001b[0mtype_1_list\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m\"fuga\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"contrapuntal duo\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"periodic Entry\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"imitative duo\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"non-imitative duo\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"homorhythm\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'curated_obs' is not defined"
     ]
    }
   ],
   "source": [
    "# Keep this!  \n",
    "# Finds wrong voices and reports  WORKS 11/2/21\n",
    "\n",
    "for obs in curated_obs:\n",
    "\n",
    "    type_1_list = [\"fuga\", \"contrapuntal duo\", \"periodic Entry\", \"imitative duo\", \"non-imitative duo\", \"homorhythm\"]\n",
    "    type_2_list = [\"cantus firmus\", \"soggetto\", \"counter soggetto\"]\n",
    "    obs_id = obs['id']\n",
    "    piece_id = obs['piece']\n",
    "    if obs[\"musical_type\"] in type_1_list:\n",
    "        search = v_data[v_data['piece.piece_id'].str.contains(piece_id)]\n",
    "        voice_list = search[\"CRIM_Obs_Name\"].to_list()\n",
    "        obs_voices = obs['details']['voices']\n",
    "        for obs_voice in obs_voices:\n",
    "            if obs_voice in voice_list:\n",
    "                pass\n",
    "            else:\n",
    "                missing = obs_voice\n",
    "                print(obs_id, piece_id, missing)\n",
    "    \n",
    "            \n",
    "    if obs[\"musical_type\"] in type_2_list:\n",
    "        search = v_data[v_data['piece.piece_id'].str.contains(piece_id)]\n",
    "        voice_list = search[\"CRIM_Obs_Name\"].to_list()\n",
    "        obs_voice = obs['details']['voice']\n",
    "        \n",
    "        if obs_voice in voice_list:\n",
    "            pass\n",
    "        else:\n",
    "            missing = obs_voice\n",
    "            print(obs_id, piece_id, missing)\n",
    "\n",
    "\n",
    "    if obs[\"musical_type\"] == \"cadence\":\n",
    "        search = v_data[v_data['piece.piece_id'].str.contains(piece_id)]\n",
    "        voice_list = search[\"CRIM_Obs_Name\"].to_list()\n",
    "        obs_voice = obs['details']['voices']['cantizans']\n",
    "        if len(obs_voice) != 0:\n",
    "            if obs_voice in voice_list:\n",
    "                pass\n",
    "            else:\n",
    "                missing = obs_voice\n",
    "                print(obs_id, piece_id, missing)\n",
    "        # the length check is needed for cadences with 'no voice for this role'\n",
    "            \n",
    "        obs_voice = obs['details']['voices']['tenorizans']\n",
    "        if len(obs_voice) != 0:\n",
    "            if obs_voice in voice_list:\n",
    "                pass\n",
    "            else:\n",
    "                missing = obs_voice\n",
    "                print(obs_id, piece_id, missing)\n",
    "        obs_voice = obs['details']['dovetail_voice']\n",
    "        if len(obs_voice) != 0:\n",
    "            if obs_voice in voice_list:\n",
    "                pass\n",
    "            else:\n",
    "                missing = obs_voice\n",
    "                print(obs_id, piece_id, missing)\n",
    "# test_obs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### More tests for Checking Voices and Voice Objects and Original Voice Names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "search = v_data[v_data['piece.piece_id'].str.contains(\"05_3\")]\n",
    "# search\n",
    "search.drop(columns=['work_title', 'voice_id', 'original_name'])\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "curated_obs['piece_id'].str.contains(\"Model_0013\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Migrating Relationships\n",
    "\n",
    "####  Format of new JSON for CRIM Django 2.0 as of 1/12/21\n",
    "\n",
    "[\n",
    "  {\n",
    "    \"name\": \"quotation\",\n",
    "    \"subtypes\": [\n",
    "      {\n",
    "        \"form\": \"radio\",\n",
    "        \"name\": \"type\",\n",
    "        \"options\": [\n",
    "          \"exact\",\n",
    "          \"monnayage\"\n",
    "        ]\n",
    "      },\n",
    "      {\n",
    "        \"form\": \"radio\",\n",
    "        \"name\": \"self\",\n",
    "        \"options\": [\n",
    "          \"none\",\n",
    "          \"enchainment\",\n",
    "          \"repetition\",\n",
    "          \"return\"\n",
    "        ]\n",
    "      }\n",
    "    ]\n",
    "  },\n",
    "  {\n",
    "    \"name\": \"mechanical transformation\",\n",
    "    \"subtypes\": [\n",
    "      {\n",
    "        \"form\": \"boolean\",\n",
    "        \"name\": \"sounding in different voices\"\n",
    "      },\n",
    "      {\n",
    "        \"form\": \"boolean\",\n",
    "        \"name\": \"melodically inverted\"\n",
    "      },\n",
    "      {\n",
    "        \"form\": \"boolean\",\n",
    "        \"name\": \"retrograde\"\n",
    "      },\n",
    "      {\n",
    "        \"form\": \"boolean\",\n",
    "        \"name\": \"metrically shifted\"\n",
    "      },\n",
    "      {\n",
    "        \"form\": \"radio\",\n",
    "        \"name\": \"transposition\",\n",
    "        \"options\": [\n",
    "          \"not transposed\",\n",
    "          \"transposed\",\n",
    "          \"transposed different amounts\"\n",
    "        ]\n",
    "      },\n",
    "      {\n",
    "        \"form\": \"boolean\",\n",
    "        \"name\": \"double or invertible counterpoint\"\n",
    "      },\n",
    "      {\n",
    "        \"form\": \"boolean\",\n",
    "        \"name\": \"systematic diminution\"\n",
    "      },\n",
    "      {\n",
    "        \"form\": \"boolean\",\n",
    "        \"name\": \"systematic augmentation\"\n",
    "      },\n",
    "      {\n",
    "        \"form\": \"radio\",\n",
    "        \"name\": \"self\",\n",
    "        \"options\": [\n",
    "          \"none\",\n",
    "          \"enchainment\",\n",
    "          \"repetition\",\n",
    "          \"return\"\n",
    "        ]\n",
    "      }\n",
    "    ]\n",
    "  },\n",
    "  {\n",
    "    \"name\": \"non-mechanical transformation\",\n",
    "    \"subtypes\": [\n",
    "      {\n",
    "        \"form\": \"radio\",\n",
    "        \"name\": \"extent\",\n",
    "        \"options\": [\n",
    "          \"amplified\",\n",
    "          \"truncated\"\n",
    "        ]\n",
    "      },\n",
    "      {\n",
    "        \"form\": \"radio\",\n",
    "        \"name\": \"activity\",\n",
    "        \"options\": [\n",
    "          \"embellished\",\n",
    "          \"reduced\"\n",
    "        ]\n",
    "      },\n",
    "      {\n",
    "        \"form\": \"boolean\",\n",
    "        \"name\": \"sounding in different voices\"\n",
    "      },\n",
    "      {\n",
    "        \"form\": \"boolean\",\n",
    "        \"name\": \"whole passage transposed\"\n",
    "      },\n",
    "      {\n",
    "        \"form\": \"boolean\",\n",
    "        \"name\": \"whole passage metrically shifted\"\n",
    "      },\n",
    "      {\n",
    "        \"form\": \"boolean\",\n",
    "        \"name\": \"melodically inverted\"\n",
    "      },\n",
    "      {\n",
    "        \"form\": \"boolean\",\n",
    "        \"name\": \"retrograde\"\n",
    "      },\n",
    "      {\n",
    "        \"form\": \"boolean\",\n",
    "        \"name\": \"new counter subject\"\n",
    "      },\n",
    "      {\n",
    "        \"form\": \"boolean\",\n",
    "        \"name\": \"old counter subject shifted metrically\"\n",
    "      },\n",
    "      {\n",
    "        \"form\": \"boolean\",\n",
    "        \"name\": \"old counter subject transposed\"\n",
    "      },\n",
    "      {\n",
    "        \"form\": \"boolean\",\n",
    "        \"name\": \"double or invertible counterpoint\"\n",
    "      },\n",
    "      {\n",
    "        \"form\": \"boolean\",\n",
    "        \"name\": \"new combination\"\n",
    "      },\n",
    "      {\n",
    "        \"form\": \"radio\",\n",
    "        \"name\": \"self\",\n",
    "        \"options\": [\n",
    "          \"none\",\n",
    "          \"enchainment\",\n",
    "          \"repetition\",\n",
    "          \"return\"\n",
    "        ]\n",
    "      }\n",
    "    ]\n",
    "  },\n",
    "  {\n",
    "    \"form\": \"boolean\",\n",
    "    \"name\": \"omission\"\n",
    "  },\n",
    "  {\n",
    "    \"form\": \"boolean\",\n",
    "    \"name\": \"new material\"\n",
    "  }\n",
    "]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Inspect an Old Relationship"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'url': 'http://crimproject.org/data/relationships/3/',\n",
       " 'id': 3,\n",
       " 'observer': {'url': 'http://crimproject.org/data/people/CRIM_Person_1010/',\n",
       "  'name': 'Drew Ivarson'},\n",
       " 'model_observation': {'url': 'http://crimproject.org/data/observations/5/',\n",
       "  'id': 5,\n",
       "  'piece': {'url': 'http://crimproject.org/data/pieces/CRIM_Model_0011/',\n",
       "   'piece_id': 'CRIM_Model_0011',\n",
       "   'full_title': 'Tota pulchra es'},\n",
       "  'ema': '5-9,14-18/1,1,1-2,1-2,1-2,3,3,3-4,3-4,3-4/@3,@1-4,@1-4+@4,@1-4+@1-4,@1-3+@1-2,@3,@1-4,@1-4+@4,@1-4+@1-4,@1-3+@1-2'},\n",
       " 'derivative_observation': {'url': 'http://crimproject.org/data/observations/6/',\n",
       "  'id': 6,\n",
       "  'piece': {'url': 'http://crimproject.org/data/pieces/CRIM_Mass_0009_5/',\n",
       "   'piece_id': 'CRIM_Mass_0009_5',\n",
       "   'full_title': 'Missa Tota pulchra es: Agnus Dei'},\n",
       "  'ema': '9-12/3-4,2-4,2,2/@4+@1-4,@4+@1-4+@1-3,@1-3,@1'},\n",
       " 'relationship_type': 'Non-mechanical transformation',\n",
       " 'musical_type': 'Fuga',\n",
       " 'rt_q': False,\n",
       " 'rt_q_x': False,\n",
       " 'rt_q_monnayage': False,\n",
       " 'rt_tm': False,\n",
       " 'rt_tm_snd': False,\n",
       " 'rt_tm_minv': False,\n",
       " 'rt_tm_retrograde': False,\n",
       " 'rt_tm_ms': False,\n",
       " 'rt_tm_transposed': False,\n",
       " 'rt_tm_invertible': False,\n",
       " 'rt_tnm': True,\n",
       " 'rt_tnm_embellished': False,\n",
       " 'rt_tnm_reduced': False,\n",
       " 'rt_tnm_amplified': False,\n",
       " 'rt_tnm_truncated': False,\n",
       " 'rt_tnm_ncs': False,\n",
       " 'rt_tnm_ocs': True,\n",
       " 'rt_tnm_ocst': True,\n",
       " 'rt_tnm_nc': False,\n",
       " 'rt_nm': False,\n",
       " 'rt_om': False,\n",
       " 'remarks': '',\n",
       " 'created': '2017-06-19T20:36:44-04:00',\n",
       " 'updated': '2020-08-02T16:19:42.196741-04:00',\n",
       " 'curated': True}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_rel = rel_data[2]\n",
    "test_rel"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Functions for Migrating Relationships\n",
    "\n",
    "* Correct as of 1/12/22"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "# getting details for types:\n",
    "# not needed for omission or new material\n",
    "def get_rel_person_id(people: list, name: str):\n",
    "    rel_pers_id = None\n",
    "    for record in people:\n",
    "            if record['name'] == name:\n",
    "                rel_pers_id = (record['person_id'])\n",
    "                # check unknown\n",
    "                if rel_pers_id == \"unknown\":\n",
    "                    rel_pers_id = \"CRIM_Person_0000\"          \n",
    "    return rel_pers_id\n",
    "\n",
    "#1/11/22 removed underscore from quotation type.  Not revised by Micah\n",
    "\n",
    "def get_quotation(rel_data):\n",
    "    if rel_data['rt_q_x'] == True:\n",
    "        quotation_type = \"exact\"\n",
    "    if rel_data['rt_q_monnayage'] == True:\n",
    "        quotation_type = \"monnayage\"\n",
    "    quotation_data = {'details':\n",
    "                      {'type': quotation_type,\n",
    "                      'self': None}} \n",
    "    return quotation_data \n",
    "\n",
    "# Micah's New Code for Mech Trans\n",
    "\n",
    "def get_mech_trans(rel_data):\n",
    "    sound_diff_voices = rel_data['rt_tm_snd']\n",
    "    mel_invert = rel_data['rt_tm_minv']\n",
    "    retrograde = rel_data['rt_tm_retrograde']\n",
    "    metric_shift = rel_data['rt_tm_ms']\n",
    "    transposed = None\n",
    "    invertible = rel_data['rt_tm_invertible']\n",
    "    \n",
    "    if rel_data['rt_tm_transposed'] is True:\n",
    "        transposition = \"transposed\"\n",
    "    elif rel_data['rt_tm_transposed'] is False:\n",
    "        transposition = \"not transposed\"\n",
    "    \n",
    "    mech_trans_data = {'details': \n",
    "                            {'melodically inverted': mel_invert, \n",
    "                            'metrically shifted': metric_shift, \n",
    "                            'retrograde': retrograde,\n",
    "                            'sounding in different voices': sound_diff_voices,\n",
    "                            'transposition': transposition,\n",
    "                            'double or invertible counterpoint': invertible,\n",
    "                            'self': None,\n",
    "                            'systematic diminution': None,\n",
    "                            'systematic augmentation': None}}\n",
    "    return mech_trans_data\n",
    "\n",
    "\n",
    "# Micah's new code:\n",
    "\n",
    "def get_non_mech_trans(rel_data):\n",
    "    activity = None\n",
    "    extent = None\n",
    "    new_counter_subject = rel_data['rt_tnm_ncs']\n",
    "    old_cs_shifted = rel_data['rt_tnm_ocs']\n",
    "    old_cs_transposed = rel_data['rt_tnm_ocst']\n",
    "    new_combination = rel_data['rt_tnm_nc']\n",
    "\n",
    "    if rel_data['rt_tnm_embellished']:\n",
    "        activity = 'embellished'\n",
    "    elif rel_data['rt_tnm_reduced']:\n",
    "        activity = 'reduced'\n",
    "\n",
    "    if rel_data['rt_tnm_amplified']:\n",
    "        extent = 'amplified'\n",
    "    elif rel_data['rt_tnm_truncated']:\n",
    "        extent = 'truncated'\n",
    "\n",
    "    non_mech_trans_data = {'details':\n",
    "                            {'activity': activity,\n",
    "                             'extent': extent,\n",
    "                             'new counter subject': new_counter_subject,\n",
    "                             'sounding in different voices': None,\n",
    "                             'whole passage transposed': None,\n",
    "                             'whole passage metrically shifted': None,\n",
    "                             'melodically inverted': None,\n",
    "                             'retrograde': None,\n",
    "                             'double or invertible counterpoint': None,\n",
    "                             'old counter subject shifted metrically': old_cs_shifted,\n",
    "                             'old counter subject transposed': old_cs_transposed,\n",
    "                             'new combination': new_combination,\n",
    "                             'double or invertible counterpoint': None,\n",
    "                             'self': None}}\n",
    "    return non_mech_trans_data  \n",
    "\n",
    "\n",
    "# Relationship Migration\n",
    "\n",
    "def migrate_rel_data(rel_data: dict, people):\n",
    "    \n",
    "    fields = {}\n",
    "    migrated_rels = {}\n",
    "    r_type = \"\"\n",
    "    details = {}\n",
    "    \n",
    "    # here we get the 'name' as recorded in the CRIM Relationship, then look up their CRIM ID\n",
    "    \n",
    "    rel_pers_name = rel_data['observer']['name']\n",
    "    rel_pers_id = get_rel_person_id(people, rel_pers_name)\n",
    "    if rel_pers_id == \"unknown\":\n",
    "        rel_pers_id = \"CRIM_Person_0000\"\n",
    "    # and now the rest of the id for the piece and rel\n",
    "    rel_id = rel_data['id']\n",
    "    \n",
    "    model_obs_id = rel_data['model_observation']['id']\n",
    "    deriv_obs_id = rel_data['derivative_observation']['id']\n",
    "    remarks = rel_data['remarks']\n",
    "    curated = rel_data['curated']\n",
    "    created = rel_data['created']\n",
    "    updated = rel_data['updated']\n",
    "    \n",
    "    # and how we test for each musical type and gather subfields\n",
    "    \n",
    "    # new material = \n",
    "    if rel_data['rt_nm'] is True:\n",
    "        r_type = \"new material\"\n",
    "        \n",
    "    # omission \n",
    "    if rel_data['rt_om'] is True:\n",
    "        r_type = \"omission\"\n",
    "        \n",
    "    # quotation\n",
    "    if rel_data['rt_q'] is True:\n",
    "        r_type = \"quotation\"\n",
    "        details = get_quotation(rel_data)\n",
    "       \n",
    "    # mechanical transformation\n",
    "    if rel_data['rt_tm'] is True:\n",
    "        r_type = \"mechanical transformation\"\n",
    "        details = get_mech_trans(rel_data)\n",
    "        \n",
    "    # non-mechanical transformation\n",
    "    if rel_data['rt_tnm'] is True:\n",
    "        r_type = \"non-mechanical transformation\"\n",
    "        details = get_non_mech_trans(rel_data)\n",
    "    \n",
    "    fields.update({'definition': 15,\n",
    "                   'observer': rel_pers_id, \n",
    "                    'relationship_type': r_type,\n",
    "                    'model_observation': model_obs_id,\n",
    "                    'derivative_observation': deriv_obs_id,\n",
    "                    'curated': curated,\n",
    "                    'created': created,\n",
    "                    'updated': updated,\n",
    "                    'remarks': remarks})\n",
    "    \n",
    "    fields.update(details)\n",
    "                   \n",
    "    # changing ID to PK, adding model information, remove data now in 'fields'\n",
    "    migrated_rels.update({'pk': rel_id,\n",
    "                        'model': \"crim.cjrelationship\",\n",
    "                        'fields': fields})\n",
    "    '''\n",
    "    this is the OLD version:\n",
    "    migrated_rels.update({'pk': rel_id,\n",
    "                        'model': \"crim.cjrelationship\",\n",
    "                        'observer': rel_pers_id, \n",
    "                        'relationship_type': r_type,\n",
    "                        'model_observation': model_obs_id,\n",
    "                        'derivative_observation': deriv_obs_id,\n",
    "                        'curated': curated,\n",
    "                        'created': created,\n",
    "                        'updated': updated,\n",
    "                        'remarks': remarks})\n",
    "    '''\n",
    "    \n",
    "    # then add the various subtypes here via 'details' assembled for each type above\n",
    "    \n",
    "    # removed 1/22/22 cf MW\n",
    "    #migrated_rels.update(details) \n",
    "    \n",
    "    return migrated_rels\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Migrate All Relationships"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "clean_rels = [migrate_rel_data(r, people) for r in rel_data]\n",
    "# clean_rels"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Migrate just one Test Relationship"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'pk': 1,\n",
       " 'model': 'crim.cjrelationship',\n",
       " 'fields': {'definition': 15,\n",
       "  'observer': 'CRIM_Person_1012',\n",
       "  'relationship_type': 'quotation',\n",
       "  'model_observation': 1,\n",
       "  'derivative_observation': 2,\n",
       "  'curated': True,\n",
       "  'created': '2017-06-19T19:22:54-04:00',\n",
       "  'updated': '2019-08-28T13:16:29.931000-04:00',\n",
       "  'remarks': '',\n",
       "  'details': {'type': 'exact', 'self': None}}}"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_rel = rel_data[0]\n",
    "clean_rel = migrate_rel_data(test_rel, people)\n",
    "clean_rel\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Export Migrated Relationships to JSON"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Sends curated_obs to JSON\n",
    "\n",
    "with open(\"cleaned_relationships.json\", \"w\") as f:\n",
    "    json.dump(clean_rels, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}