ChordEstimation/ACE_MIP-Frontiers.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import essentia\n",
    "import essentia.standard as ess\n",
    "import os\n",
    "import json\n",
    "import numpy as np\n",
    "import madmom.features.chords as mdm\n",
    "from IPython.display import clear_output\n",
    "import mir_eval\n",
    "import re\n",
    "import time\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "from platform import python_version\n",
    "py_version = float(python_version()[:3])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Automatic Chord Estimation (ACE) Task Overview\n",
    "\n",
    "Description from MIREX(https://www.music-ir.org/mirex/wiki/2018:Audio_Chord_Estimation):\n",
    "\n",
    ">This task requires participants to extract or transcribe a sequence of chords from an audio music recording. For many applications in music information retrieval, extracting the harmonic structure of an audio track is very desirable, for example for segmenting pieces into characteristic segments, for finding similar pieces, or for semantic analysis of music. The extraction of the harmonic structure requires the estimation of a sequence of chords that is as precise as possible. This includes the full characterisation of chords – root, quality, and bass note – as well as their chronological order, including specific onset times and durations.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dataset: Jazz Audio-Aligned Harmony (JAAH) Dataset\n",
    "\n",
    "Documentation: https://mtg.github.io/JAAH/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Feature Extraction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def computeTuning(filename, frameSize=4096, hopSize=2048):\n",
    "    audio=ess.MonoLoader(filename=filename, sampleRate=44100)()\n",
    "    \n",
    "    frameGenerator = ess.FrameGenerator(audio, \n",
    "                                        frameSize=frameSize, \n",
    "                                        hopSize=hopSize)\n",
    "    window = ess.Windowing(type='blackmanharris62')\n",
    "    spectrum = ess.Spectrum()\n",
    "    spectralPeaks = ess.SpectralPeaks(magnitudeThreshold=1e-05,\n",
    "                                        maxFrequency=5000,\n",
    "                                        minFrequency=40,\n",
    "                                        maxPeaks=1000,\n",
    "                                        orderBy=\"frequency\",\n",
    "                                        sampleRate=44100)\n",
    "    tuning = ess.TuningFrequency()\n",
    "\n",
    "    pool = essentia.Pool()\n",
    "    \n",
    "    for frame in ess.FrameGenerator(audio, frameSize=frameSize, hopSize=hopSize, startFromZero=True):\n",
    "        spectrum_mag = spectrum(window(frame))\n",
    "        frequencies, magnitudes = spectralPeaks(spectrum_mag)\n",
    "        tuneFrame, _ = tuning(frequencies, magnitudes)\n",
    "        pool.add('tuning', tuneFrame)\n",
    "    \n",
    "    return np.average(pool['tuning'])\n",
    "\n",
    "def computeHPCP(filename, frameSize=4096, hopSize=2048, tuningFrequency=440.0):\n",
    "    audio = ess.MonoLoader(filename=filename, sampleRate=44100)()\n",
    "    \n",
    "    frameGenerator = ess.FrameGenerator(audio, \n",
    "                                        frameSize=frameSize,\n",
    "                                        hopSize=hopSize,\n",
    "                                        startFromZero=True)\n",
    "    window = ess.Windowing(type='blackmanharris62')\n",
    "    spectrum = ess.Spectrum()\n",
    "    spectralPeaks = ess.SpectralPeaks(magnitudeThreshold=1e-05,\n",
    "                                      maxFrequency=5000,\n",
    "                                      minFrequency=40,\n",
    "                                      maxPeaks=1000,\n",
    "                                      orderBy=\"frequency\",\n",
    "                                      sampleRate=44100)\n",
    "    spectralWhitening = ess.SpectralWhitening(maxFrequency= 5000,\n",
    "                                              sampleRate=44100)\n",
    "    \n",
    "    hpcp = ess.HPCP(sampleRate=44100,\n",
    "                    maxFrequency=5000,\n",
    "                    minFrequency=40,\n",
    "                    referenceFrequency=tuningFrequency,\n",
    "                    nonLinear=False,\n",
    "                    harmonics=8,\n",
    "                    size=12)\n",
    "    \n",
    "    key = ess.Key(profileType=\"tonictriad\", usePolyphony=False)\n",
    "    \n",
    "    pool = essentia.Pool()\n",
    "    \n",
    "    for frame in frameGenerator:\n",
    "        spectrum_mag = spectrum(window(frame))\n",
    "        frequencies, magnitudes = spectralPeaks(spectrum_mag)\n",
    "        w_magnitudes = spectralWhitening(spectrum_mag, frequencies, magnitudes)\n",
    "        hpcp_vector = hpcp(frequencies, w_magnitudes)\n",
    "        pool.add('hpcp',hpcp_vector)\n",
    "        \n",
    "    return pool['hpcp']\n",
    "\n",
    "def computeBeats(filename):\n",
    "    audio = ess.MonoLoader(filename=filename, sampleRate=44100)()\n",
    "    \n",
    "    bt = ess.BeatTrackerMultiFeature()\n",
    "    \n",
    "    beats, confidence = bt(audio)\n",
    "    beats = essentia.array([round(beat,2) for beat in beats])\n",
    "    \n",
    "    duration = len(audio) / 44100.0\n",
    "    frameOnsets = np.arange(0, duration-(2048/44100), float(2048/44100.0))\n",
    "    frameOnsets = [round(onset,2) for onset in frameOnsets] #2 decimals\n",
    "    \n",
    "    return beats, duration, frameOnsets\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Utility functions for creating '.lab' files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ChordSegment:\n",
    "    startTime = 0.0\n",
    "    endTime = 0.0\n",
    "    chord = ''\n",
    "    def __init__(self, startTime, endTime, chord):\n",
    "        self.startTime = startTime\n",
    "        self.endTime = endTime\n",
    "        self.chord = chord\n",
    "    def __repr__(self):\n",
    "        return '{:.2f}\\t{:.2f}\\t{}'.format(self.startTime, self.endTime, self.chord)\n",
    "\n",
    "def createLabFileEssentia(duration, onsets, chords, strengths, save=False, outputName=''):\n",
    "    if (len(onsets) == len(chords)):\n",
    "        onsets.append(duration)\n",
    "    \n",
    "    allSegments = []\n",
    "    \n",
    "    if (0.0 < onsets[0]):\n",
    "        allSegments.append(ChordSegment(0.0, onsets[0], 'N'))\n",
    "    \n",
    "    for i in range(len(chords)):\n",
    "        chord = chords[i] if strengths[i] > 0 else 'N'\n",
    "        allSegments.append(ChordSegment(onsets[i], onsets[i+1], chord))\n",
    "        \n",
    "    if (allSegments[-1].endTime < duration):\n",
    "        allSegments.append(ChordSegment(allSegments[-1].endTime, duration, 'N'))\n",
    "    \n",
    "    mergedSegments = []\n",
    "    currentSegment = allSegments[0]\n",
    "    for segment in allSegments[1:]:\n",
    "        if (segment.chord == currentSegment.chord):\n",
    "            currentSegment.endTime = segment.endTime\n",
    "        else:\n",
    "            mergedSegments.append(currentSegment)\n",
    "            currentSegment = segment\n",
    "    mergedSegments.append(currentSegment)\n",
    "    \n",
    "    for segment in mergedSegments:\n",
    "        segment.chord = re.sub('m$', ':min', segment.chord)\n",
    "        \n",
    "    if save:\n",
    "        with open(outputName, 'w') as f:\n",
    "            for s in mergedSegments:\n",
    "                f.write('{}\\n'.format(s))\n",
    "        \n",
    "    return mergedSegments\n",
    "\n",
    "def createLabFileMadmom(chords, save=False, outputName=''):\n",
    "    labSegments = []\n",
    "    for segment in chords:\n",
    "        labSegments.append('{:.2f}\\t{:.2f}\\t{}'.format(segment[0], segment[1], segment[2]))\n",
    "        \n",
    "    if save:\n",
    "        with open(outputName, 'w') as f:\n",
    "            for s in labSegments:\n",
    "                f.write('{}\\n'.format(s))\n",
    "    \n",
    "    return labSegments\n",
    "\n",
    "if py_version == 3.5:\n",
    "    def createLabFileCREMA(intervals, chords, save=False, outputName=''):\n",
    "        labSegments = []\n",
    "\n",
    "        for i in range(len(chords)):\n",
    "            labSegments.append('{:.2f}\\t{:.2f}\\t{}'.format(intervals[i][0], intervals[i][1], chords[i]))\n",
    "\n",
    "        if save:\n",
    "            with open(outputName, 'w') as f:\n",
    "                for s in labSegments:\n",
    "                    f.write('{}\\n'.format(s))\n",
    "\n",
    "        return labSegments    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ACE algorithms\n",
    "### 1 - Essentia Chord Estimation by Frames\n",
    "\n",
    "This function uses pre-defined chord templates to match the corresponding PCP to a chord estimate.\n",
    "\n",
    "The chord vocabulary of this function consists of major and minor chords. \n",
    "\n",
    "You can easily give a PCP tensor to the algorithm to get the corresponding chords and confidence values with feeding them to 'ChordsDetection' function."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def essentiaChordsByFrames(hpcp):    \n",
    "    chord_extractor = ess.ChordsDetection()\n",
    "    chords, strengths = chord_extractor(hpcp)\n",
    "    \n",
    "    return chords, strengths"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The above function does not provide many parameters to modify. What 'ChordsDetection' function does can be seen below."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def estimateChordsByFrames(hpcp, onsets, hopSize=2048, profileType='tonictriad', usePolyphony=False):\n",
    "    a, b = hpcp.shape\n",
    "    \n",
    "    key = ess.Key(profileType=profileType, usePolyphony=usePolyphony)\n",
    "    \n",
    "    estimatedChords = []\n",
    "    estimatedStrengths = []\n",
    "    \n",
    "    numFramesWindow = int(2 * 44100 / hopSize) - 1\n",
    "    \n",
    "    for i in range(a):\n",
    "        begin = max(0, i - (numFramesWindow/2) - 1)\n",
    "        end = min(i + (numFramesWindow/2) - 1, a)\n",
    "        \n",
    "        meanFrame = np.mean(hpcp[begin:end], axis=0)\n",
    "        maxVal = np.max(meanFrame)\n",
    "        meanFrame /= maxVal\n",
    "        \n",
    "        estimatedKey, estimatedScale, estimatedStrength, _ = key(meanFrame)\n",
    "        if estimatedScale == 'minor':\n",
    "            estimatedChords.append('{}m'.format(estimatedKey))\n",
    "        else:\n",
    "            estimatedChords.append('{}'.format(estimatedKey))\n",
    "        estimatedStrengths.append(estimatedStrength)\n",
    "            \n",
    "    return estimatedChords, estimatedStrengths"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In the above function, we see that the algorithms calls another algorithm called 'Key', and processes the results accordingly. The documentation for 'Key' algorithm can be found in https://essentia.upf.edu/documentation/reference/std_Key.html. \n",
    "\n",
    "The default key profile is 'TonicTriad' but feel free to experiment with other profile types shown in the documentation."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2 - Essentia Chord Estimation by Beats\n",
    "\n",
    "This algorithm aggregatates the PCP tensors between beats to obtain one PCP vector per beat.\n",
    "\n",
    "The chord vocabulary of this function consists of major and minor chords.\n",
    "\n",
    "Like the previous algorithm, you can simply feed PCP tensors to the function but this time another vector that gives the beat onsets should be provided as well."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def essentiaChordsByBeats(hpcp, beats):    \n",
    "    chord_extractor = ess.ChordsDetectionBeats()    \n",
    "    chords, strengths = chord_extractor(hpcp, beats)\n",
    "    return chords, strengths"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As in the previous case, 'ChordsDetectionBeats' function aggregates the PCP vectors and calls 'Key' algorithm to obtain a final chord estimate. Decomposed version of 'ChordsDetectionBeats' can be found below."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def computeChordsByBeats(hpcp, beats, hopSize=2048, profileType='tonictriad', usePolyphony=False):\n",
    "    a, b = hpcp.shape\n",
    "    \n",
    "    key = ess.Key(profileType=profileType, usePolyphony=usePolyphony)\n",
    "    \n",
    "    estimatedChords = []\n",
    "    estimatedStrengths = []\n",
    "    \n",
    "    for i in range(len(beats)-1):\n",
    "        diffBeats = beats[i+1] - beats[i]\n",
    "        \n",
    "        numFramesWindow = int(diffBeats * 44100 / hopSize)\n",
    "        \n",
    "        begin = int(beats[i] * 44100 / hopSize)\n",
    "        end = begin + numFramesWindow - 1\n",
    "        if(begin>=end):\n",
    "            end = begin+1\n",
    "            \n",
    "        medianFrame = np.median(hpcp[begin:end], axis=0)\n",
    "        maxVal = np.max(medianFrame)\n",
    "        medianFrame /= maxVal\n",
    "        \n",
    "        estimatedKey, estimatedScale, estimatedStrength, _ = key(medianFrame)\n",
    "        if estimatedScale == 'minor':\n",
    "            estimatedChords.append('{}m'.format(estimatedKey))\n",
    "        else:\n",
    "            estimatedChords.append('{}'.format(estimatedKey))\n",
    "        estimatedStrengths.append(estimatedStrength)\n",
    "    \n",
    "    return estimatedChords, estimatedStrengths"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For this function, the default key profile is again 'Tonic Triad', and the other key profiles can be found in the documentation of 'Key' algorithm: https://essentia.upf.edu/documentation/reference/std_Key.html"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3 - Madmom Deep Chroma Chord Processor\n",
    "\n",
    "This algorithm is described in the following paper: \n",
    "\n",
    ">Filip Korzeniowski and Gerhard Widmer, “Feature Learning for Chord Recognition: The Deep Chroma Extractor”, Proceedings of the 17th International Society for Music Information Retrieval Conference (ISMIR), 2016.\n",
    "\n",
    "Korzeniowski and Widmer approaches the problem of Automatic Chord Estimation with emphasizing the importance of using a proper PCP for the chord estimation. They use a neural network that extracts 'Deep Chroma' features from a spectrogram with the following architecture:\n",
    "<img src=\"figures/deepchroma.png\" width=\"400\">\n",
    "\n",
    "After extracting the PCP tensors, in the original paper, they use a logistic regression classifier to map PCP vectors to chord labels, but in Madmom library, they use a post-processing step that uses Conditional Random Fields (CRF) to obtain the final chord label estimates.\n",
    "\n",
    "The chord vocabulary of this method consists of major and minor chords."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from madmom.audio.chroma import DeepChromaProcessor\n",
    "from madmom.features.chords import DeepChromaChordRecognitionProcessor\n",
    "\n",
    "def madmomDeepChroma(filename):\n",
    "    dcp = DeepChromaProcessor()\n",
    "    decode = DeepChromaChordRecognitionProcessor()\n",
    "    \n",
    "    pcp = dcp(filename)\n",
    "    estimatedChords = decode(pcp)\n",
    "    \n",
    "    return estimatedChords"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4 - Madmom CRF Chord Processor\n",
    "\n",
    "This algorithm is described in the following paper:\n",
    "\n",
    ">Filip Korzeniowski and Gerhard Widmer, “A Fully Convolutional Deep Auditory Model for Musical Chord Recognition”, Proceedings of IEEE International Workshop on Machine Learning for Signal Processing (MLSP), 2016.\n",
    "\n",
    "This algorithm uses a Convolutional Neural Network to obtain a latent representation to be used for chord estimation. The size of this latent representation is Tx128, T being the number of frames. The network architecture used in this feature extraction stage can be seen below:\n",
    "<img src=\"figures/crf.png\" width=\"400\">\n",
    "\n",
    "This latent representation is obtained before the last three layers. \n",
    "\n",
    "The obtained representation is then decoded with using a post-processing algorithm that uses Conditional Random Fields (CRF) to obtain final chord label estimations.\n",
    "\n",
    "The chord vocabulary of this method consists of major and minor chords."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from madmom.features.chords import CNNChordFeatureProcessor\n",
    "from madmom.features.chords import CRFChordRecognitionProcessor\n",
    "\n",
    "def madmomCRF(filename):\n",
    "    featproc = CNNChordFeatureProcessor()\n",
    "    decode = CRFChordRecognitionProcessor()\n",
    "    \n",
    "    feats = featproc(filename)\n",
    "    estimatedChords = decode(feats)\n",
    "    \n",
    "    return estimatedChords"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5 - Convolutional and Recurrent Estimators for Music Analysis (CREMA) (Only for Python 3.5)\n",
    "\n",
    "This algorithm is described in the following paper:\n",
    ">Brian McFee, Juan Pablo Bello “Structured training for large-vocabulary chord recognition”, Proceedings of the 18th International Society for Music Information Retrieval Conference (ISMIR), 2017.\n",
    "\n",
    "This method obtains a latent representation with a neural network that consists of convolutional and bi-directional gated recurrent unit (GRU) as can be seen in Figure 1. After this encoding, it uses the CR2+S architecture shown in Figure 3 to get Root, Pitch Classes and Bass representations to be used in chord label estimation. \n",
    "<img src=\"figures/encoder-crema.png\" width=\"400\"><img src=\"figures/network-crema.png\" width=\"400\">\n",
    "\n",
    "An example of the Root, Pitch Classes and Bass representation can be seen in Figure 2.\n",
    "<img src=\"figures/repr-crema.png\" width=\"400\">\n",
    "\n",
    "The chord vocabulary of this method consists of maj, min, dim, aug, min6, maj6, min7, minmaj7, maj7, 7, dim7, hdim7, sus2 and sus4 chords. \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "if py_version == 3.5:\n",
    "    from crema.analyze import analyze\n",
    "\n",
    "    def CREMA(filename):\n",
    "        jam = analyze(filename=filename)\n",
    "        intervals, chords = jam.annotations[0].to_interval_values()\n",
    "\n",
    "        return intervals, chords"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluation of the results\n",
    "\n",
    "After feature extraction and automatic chord estimation, the final step is to evaluate the obtained results. For this, we can use 'mir_eval' library, and the documentation can be found in https://craffel.github.io/mir_eval/. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluateResults(queryFile, referenceFile):\n",
    "    '''\n",
    "    expects .lab files\n",
    "    performs mir_eval on triads\n",
    "    return an object called result '''\n",
    "    \n",
    "    refIntervals, refLabels = mir_eval.io.load_labeled_intervals(referenceFile)\n",
    "    estIntervals, estLabels = mir_eval.io.load_labeled_intervals(queryFile)\n",
    "\n",
    "    estIntervals, estLabels = mir_eval.util.adjust_intervals(estIntervals,\n",
    "                                                             estLabels,\n",
    "                                                             refIntervals.min(),\n",
    "                                                             refIntervals.max(),\n",
    "                                                             mir_eval.chord.NO_CHORD,\n",
    "                                                             mir_eval.chord.NO_CHORD)\n",
    "\n",
    "    intervals, refLabels, estLabels = mir_eval.util.merge_labeled_intervals(refIntervals,\n",
    "                                                                            refLabels,\n",
    "                                                                            estIntervals,\n",
    "                                                                            estLabels)\n",
    "\n",
    "    durations = mir_eval.util.intervals_to_durations(intervals)\n",
    "    comparisons = mir_eval.chord.triads(refLabels, estLabels)\n",
    "    score = mir_eval.chord.weighted_accuracy(comparisons, durations)\n",
    "\n",
    "    return round(score, 2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Pipeline for benchmarking"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pipelineACE(filename, algos, labDir, labDirRef):\n",
    "    songName = os.path.basename(filename).split('.')[0]\n",
    "    \n",
    "    # Feature extraction\n",
    "    \n",
    "    tuning = computeTuning(filename)\n",
    "    pcp = computeHPCP(filename)\n",
    "    beats, duration, onsets = computeBeats(filename)\n",
    "\n",
    "    # Automatic chord estimation\n",
    "    \n",
    "    if 'frame' in algos:\n",
    "        # 1 - Essentia chord estimation by frames\n",
    "        chordsFrame, strengthsFrame = essentiaChordsByFrames(pcp)\n",
    "        # Save lab files\n",
    "        labFrameEssentia = createLabFileEssentia(duration, \n",
    "                                                 onsets, \n",
    "                                                 chordsFrame, \n",
    "                                                 strengthsFrame,\n",
    "                                                 save=True,\n",
    "                                                 outputName='{}{}_frame.lab'.format(labDir, songName))\n",
    "    if 'beats' in algos:\n",
    "        # 2 - Essentia chord estimation by beats\n",
    "        chordsBeats, strengthsBeats = essentiaChordsByBeats(pcp, beats)\n",
    "        # Save lab files\n",
    "        labBeatsEssentia = createLabFileEssentia(duration,\n",
    "                                                 beats,\n",
    "                                                 chordsBeats,\n",
    "                                                 strengthsBeats,\n",
    "                                                 save=True,\n",
    "                                                 outputName='{}{}_beats.lab'.format(labDir, songName))\n",
    "    \n",
    "    if 'deepchroma' in algos:\n",
    "        # 3 - Madmom Deep Chroma chord processor\n",
    "        chordsDeepChroma = madmomDeepChroma(filename)\n",
    "        # Save lab files\n",
    "        labDeepChromaMadmom = createLabFileMadmom(chordsDeepChroma, \n",
    "                                                  save=True,\n",
    "                                                  outputName='{}{}_deepchroma.lab'.format(labDir, songName))\n",
    "\n",
    "    if 'crf' in algos:\n",
    "        # 4 - Madmom CRF chord processor\n",
    "        chordsCRF = madmomCRF(filename)\n",
    "        # Save lab files\n",
    "        labCRFMadmom = createLabFileMadmom(chordsCRF, \n",
    "                                           save=True,\n",
    "                                           outputName='{}{}_crf.lab'.format(labDir, songName))\n",
    "    \n",
    "    if 'crema' in algos:\n",
    "        # 5 - CREMA (Only for Python 3.5)\n",
    "        intervalsCREMA, chordsCREMA = CREMA(filename)\n",
    "        # Save lab files\n",
    "        labCREMA = createLabFileCREMA(intervalsCREMA, \n",
    "                                      chordsCREMA,\n",
    "                                      save=True,\n",
    "                                      outputName='{}{}_crema.lab'.format(labDir, songName))\n",
    "        \n",
    "    # Evaluation\n",
    "\n",
    "    scores = {}\n",
    "\n",
    "    for algo in algos:\n",
    "        scores[algo] = evaluateResults('{}{}_{}.lab'.format(labDir, songName, algo), \n",
    "                                       '{}{}_ref.lab'.format(labDirRef, songName))\n",
    "\n",
    "    return scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getAverageScores(scoreDict, algos):\n",
    "    # Initializing lists for averaging\n",
    "    scores = {}\n",
    "    for algo in algos:\n",
    "        scores[algo] = []\n",
    "        \n",
    "    for song in scoreDict.keys():\n",
    "        for algo in algos:\n",
    "            scores[algo].append(scoreDict[song][algo])\n",
    "    \n",
    "    averageResults = {}\n",
    "    for key in scores.keys():\n",
    "        averageResults[key] = round(np.mean(np.array(scores[key])), 2)\n",
    "        \n",
    "    return averageResults"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Processing all the files in the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Score for frame method is 0.4\n",
      "Score for crema method is 0.56\n",
      "Score for crf method is 0.55\n",
      "Score for deepchroma method is 0.5\n",
      "Score for beats method is 0.37\n"
     ]
    }
   ],
   "source": [
    "rootDir = 'songs_debug/'\n",
    "labDir = 'labFiles/'\n",
    "labDirRef = 'labFiles_ref/'\n",
    "\n",
    "scoreDict = {}\n",
    "\n",
    "algos = ['frame', 'beats', 'deepchroma', 'crf']\n",
    "if py_version == 3.5:\n",
    "    algos.append('crema')\n",
    "\n",
    "for paths, subdirs, files in os.walk(rootDir):\n",
    "    for file in files:\n",
    "        songName = file.split('.')[0]\n",
    "\n",
    "        startTime = time.time()\n",
    "        \n",
    "        print('Processing file: {}'.format(file))\n",
    "        \n",
    "        scoreDict[songName] = pipelineACE('{}{}'.format(rootDir, file), algos, labDir, labDirRef)\n",
    "        \n",
    "        print('Total processing time for {} is {:.2f} seconds'.format(file, time.time()-startTime))\n",
    "        \n",
    "averageScores = getAverageScores(scoreDict, algos)\n",
    "\n",
    "clear_output()\n",
    "for algo in averageScores.keys():\n",
    "    print('Score for {} method is {}'.format(algo, averageScores[algo]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}