demo/aurehal_struct_incoming_list.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "2457d7ad",
   "metadata": {},
   "source": [
    "# Former un tableau des structures non valides avec le nombre de documents associés"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1d0184c7",
   "metadata": {},
   "source": [
    "- Récupérer la liste des structures non valides\n",
    "- La transformer en tableau (dataframe)\n",
    "- Renommer les colonnes : `structId, label`\n",
    "- Définir une fonction pour récupérer à partir du `structId` le nb de publication\n",
    "- Ajouter la colonne nombre de documents\n",
    "- Ajouter une colonne de liens vers la structure dans Aurehal\n",
    "- Trier le tableau par ordre decroissant de nombre de documents\n",
    "- Extraire le tableau"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "94106d2e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests, json\n",
    "#uvsq 81173, U orléans 300297, UPCité 557826, angers 74911\n",
    "structId = 557826"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "3fe43a3f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'response': {'numFound': 24,\n",
       "  'start': 0,\n",
       "  'numFoundExact': True,\n",
       "  'docs': [{'docid': '1049198',\n",
       "    'label_s': 'Service de Psychopathologie [Bobigny]'},\n",
       "   {'docid': '1007985',\n",
       "    'label_s': 'European Reference Network on respiratory diseases [ERN LUNG]'},\n",
       "   {'docid': '1063040',\n",
       "    'label_s': \"Equipe Mobile d'Infectiologie [APHP Centre, Paris]\"},\n",
       "   {'docid': '1004389',\n",
       "    'label_s': 'Reference Center for Constitutional Bone Diseases [Paris]'},\n",
       "   {'docid': '1005164', 'label_s': 'Biophysique Cellulaire'},\n",
       "   {'docid': '1004052',\n",
       "    'label_s': 'Centre de ressources pour la remédiation cognitive et la réadaptation psychosociale [Hôpital Sainte-Anne - APHP] [C3RP]'},\n",
       "   {'docid': '1071834', 'label_s': 'Equipe INSIGHT [Paris]'},\n",
       "   {'docid': '1059686',\n",
       "    'label_s': 'Biostatistique, Traitement et Modélisation des données biologiques'},\n",
       "   {'docid': '1055760',\n",
       "    'label_s': \"Service de Psychiatrie et d'Addictologie de l'adulte et du sujet âgé [DMU Psychiatrie et Addictologie]\"},\n",
       "   {'docid': '1072632',\n",
       "    'label_s': 'Ecole doctorale Médicament -Toxicologie - Chimie - Imageries [Université de Paris] [ED 563]'},\n",
       "   {'docid': '1077249',\n",
       "    'label_s': 'Pediatric and Perinatal Drug Evaluation and Pharmacology'},\n",
       "   {'docid': '1079019',\n",
       "    'label_s': 'UFR des Sciences Fondamentales et Biomédicales [Université de Paris]'},\n",
       "   {'docid': '1096119',\n",
       "    'label_s': 'aboratoire Informatique Paris Descartes [LIPADE]'},\n",
       "   {'docid': '1062246',\n",
       "    'label_s': 'FHU NeuroVasc [Site Sainte-Anne, Paris] [GHU-PPN]'},\n",
       "   {'docid': '1008089', 'label_s': 'geoteca [geoteca]'},\n",
       "   {'docid': '1008222', 'label_s': 'Thoracic Surgery Department [Paris]'},\n",
       "   {'docid': '1004387',\n",
       "    'label_s': 'Department of Bone and Joint Imaging [Paris]'},\n",
       "   {'docid': '1007316',\n",
       "    'label_s': 'Department of Diabetes and Endocrinology [Paris]'},\n",
       "   {'docid': '1030919', 'label_s': 'Plastic Surgery Department [Paris]'},\n",
       "   {'docid': '1077254', 'label_s': 'Paris Centre University Hospitals'},\n",
       "   {'docid': '1004388',\n",
       "    'label_s': 'Department of Biochemistry and Genetics [Paris]'},\n",
       "   {'docid': '1004391',\n",
       "    'label_s': 'Department of Physical Medicine and Rehabilitation [Paris]'},\n",
       "   {'docid': '1042127',\n",
       "    'label_s': \"Service de psychiatrie et d'addictologie de l'adulte et du sujet âgé [Centre-Université de Paris]\"},\n",
       "   {'docid': '1042137',\n",
       "    'label_s': 'Service de Psychiatrie-Addictologie [CHU Cochin - APHP] [Site Tarnier]'}]}}"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "req = requests.get(f\"https://api.archives-ouvertes.fr/ref/structure/?q=parentDocid_i:{structId}\\\n",
    "&fq=valid_s:INCOMING&rows=10000\")\n",
    "l_incoming = req.json()\n",
    "l_incoming"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bb208493",
   "metadata": {},
   "source": [
    "## Transformer la liste en tableau"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "6977e887",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "499cdb7b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>docid</th>\n",
       "      <th>label_s</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1049198</td>\n",
       "      <td>Service de Psychopathologie [Bobigny]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1007985</td>\n",
       "      <td>European Reference Network on respiratory dise...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1063040</td>\n",
       "      <td>Equipe Mobile d'Infectiologie [APHP Centre, Pa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1004389</td>\n",
       "      <td>Reference Center for Constitutional Bone Disea...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     docid                                            label_s\n",
       "0  1049198              Service de Psychopathologie [Bobigny]\n",
       "1  1007985  European Reference Network on respiratory dise...\n",
       "2  1063040  Equipe Mobile d'Infectiologie [APHP Centre, Pa...\n",
       "3  1004389  Reference Center for Constitutional Bone Disea..."
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "struct_incoming = l_incoming[\"response\"][\"docs\"]\n",
    "df = pd.DataFrame(struct_incoming)\n",
    "df[:4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "3491f6e4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>structId</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>1079019</td>\n",
       "      <td>UFR des Sciences Fondamentales et Biomédicales...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>1077254</td>\n",
       "      <td>Paris Centre University Hospitals</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1004052</td>\n",
       "      <td>Centre de ressources pour la remédiation cogni...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   structId                                              label\n",
       "11  1079019  UFR des Sciences Fondamentales et Biomédicales...\n",
       "19  1077254                  Paris Centre University Hospitals\n",
       "5   1004052  Centre de ressources pour la remédiation cogni..."
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# renomer les colonnes\n",
    "df.columns = [\"structId\", \"label\"]\n",
    "df.sample(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0ba4e20c",
   "metadata": {},
   "source": [
    "## Definir une fonction pour récupérer à partir de `structId` le nb de publications"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "e69d627d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def req_hal(structId):\n",
    "    req = requests.get(f\"https://api.archives-ouvertes.fr/search/?q=structId_i:{structId}&rows=0\")\n",
    "    req = req.json()\n",
    "    return req[\"response\"][\"numFound\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "cf46531c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# test\n",
    "req_hal(49833)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "35cc55a2",
   "metadata": {},
   "source": [
    "## Ajouter la colonne nombre de documents à la DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "f854cd45",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>structId</th>\n",
       "      <th>label</th>\n",
       "      <th>nb_docs</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>1008089</td>\n",
       "      <td>geoteca [geoteca]</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1077249</td>\n",
       "      <td>Pediatric and Perinatal Drug Evaluation and Ph...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>1030919</td>\n",
       "      <td>Plastic Surgery Department [Paris]</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>1004391</td>\n",
       "      <td>Department of Physical Medicine and Rehabilita...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1049198</td>\n",
       "      <td>Service de Psychopathologie [Bobigny]</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   structId                                              label  nb_docs\n",
       "14  1008089                                  geoteca [geoteca]        9\n",
       "10  1077249  Pediatric and Perinatal Drug Evaluation and Ph...        1\n",
       "18  1030919                 Plastic Surgery Department [Paris]        2\n",
       "21  1004391  Department of Physical Medicine and Rehabilita...        2\n",
       "0   1049198              Service de Psychopathologie [Bobigny]        1"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"nb_docs\"] = df.apply(lambda row : req_hal(row[\"structId\"]), axis = 1)\n",
    "df.sample(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "852501d8",
   "metadata": {},
   "source": [
    "##  Ajouter une colonne de liens vers la structure dans Aurehal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "509a45ef",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>structId</th>\n",
       "      <th>label</th>\n",
       "      <th>nb_docs</th>\n",
       "      <th>aurehal_link</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>1004052</td>\n",
       "      <td>Centre de ressources pour la remédiation cogni...</td>\n",
       "      <td>2</td>\n",
       "      <td>https://aurehal.archives-ouvertes.fr/structure...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1077258</td>\n",
       "      <td>Faculté de Santé</td>\n",
       "      <td>1</td>\n",
       "      <td>https://aurehal.archives-ouvertes.fr/structure...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1049198</td>\n",
       "      <td>Service de Psychopathologie [Bobigny]</td>\n",
       "      <td>1</td>\n",
       "      <td>https://aurehal.archives-ouvertes.fr/structure...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>1004388</td>\n",
       "      <td>Department of Biochemistry and Genetics [Paris]</td>\n",
       "      <td>1</td>\n",
       "      <td>https://aurehal.archives-ouvertes.fr/structure...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    structId                                              label  nb_docs  \\\n",
       "23   1004052  Centre de ressources pour la remédiation cogni...        2   \n",
       "3    1077258                                   Faculté de Santé        1   \n",
       "2    1049198              Service de Psychopathologie [Bobigny]        1   \n",
       "20   1004388    Department of Biochemistry and Genetics [Paris]        1   \n",
       "\n",
       "                                         aurehal_link  \n",
       "23  https://aurehal.archives-ouvertes.fr/structure...  \n",
       "3   https://aurehal.archives-ouvertes.fr/structure...  \n",
       "2   https://aurehal.archives-ouvertes.fr/structure...  \n",
       "20  https://aurehal.archives-ouvertes.fr/structure...  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"aurehal_link\"] = \"https://aurehal.archives-ouvertes.fr/structure/read/id/\" + df[\"structId\"].apply(str)\n",
    "df.sample(4)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "695e36c3",
   "metadata": {},
   "source": [
    "## Trier le tableau par ordre décroissant de nombre de documents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "2b2de39d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>structId</th>\n",
       "      <th>label</th>\n",
       "      <th>nb_docs</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>1062246</td>\n",
       "      <td>FHU NeuroVasc [Site Sainte-Anne, Paris] [GHU-PPN]</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>1008089</td>\n",
       "      <td>geoteca [geoteca]</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1072632</td>\n",
       "      <td>Ecole doctorale Médicament -Toxicologie - Chim...</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1007985</td>\n",
       "      <td>European Reference Network on respiratory dise...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   structId                                              label  nb_docs\n",
       "13  1062246  FHU NeuroVasc [Site Sainte-Anne, Paris] [GHU-PPN]       16\n",
       "14  1008089                                  geoteca [geoteca]        9\n",
       "9   1072632  Ecole doctorale Médicament -Toxicologie - Chim...        6\n",
       "1   1007985  European Reference Network on respiratory dise...        3"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.sort_values(by = [\"nb_docs\"], ascending =  False)[:4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "273c032c",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_sorted = df.sort_values(by = [\"nb_docs\"], ascending =  False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "959aac43",
   "metadata": {},
   "outputs": [],
   "source": [
    "#exporter au format .csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "9e106cb7",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_sorted.to_csv(\"incoming_struct.csv\", index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "88d07d7a",
   "metadata": {},
   "source": [
    "----\n",
    "## Allez plus loin\n",
    "\n",
    "ajouter une colonne \"last update structure\" (champs `updateDate_s` de l'API)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}