Skip to content

Commit

Permalink
cleanup
Browse files Browse the repository at this point in the history
  • Loading branch information
@fekad
fekad committed Jan 23, 2025
1 parent db5436a commit 5d8cc85
Showing 1 changed file with 34 additions and 65 deletions.
99 changes: 34 additions & 65 deletions notebooks/tutorial_stats.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
"import json\n",
"\n",
"with open('tutorials.json') as f:\n",
" tutorials = json.load(f)['tutorials']\n",
" tutorials = json.load(f)['tutorials']\n",
"\n",
"print(tutorials)"
]
Expand All @@ -32,7 +32,7 @@
"metadata": {},
"outputs": [],
"source": [
"keys=set()\n",
"keys = set()\n",
"for tutorial in tutorials:\n",
" keys.update(list(tutorial.keys()))\n",
"keys"
Expand All @@ -44,9 +44,9 @@
"metadata": {},
"outputs": [],
"source": [
"keys=set()\n",
"keys = set()\n",
"for tutorial in tutorials:\n",
" keys.update(list(tutorial.get('flags',{}).keys()))\n",
" keys.update(list(tutorial.get('flags', {}).keys()))\n",
"'flags', keys"
]
},
Expand All @@ -56,7 +56,7 @@
"metadata": {},
"outputs": [],
"source": [
"keys=set()\n",
"keys = set()\n",
"for tutorial in tutorials:\n",
" keys.update(list(tutorial['labels'].keys()))\n",
"keys"
Expand All @@ -68,9 +68,9 @@
"metadata": {},
"outputs": [],
"source": [
"keys=set()\n",
"keys = set()\n",
"for tutorial in tutorials:\n",
" keys.update(list(tutorial.get('labels').get('application_keyword',[])))\n",
" keys.update(list(tutorial.get('labels').get('application_keyword', [])))\n",
"keys"
]
},
Expand All @@ -80,9 +80,9 @@
"metadata": {},
"outputs": [],
"source": [
"keys=set()\n",
"keys = set()\n",
"for tutorial in tutorials:\n",
" keys.update(list(tutorial.get('labels').get('application_section',[])))\n",
" keys.update(list(tutorial.get('labels').get('application_section', [])))\n",
"keys"
]
},
Expand All @@ -92,9 +92,9 @@
"metadata": {},
"outputs": [],
"source": [
"keys=set()\n",
"keys = set()\n",
"for tutorial in tutorials:\n",
" keys.update(list(tutorial.get('labels').get('application_system',[])))\n",
" keys.update(list(tutorial.get('labels').get('application_system', [])))\n",
"keys"
]
},
Expand All @@ -104,7 +104,7 @@
"metadata": {},
"outputs": [],
"source": [
"keys=set()\n",
"keys = set()\n",
"for tutorial in tutorials:\n",
" keys.update(list(tutorial.get('labels').get('category')))\n",
"keys"
Expand All @@ -116,9 +116,9 @@
"metadata": {},
"outputs": [],
"source": [
"keys=set()\n",
"keys = set()\n",
"for tutorial in tutorials:\n",
" keys.update(list(tutorial.get('labels').get('ai_methods',[])))\n",
" keys.update(list(tutorial.get('labels').get('ai_methods', [])))\n",
"keys"
]
},
Expand All @@ -128,9 +128,9 @@
"metadata": {},
"outputs": [],
"source": [
"keys=set()\n",
"keys = set()\n",
"for tutorial in tutorials:\n",
" keys.update(list(tutorial.get('labels').get('language',[])))\n",
" keys.update(list(tutorial.get('labels').get('language', [])))\n",
"keys"
]
},
Expand All @@ -140,7 +140,7 @@
"metadata": {},
"outputs": [],
"source": [
"keys=set()\n",
"keys = set()\n",
"for tutorial in tutorials:\n",
" keys.update(list(tutorial.get('labels').get('platform')))\n",
"keys"
Expand All @@ -153,7 +153,7 @@
"outputs": [],
"source": [
"tutorial = tutorials[0]\n",
"tutorial\n"
"tutorial"
]
},
{
Expand All @@ -162,13 +162,10 @@
"metadata": {},
"outputs": [],
"source": [
"authors=[]\n",
"authors = []\n",
"for author in tutorial.get('authors'):\n",
" last_name, first_name = author.split(',', 1)\n",
" authors.append({\n",
" 'last_name':last_name.strip(),\n",
" 'first_name': first_name.strip()\n",
" })\n",
" authors.append({'last_name': last_name.strip(), 'first_name': first_name.strip()})\n",
"\n",
"authors[-1]['email'] = tutorial.get('email')\n",
"authors"
Expand All @@ -182,9 +179,9 @@
"source": [
"map_categories = {\n",
" 'advanced_tutorial': 'Advanced tutorial',\n",
" 'beginner_tutorial': 'Beginner tutorial',\n",
" 'beginner_tutorial': 'Beginner tutorial',\n",
" 'intermediate_tutorial': 'Intermediate tutorial',\n",
" 'query_tutorial':'Query tutorial'\n",
" 'query_tutorial': 'Query tutorial',\n",
"}"
]
},
Expand All @@ -196,25 +193,23 @@
"source": [
"def build_new_tutorial(tutorial):\n",
" new = {\n",
" \"m_def\": \"nomad_aitoolkit.schema.AIToolkitNotebook\",\n",
" 'm_def': 'nomad_aitoolkit.schema.AIToolkitNotebook',\n",
" 'name': tutorial['title'],\n",
" 'description': tutorial['description'],\n",
" 'date': tutorial['updated'],\n",
"\n",
" 'category': map_categories[tutorial['labels']['category'][0]],\n",
" 'methods': [{'name': v} for v in tutorial['labels']['ai_methods']],\n",
" 'applications': [{'name': v} for v in tutorial['labels']['application_system']],\n",
" 'platform': 'Python'\n",
" 'platform': 'Python',\n",
" }\n",
"\n",
" new['authors'] = []\n",
" for author in tutorial['authors']:\n",
" last_name, first_name = author.split(',', 1)\n",
"\n",
" new['authors'].append({\n",
" 'last_name': last_name.strip(),\n",
" 'first_name': first_name.strip()\n",
" })\n",
" new['authors'].append(\n",
" {'last_name': last_name.strip(), 'first_name': first_name.strip()}\n",
" )\n",
"\n",
" # print(tutorial.get('email'))\n",
" # new['authors'][-1]['email'] = tutorial.get('email')\n",
Expand All @@ -229,46 +224,20 @@
" # ]\n",
"\n",
" new['references'].append(\n",
" {\n",
" 'kind': 'article_doi',\n",
" 'uri': tutorial.get('link_doi_paper')\n",
" }\n",
" {'kind': 'article_doi', 'uri': tutorial.get('link_doi_paper')}\n",
" )\n",
"\n",
"\n",
" if tutorial.get('link_public'):\n",
" new['references'].append(\n",
" {\n",
" 'kind': 'hub',\n",
" 'uri': tutorial['link_public']\n",
" }\n",
" )\n",
"\n",
" new['references'].append({'kind': 'hub', 'uri': tutorial['link_public']})\n",
"\n",
" if tutorial.get('link_paper'):\n",
" new['references'].append(\n",
" {\n",
" 'kind': 'article_url',\n",
" 'uri': tutorial['link_paper']\n",
" }\n",
" )\n",
"\n",
" new['references'].append({'kind': 'article_url', 'uri': tutorial['link_paper']})\n",
"\n",
" if tutorial.get('url'):\n",
" new['references'].append(\n",
" {\n",
" 'kind': 'repository',\n",
" 'uri': tutorial['url']\n",
" }\n",
" )\n",
" new['references'].append({'kind': 'repository', 'uri': tutorial['url']})\n",
"\n",
" if tutorial.get('link_video'):\n",
" new['references'].append(\n",
" {\n",
" 'kind': 'video',\n",
" 'uri': tutorial['link_video']\n",
" }\n",
" )\n",
" new['references'].append({'kind': 'video', 'uri': tutorial['link_video']})\n",
"\n",
" slug = tutorial['url'].rsplit('/', 1)[1]\n",
" if slug.startswith('analytics-'):\n",
Expand All @@ -280,8 +249,8 @@
"for tutorial in tutorials:\n",
" slug, new_tutorial = build_new_tutorial(tutorial)\n",
"\n",
" with open(slug+\".archive.json\", \"w\") as outfile:\n",
" json.dump(new_tutorial, outfile, indent=2)\n"
" with open(slug + '.archive.json', 'w') as outfile:\n",
" json.dump(new_tutorial, outfile, indent=2)"
]
},
{
Expand Down

0 comments on commit 5d8cc85

Please sign in to comment.