Add MathWorld

katjabercic · Oct 31, 2023 · ad255e6 · ad255e6
1 parent bfd5e0c
commit ad255e6
Show file tree

Hide file tree

Showing 5 changed files with 61 additions and 1 deletion.
diff --git a/web/concepts/migrations/0005_alter_item_options_alter_item_source.py b/web/concepts/migrations/0005_alter_item_options_alter_item_source.py
@@ -0,0 +1,30 @@
+# Generated by Django 4.2.6 on 2023-10-31 18:00
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("concepts", "0004_alter_item_description_alter_item_name"),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name="item",
+            options={"ordering": ["name", "source", "identifier"]},
+        ),
+        migrations.AlterField(
+            model_name="item",
+            name="source",
+            field=models.CharField(
+                choices=[
+                    ("Wd", "Wikidata"),
+                    ("nL", "nLab"),
+                    ("MW", "MathWorld"),
+                    ("WpEN", "Wikipedia (English)"),
+                    ("AUm", "Agda Unimath"),
+                ],
+                max_length=4,
+            ),
+        ),
+    ]
diff --git a/web/concepts/models.py b/web/concepts/models.py
@@ -5,6 +5,7 @@ class Item(models.Model):
     class Source(models.TextChoices):
         WIKIDATA = "Wd", "Wikidata"
         NLAB = "nL", "nLab"
+        MATHWORLD = "MW", "MathWorld"
         WIKIPEDIA_EN = "WpEN", "Wikipedia (English)"
         AGDA_UNIMATH = "AUm", "Agda Unimath"
 

diff --git a/web/slurper/management/commands/clear_wikidata.py b/web/slurper/management/commands/clear_wikidata.py
@@ -6,3 +6,4 @@ class Command(BaseCommand):
     def handle(self, *args, **options):
         Item.objects.filter(source=Item.Source.WIKIDATA).delete()
         Item.objects.filter(source=Item.Source.NLAB).delete()
+        Item.objects.filter(source=Item.Source.MATHWORLD).delete()
diff --git a/web/slurper/management/commands/import_wikidata.py b/web/slurper/management/commands/import_wikidata.py
@@ -6,5 +6,7 @@ class Command(BaseCommand):
     def handle(self, *args, **options):
         source_wikidata.WD_SLURPER.save_items()
         source_wikidata.WD_NLAB_SLURPER.save_items()
+        source_wikidata.WD_MATHWORLD_SLURPER.save_items()
         source_wikidata.WD_SLURPER.save_links()
         source_wikidata.WD_NLAB_SLURPER.save_links()
+        source_wikidata.WD_MATHWORLD_SLURPER.save_links()
diff --git a/web/slurper/source_wikidata.py b/web/slurper/source_wikidata.py
@@ -50,14 +50,21 @@ def save_links(self):
         for json_item in self.raw_data:
             currentItem = Item.objects.get(source=self.source, identifier=self.id_map(json_item))
             if self.source == Item.Source.WIKIDATA:
-                # nLab
+                # nLab, MathWorld
                 if "nlabID" in json_item:
                     nlab_id = json_item["nlabID"]["value"]
                     try:
                         linkToItem = Item.objects.get(source=Item.Source.NLAB, identifier=nlab_id)
                         currentItem.links.add(linkToItem)
                     except:
                         logging.log(logging.WARNING, f" NLab item {nlab_id} does not exist in the database.")
+                elif "mwID" in json_item:
+                    mw_id = json_item["mwID"]["value"]
+                    try:
+                        linkToItem = Item.objects.get(source=Item.Source.MATHWORLD, identifier=mw_id)
+                        currentItem.links.add(linkToItem)
+                    except:
+                        logging.log(logging.WARNING, f" MathWorld item {mw_id} does not exist in the database.")
             else: # link back to WD items
                 wd_id = json_item["item"]["value"].split("/")[-1]
                 try:
@@ -129,3 +136,22 @@ def save_links(self):
     desc_map=lambda _: None
     )
 
+WD_MATHWORLD_SLURPER = WikidataSlurper(
+    Item.Source.MATHWORLD, 
+    """
+SELECT
+  DISTINCT ?item ?mwID
+WHERE {
+  # anything that has the MathWorld identifier property
+  ?item wdt:P4215 ?mwID .
+  # except for humans
+  FILTER NOT EXISTS{ ?item wdt:P31 wd:Q5 . }
+  # collect the label and description
+  SERVICE wikibase:label { bd:serviceParam wikibase:language "en". }
+}
+""", 
+    id_map=lambda item: item["mwID"]["value"],
+    url_map=lambda item: "https://mathworld.wolfram.com/" + item["mwID"]["value"] + ".html",
+    name_map=lambda item: item["mwID"]["value"],
+    desc_map=lambda _: None
+    )