From ad255e6699fb5faf623eae3e83a02fbf3c0f86bb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Katja=20Ber=C4=8Di=C4=8D?= <katja.bercic@gmail.com>
Date: Tue, 31 Oct 2023 19:02:59 +0100
Subject: [PATCH] Add MathWorld

---
 ...05_alter_item_options_alter_item_source.py | 30 +++++++++++++++++++
 web/concepts/models.py                        |  1 +
 .../management/commands/clear_wikidata.py     |  1 +
 .../management/commands/import_wikidata.py    |  2 ++
 web/slurper/source_wikidata.py                | 28 ++++++++++++++++-
 5 files changed, 61 insertions(+), 1 deletion(-)
 create mode 100644 web/concepts/migrations/0005_alter_item_options_alter_item_source.py

diff --git a/web/concepts/migrations/0005_alter_item_options_alter_item_source.py b/web/concepts/migrations/0005_alter_item_options_alter_item_source.py
new file mode 100644
index 0000000..5d752b3
--- /dev/null
+++ b/web/concepts/migrations/0005_alter_item_options_alter_item_source.py
@@ -0,0 +1,30 @@
+# Generated by Django 4.2.6 on 2023-10-31 18:00
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("concepts", "0004_alter_item_description_alter_item_name"),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name="item",
+            options={"ordering": ["name", "source", "identifier"]},
+        ),
+        migrations.AlterField(
+            model_name="item",
+            name="source",
+            field=models.CharField(
+                choices=[
+                    ("Wd", "Wikidata"),
+                    ("nL", "nLab"),
+                    ("MW", "MathWorld"),
+                    ("WpEN", "Wikipedia (English)"),
+                    ("AUm", "Agda Unimath"),
+                ],
+                max_length=4,
+            ),
+        ),
+    ]
diff --git a/web/concepts/models.py b/web/concepts/models.py
index 67bea90..9cdd29b 100644
--- a/web/concepts/models.py
+++ b/web/concepts/models.py
@@ -5,6 +5,7 @@ class Item(models.Model):
     class Source(models.TextChoices):
         WIKIDATA = "Wd", "Wikidata"
         NLAB = "nL", "nLab"
+        MATHWORLD = "MW", "MathWorld"
         WIKIPEDIA_EN = "WpEN", "Wikipedia (English)"
         AGDA_UNIMATH = "AUm", "Agda Unimath"
 
diff --git a/web/slurper/management/commands/clear_wikidata.py b/web/slurper/management/commands/clear_wikidata.py
index e4184b0..5a24dcc 100644
--- a/web/slurper/management/commands/clear_wikidata.py
+++ b/web/slurper/management/commands/clear_wikidata.py
@@ -6,3 +6,4 @@ class Command(BaseCommand):
     def handle(self, *args, **options):
         Item.objects.filter(source=Item.Source.WIKIDATA).delete()
         Item.objects.filter(source=Item.Source.NLAB).delete()
+        Item.objects.filter(source=Item.Source.MATHWORLD).delete()
diff --git a/web/slurper/management/commands/import_wikidata.py b/web/slurper/management/commands/import_wikidata.py
index e1ff877..1e8628e 100644
--- a/web/slurper/management/commands/import_wikidata.py
+++ b/web/slurper/management/commands/import_wikidata.py
@@ -6,5 +6,7 @@ class Command(BaseCommand):
     def handle(self, *args, **options):
         source_wikidata.WD_SLURPER.save_items()
         source_wikidata.WD_NLAB_SLURPER.save_items()
+        source_wikidata.WD_MATHWORLD_SLURPER.save_items()
         source_wikidata.WD_SLURPER.save_links()
         source_wikidata.WD_NLAB_SLURPER.save_links()
+        source_wikidata.WD_MATHWORLD_SLURPER.save_links()
diff --git a/web/slurper/source_wikidata.py b/web/slurper/source_wikidata.py
index 7a23c3b..7279c58 100644
--- a/web/slurper/source_wikidata.py
+++ b/web/slurper/source_wikidata.py
@@ -50,7 +50,7 @@ def save_links(self):
         for json_item in self.raw_data:
             currentItem = Item.objects.get(source=self.source, identifier=self.id_map(json_item))
             if self.source == Item.Source.WIKIDATA:
-                # nLab
+                # nLab, MathWorld
                 if "nlabID" in json_item:
                     nlab_id = json_item["nlabID"]["value"]
                     try:
@@ -58,6 +58,13 @@ def save_links(self):
                         currentItem.links.add(linkToItem)
                     except:
                         logging.log(logging.WARNING, f" NLab item {nlab_id} does not exist in the database.")
+                elif "mwID" in json_item:
+                    mw_id = json_item["mwID"]["value"]
+                    try:
+                        linkToItem = Item.objects.get(source=Item.Source.MATHWORLD, identifier=mw_id)
+                        currentItem.links.add(linkToItem)
+                    except:
+                        logging.log(logging.WARNING, f" MathWorld item {mw_id} does not exist in the database.")
             else: # link back to WD items
                 wd_id = json_item["item"]["value"].split("/")[-1]
                 try:
@@ -129,3 +136,22 @@ def save_links(self):
     desc_map=lambda _: None
     )
 
+WD_MATHWORLD_SLURPER = WikidataSlurper(
+    Item.Source.MATHWORLD, 
+    """
+SELECT
+  DISTINCT ?item ?mwID
+WHERE {
+  # anything that has the MathWorld identifier property
+  ?item wdt:P4215 ?mwID .
+  # except for humans
+  FILTER NOT EXISTS{ ?item wdt:P31 wd:Q5 . }
+  # collect the label and description
+  SERVICE wikibase:label { bd:serviceParam wikibase:language "en". }
+}
+""", 
+    id_map=lambda item: item["mwID"]["value"],
+    url_map=lambda item: "https://mathworld.wolfram.com/" + item["mwID"]["value"] + ".html",
+    name_map=lambda item: item["mwID"]["value"],
+    desc_map=lambda _: None
+    )
\ No newline at end of file