Finish adding Wikipedia (English)

web/concepts/templates/index.html

@@ -22,6 +22,7 @@ <h2>About</h2>
 <p>Mathswitch collects entries from the following sources:</p>
 <ul>
   <li><a href="https://www.wikidata.org/wiki/Wikidata:Main_Page">Wikidata</a> ({{ number_of_links.wikidata }} entries)</li>
+  <li><a href="https://en.m.wikipedia.org/wiki/Main_Page">Wikipedia (EN)</a> ({{ number_of_links.wikipedia_en }} entries)</li>
   <li><a href="https://ncatlab.org/nlab/show/HomePage">nLab</a> ({{ number_of_links.nlab }} entries)</li>
   <li><a href="https://mathworld.wolfram.com">Mathworld</a> ({{ number_of_links.mathworld }} entries)</li>
   <li><a href="https://proofwiki.org/wiki/Main_Page">ProofWiki</a> ({{ number_of_links.proof_wiki }} entries),</li>

web/concepts/views.py

@@ -25,6 +25,7 @@ def home(request):
         "concepts": autocomplete_names,
         "number_of_links": {
             "wikidata": Item.objects.filter(source=Item.Source.WIKIDATA).count(),
+            "wikipedia_en": Item.objects.filter(source=Item.Source.WIKIPEDIA_EN).count(),
             "nlab": Item.objects.filter(source=Item.Source.NLAB).count(),
             "mathworld": Item.objects.filter(source=Item.Source.MATHWORLD).count(),
             "proof_wiki": Item.objects.filter(source=Item.Source.PROOF_WIKI).count(),

web/slurper/management/commands/clear_wikidata.py

@@ -1,9 +1,11 @@
 from concepts.models import Item
 from django.core.management.base import BaseCommand
+from wd_raw_item import WD_OTHER_SOURCES
 
 
 class Command(BaseCommand):
     def handle(self, *args, **options):
         Item.objects.filter(source=Item.Source.WIKIDATA).delete()
-        Item.objects.filter(source=Item.Source.NLAB).delete()
-        Item.objects.filter(source=Item.Source.MATHWORLD).delete()
+        Item.objects.filter(source=Item.Source.WIKIPEDIA_EN).delete()
+        for source in WD_OTHER_SOURCES:
+            Item.objects.filter(source=source).delete()

web/slurper/source_wikidata.py

@@ -73,9 +73,9 @@ def get_items(self):
             raw_item = BaseWdRawItem.raw_item(self.source, json_item)
             yield raw_item.to_item()
             if self.source != Item.Source.WIKIDATA:
-                raw_item.switch_source_to(Item.Source.WIKIDATA).yield_item_if_not_exists()
+                raw_item.yield_switched_if_not_exists(Item.Source.WIKIDATA)
             if raw_item.has_source(Item.Source.WIKIPEDIA_EN):
-                raw_item.switch_source_to(Item.Source.WIKIPEDIA_EN).yield_item_if_not_exists()
+                raw_item.yield_switched_if_not_exists(Item.Source.WIKIPEDIA_EN)
 
     def save_items(self):
         for item in self.get_items():
@@ -105,26 +105,20 @@ def save_links(self):
         """
   # concepts studied by an area of mathematics
   ?item wdt:P2579 ?area .
-  ?area wdt:P31 wd:Q1936384
+  ?area wdt:P31 wd:Q1936384 .
 """,
         """
   # concepts of areas of mathematics
   ?item p:P31 ?of .
   ?of ps:P31 wd:Q151885 .
-  ?of pq:P642/p:P31/ps:P31 wd:Q1936384
-""",
-        """
-  # entities with nLab and MathWorld links
-  { ?item wdt:P4215 ?nlabID . }
-  UNION
-  { ?item wdt:P2812 ?mwID . }
+  ?of pq:P642/p:P31/ps:P31 wd:Q1936384 .
 """,
     ]
 ]
 
 SLURPERS += [
     WikidataSlurper(
-        source, f"?item {source_property['wd_property']} ?{source_property['json_key']}"
+        source, f"\n  ?item {source_property['wd_property']} ?{source_property['json_key']} .\n"
     )
     for source, source_property in WD_OTHER_SOURCES.items()
 ]

web/slurper/wd_raw_item.py

@@ -63,14 +63,15 @@ def _get_item_queryset(self):
         return Item.objects.filter(source=self.source, identifier=self.identifier())
 
     def item_exists(self):
-        return self._get_item_queryset().exists
+        return self._get_item_queryset().exists()
 
     def get_item(self) -> Optional[Item]:
         return self._get_item_queryset().first()
-
-    def yield_item_if_not_exists(self):
-        if not self.item_exists():
-            yield self.to_item()
+
+    def yield_switched_if_not_exists(self, source):
+        switched = self.switch_source_to(source)
+        if not switched.item_exists():
+            yield switched.to_item()
 
     def save_link_to(self, source):
         target = self.switch_source_to(source)
@@ -113,7 +114,7 @@ def url(self):
 
     def name(self):
         if "itemLabel" in self.raw:
-            return self.wd_id
+            return self.raw["itemLabel"]["value"]
         else:
             return None