Merge pull request #1059 from lrzpellegrini/master

Add class accuracy/AMCA to doc. Enable methods doc generation.

avalanche/evaluation/metrics/amca.py

@@ -46,11 +46,13 @@ class AverageMeanClassAccuracy(Metric[Dict[int, float]]):
     Each time `result` is called, this metric emits the average mean accuracy
     as the average accuracy of all previous experiences (also considering the
     accuracy in the current experience).
-    The metric expects that the `next_train_experience()` method will be called
+    The metric expects that the :meth:`next_experience` method will be called
     after each experience. This is needed to consolidate the current mean
-    accuracy. After calling `next_train_experience()`, a new experience with
+    accuracy. After calling :meth:`next_experience`, a new experience with
     accuracy 0.0 is immediately started. If you need to obtain the AMCA up to
-    experience `t-1`, obtain the `result()` before calling `next_experience()`.
+    experience `t-1`, obtain the :meth:`result` before calling
+    :meth:`next_experience`.
+
 
     The set of classes to be tracked can be reduced (please refer to the
     constructor parameters).
@@ -305,12 +307,29 @@ def _is_stream_tracked(self, stream_name):
 
 class AMCAPluginMetric(_ExtendedGenericPluginMetric):
     """
-    Base class for all class accuracy plugin metrics
+    Plugin metric for the Average Mean Class Accuracy (AMCA).
+
+    The AMCA is tracked for the classes and streams defined in the constructor.
+
+    In addition, by default, the results obtained through the periodic
+    evaluation (mid-training validation) mechanism are ignored.
     """
 
     VALUE_NAME = '{metric_name}/{stream_name}_stream/Task{task_label:03}'
 
     def __init__(self, classes=None, streams=None, ignore_validation=True):
+        """
+        Instantiates the AMCA plugin metric.
+
+        :param classes: The classes to track. Refer to :class:`MultiStreamAMCA`
+            for more details.
+        :param streams: The streams to track. Defaults to None, which means that
+            all streams will be considered. Beware that, when creating instances
+            of this class using the :func:`amca_metrics` helper, the resulting
+            metric will only track the "test" stream by default.
+        :param ignore_validation: Defaults to True, which means that periodic
+            evaluations will be ignored (recommended).
+        """
         self._ms_amca = MultiStreamAMCA(classes=classes, streams=streams)
         self._ignore_validation = ignore_validation
 

docs/Makefile

@@ -2,7 +2,7 @@
 #
 
 # You can set these variables from the command line.
-SPHINXOPTS    =
+SPHINXOPTS    = 
 SPHINXBUILD   = sphinx-build
 SOURCEDIR     = .
 BUILDDIR      = _build
@@ -16,4 +16,4 @@ help:
 # Catch-all target: route all unknown targets to Sphinx using the new
 # "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
 %: Makefile
-	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/_templates/autosummary/no-attributes-class.rst → docs/_templates/autosummary/documented-methods-class.rst

@@ -6,8 +6,6 @@
 
 {{ fullname | escape | underline}}
 
-This is my custom template.
-
 .. currentmodule:: {{ module }}
 
 .. autoclass:: {{ objname }}
@@ -19,8 +17,21 @@ This is my custom template.
    .. rubric:: {{ _('Methods') }}
 
    .. autosummary::
+      :toctree:
    {% for item in methods %}
       ~{{ name }}.{{ item }}
    {%- endfor %}
    {% endif %}
    {% endblock %}
+
+   {% block attributes %}
+   {% if attributes %}
+   .. rubric:: {{ _('Attributes') }}
+
+   .. autosummary::
+   {% for item in attributes %}
+      ~{{ name }}.{{ item }}
+   {%- endfor %}
+   {% endif %}
+   {% endblock %}
+

docs/evaluation.rst

@@ -24,6 +24,8 @@ Metrics helper functions
     :toctree: generated
 
     accuracy_metrics
+    class_accuracy_metrics
+    amca_metrics
     topk_acc_metrics
     loss_metrics
     bwt_metrics
@@ -51,6 +53,8 @@ Stream Metrics
    :toctree: generated
 
     StreamAccuracy
+    StreamClassAccuracy
+    AMCAPluginMetric
     TrainedExperienceAccuracy
     StreamLoss
     StreamBWT
@@ -75,6 +79,7 @@ Experience Metrics
    :toctree: generated
 
     ExperienceAccuracy
+    ExperienceClassAccuracy
     ExperienceLoss
     ExperienceBWT
     ExperienceForgetting
@@ -98,6 +103,7 @@ Epoch Metrics
    :toctree: generated
 
     EpochAccuracy
+    EpochClassAccuracy
     EpochLoss
     EpochCPUUsage
     EpochDiskUsage
@@ -117,6 +123,7 @@ RunningEpoch Metrics
    :toctree: generated
 
     RunningEpochAccuracy
+    RunningEpochClassAccuracy
     RunningEpochTopkAccuracy
     RunningEpochLoss
     RunningEpochCPUUsage
@@ -131,6 +138,7 @@ Minibatch Metrics
    :toctree: generated
 
     MinibatchAccuracy
+    MinibatchClassAccuracy
     MinibatchLoss
     MinibatchCPUUsage
     MinibatchDiskUsage
@@ -155,25 +163,28 @@ Standalone Metrics
 .. autosummary::
    :toctree: generated
 
-    MeanNewOldScores
-    MaxRAM
-    MAC
+    Accuracy
+    AverageMeanClassAccuracy
     BWT
-    Mean
-    DiskUsage
     CPUUsage
-    Loss
+    ClassAccuracy
     ConfusionMatrix
-    ForwardTransfer
-    Forgetting
+    DiskUsage
     ElapsedTime
+    Forgetting
+    ForwardTransfer
     LabelsRepartition
+    Loss
+    MAC
     MaxGPU
-    TopkAccuracy
-    TrainedExperienceTopkAccuracy
-    Accuracy
+    MaxRAM
+    Mean
+    MeanNewOldScores
     MeanScores
+    MultiStreamAMCA
     Sum
+    TopkAccuracy
+    TrainedExperienceTopkAccuracy
 
 
 

docs/training.rst

@@ -29,7 +29,7 @@ Templates are defined in the `avalanche.training.templates` module.
     BaseTemplate
     BaseSGDTemplate
     SupervisedTemplate
-    SupervisedOnlineTemplate
+    OnlineSupervisedTemplate
 
 
 Plugins ABCs