[FIX] Nomogram: Use softmax for multinomial logistic regression

Orange/widgets/visualize/ownomogram.py

@@ -2,11 +2,12 @@
 import warnings
 from itertools import chain
 from typing import List
-from functools import singledispatch
+from functools import singledispatch, partial
 from enum import IntEnum
 from collections import OrderedDict
 
 import numpy as np
+from scipy.special import logsumexp
 
 from AnyQt.QtWidgets import (
     QGraphicsView, QGraphicsScene, QGraphicsItem,
@@ -156,6 +157,8 @@ def __init__(self, radius, scale, offset, min_x, max_x):
         self._probs_dot = None
         self._vertical_line = None
         self._mousePressFunc = None
+        # called with the new value when the marker is dragged
+        self.on_moved = None
 
     @property
     def vertical_line(self):
@@ -225,6 +228,8 @@ def mouseMoveEvent(self, event):
         elif self.x() + delta_x > self._max_x:
             self.move(self._max_x)
         self._show_vertical_line_and_point_dot()
+        if self.on_moved is not None:
+            self.on_moved(self.value)
         self.probs_dot.move_to_sum()
 
     def mouseReleaseEvent(self, event):
@@ -703,7 +708,10 @@ def __init__(self):
         self.points = []
         self.feature_items = {}
         self.feature_marker_values = []
+        # points of all class values at the markers (logistic regression)
+        self.class_marker_values = None
         self.scale_marker_values = lambda x: x
+        self.unscale_marker_value = lambda _, x: x
         self.nomogram_main = None
         self.vertical_line = None
         self.hidden_vertical_line = None
@@ -854,6 +862,12 @@ def sizeHint(self):
         )
 
     def _class_combo_changed(self):
+        if self.class_marker_values is not None:
+            self.feature_marker_values = \
+                self.class_marker_values[:, self.target_class_index].copy()
+            self.update_scene()
+            self.old_target_class_index = self.target_class_index
+            return
         with np.errstate(invalid='ignore'):
             coeffs = [np.nan_to_num(p[self.target_class_index] /
                                     p[self.old_target_class_index])
@@ -871,6 +885,7 @@ def _n_spin_changed(self):
         self.update_scene()
 
     def update_controls(self):
+        is_logistic = isinstance(self.classifier, LogisticRegressionClassifier)
         self.class_combo.clear()
         self.norm_check.setHidden(True)
         self.cont_feature_dim_combo.setEnabled(True)
@@ -881,14 +896,14 @@ def update_controls(self):
                 self.target_class_index = 0
             if len(self.domain.attributes) > self.MAX_N_ATTRS:
                 self.display_index = 1
-            if len(self.domain.class_vars[0].values) > 2:
+            # softmax probabilities of logistic regression already sum to 1
+            if len(self.domain.class_vars[0].values) > 2 and not is_logistic:
                 self.norm_check.setHidden(False)
             if not self.domain.has_continuous_attributes():
                 self.cont_feature_dim_combo.setEnabled(False)
                 self.cont_feature_dim_index = 0
         model = self.sort_combo.model()
 
-        is_logistic = isinstance(self.classifier, LogisticRegressionClassifier)
         inapplicable = (SortBy.POSITIVE, SortBy.NEGATIVE)
         if is_logistic and self.sort_index in inapplicable:
             self.sort_index = SortBy.ABSOLUTE
@@ -908,6 +923,7 @@ def update_controls(self):
     def set_data(self, data):
         self.instances = data
         self.feature_marker_values = []
+        self.class_marker_values = None
         self.set_feature_marker_values()
         self.update_scene()
 
@@ -928,6 +944,7 @@ def set_classifier(self, classifier):
                          else None)
         self.points = self.log_odds_ratios or self.log_reg_coeffs
         self.feature_marker_values = []
+        self.class_marker_values = None
         self.old_target_class_index = self.target_class_index
         self.update_scene()
 
@@ -1026,26 +1043,22 @@ def text_item(text):
         minimums = [p[self.target_class_index].min() for p in self.points]
         if self.scale == OWNomogram.POINT_SCALE:
             points = [p * d for p in points]
-
-            if self.align == OWNomogram.ALIGN_LEFT:
-                self.scale_marker_values = lambda x: (x - minimums) * d
-            else:
-                self.scale_marker_values = lambda x: x * d
-        else:
-            if self.align == OWNomogram.ALIGN_LEFT:
-                self.scale_marker_values = lambda x: x - minimums
-            else:
-                self.scale_marker_values = lambda x: x
+        shift = np.array(minimums) if self.align == OWNomogram.ALIGN_LEFT \
+            else np.zeros(len(minimums))
+        scale = d if self.scale == OWNomogram.POINT_SCALE else 1
+        self.scale_marker_values = lambda x: (x - shift) * scale
+        self.unscale_marker_value = lambda i, x: x / scale + shift[i]
 
         point_item, nomogram_head = self.create_main_nomogram(
             attributes, attr_inds,
             name_items, points, max_width, point_text, name_offset)
         probs_item, nomogram_foot = self.create_footer_nomogram(
             probs_text, d, minimums, max_width, name_offset)
-        for item in self.feature_items.values():
+        for i, item in self.feature_items.items():
             item.dot.point_dot = point_item.dot
             item.dot.probs_dot = probs_item.dot
             item.dot.vertical_line = self.hidden_vertical_line
+            item.dot.on_moved = partial(self._marker_moved, i)
 
         self.dot_animator.setGraphicsItems(
             [item.dot for item in self.feature_items.values()]
@@ -1210,8 +1223,16 @@ def create_footer_nomogram(self, probs_text, d, minimums,
         # pylint: disable=invalid-unary-operand-type
         eps, d_ = 0.05, 1
         k = - np.log(self.p / (1 - self.p)) if self.p is not None else - self.b0
-        min_sum = k[self.target_class_index] - np.log((1 - eps) / eps)
-        max_sum = k[self.target_class_index] - np.log(eps / (1 - eps))
+        cls_var, cls_index = self.domain.class_var, self.target_class_index
+        min_sum = k[cls_index] - np.log((1 - eps) / eps)
+        max_sum = k[cls_index] - np.log(eps / (1 - eps))
+        # Multinomial logistic regression computes probabilities with softmax,
+        # which depends on the scores of all class values, so the ruler must
+        # also cover all totals that can be reached
+        softmax = self.b0 is not None and len(cls_var.values) > 2
+        if softmax:
+            min_sum = min(min_sum, sum(minimums))
+            max_sum = max(max_sum, sum(p[cls_index].max() for p in self.points))
         if self.align == OWNomogram.ALIGN_LEFT:
             max_sum = max_sum - sum(minimums)
             min_sum = min_sum - sum(minimums)
@@ -1226,20 +1247,33 @@ def create_footer_nomogram(self, probs_text, d, minimums,
         min_sum, max_sum = min(values), max(values)
         diff_ = np.nan_to_num(max_sum - min_sum)
         scale_x = max_width / diff_ if diff_ else max_width
-        cls_var, cls_index = self.domain.class_var, self.target_class_index
         nomogram_footer = NomogramItem()
 
+        def get_softmax_scores(val):
+            # scores (logits) of all class values at the markers;
+            # the target's score is computed from the total `val`
+            scores = self.class_marker_values.sum(axis=0) + self.b0
+            scores[cls_index] = val / d_ - k[cls_index]
+            return scores
+
         def get_normalized_probabilities(val):
+            if softmax:
+                scores = get_softmax_scores(val)
+                return np.exp(scores[cls_index] - logsumexp(scores))
             if not self.normalize_probabilities:
                 return 1 / (1 + np.exp(k[cls_index] - val / d_))
-            totals = self.__get_totals_for_class_values(minimums)
+            totals = self.__get_totals_for_class_values(minimums, d)
             p_sum = np.sum(1 / (1 + np.exp(k - totals / d_)))
             return 1 / (1 + np.exp(k[cls_index] - val / d_)) / p_sum
 
         def get_points(prob):
+            if softmax:
+                scores = np.delete(get_softmax_scores(0), cls_index)
+                return (k[cls_index] + np.log(prob / (1 - prob))
+                        + logsumexp(scores)) * d_
             if not self.normalize_probabilities:
                 return (k[cls_index] - np.log(1 / prob - 1)) * d_
-            totals = self.__get_totals_for_class_values(minimums)
+            totals = self.__get_totals_for_class_values(minimums, d)
             p_sum = np.sum(1 / (1 + np.exp(k - totals / d_)))
             return (k[cls_index] - np.log(1 / (prob * p_sum) - 1)) * d_
 
@@ -1251,7 +1285,7 @@ def get_points(prob):
         nomogram_footer.add_items([probs_item])
         return probs_item, nomogram_footer
 
-    def __get_totals_for_class_values(self, minimums):
+    def __get_totals_for_class_values(self, minimums, d):
         cls_index = self.target_class_index
         marker_values = self.scale_marker_values(self.feature_marker_values)
         totals = np.full(len(self.domain.class_var.values), np.nan)
@@ -1266,7 +1300,7 @@ def __get_totals_for_class_values(self, minimums):
             if self.align == OWNomogram.ALIGN_LEFT:
                 points = [p - m for m, p in zip(minimums, points)]
                 total -= sum([min(p) for p in [p[i] for p in self.points]])
-            d = 100 / max(max(abs(p)) for p in points)
+            # use the same scale as for the shown features
             if self.scale == OWNomogram.POINT_SCALE:
                 total *= d
             totals[i] = total
@@ -1297,6 +1331,8 @@ def _init_feature_marker_values(self):
         instances = self.instances.transform(self.domain) \
             if self.instances else None
         values = []
+        class_values = np.zeros((len(self.domain.attributes),
+                                 len(self.domain.class_var.values)))
         for i, attr in enumerate(self.domain.attributes):
             value, feature_val = 0, None
             if len(self.log_reg_coeffs):
@@ -1313,11 +1349,33 @@ def _init_feature_marker_values(self):
                 feature_val = instances[0][attr]
 
             if feature_val is not None:
-                value = (self.points[i][cls_index][int(feature_val)]
-                         if attr.is_discrete else
-                         self.log_reg_coeffs_orig[i][cls_index][0] * feature_val)
+                class_values[i] = (
+                    self.points[i][:, int(feature_val)] if attr.is_discrete
+                    else self.log_reg_coeffs_orig[i][:, 0] * feature_val)
+                value = class_values[i][cls_index]
             values.append(value)
-        self.feature_marker_values = np.asarray(values)
+        self.feature_marker_values = np.asarray(values, dtype=float)
+        if self.b0 is not None:
+            self.class_marker_values = class_values
+
+    def _marker_moved(self, attr_index, value):
+        """Store the position of a dragged marker, so that it is kept when
+        the scene is redrawn and used for the totals of other class values"""
+        cls_index = self.target_class_index
+        target_value = self.unscale_marker_value(attr_index, value)
+        self.feature_marker_values[attr_index] = target_value
+        if self.class_marker_values is None:
+            return
+        points = self.points[attr_index]
+        if np.ptp(points[cls_index]) == 0:
+            # the marker can't move; keep the points of other class values
+            return
+        # between neighbouring values (or data extremes), points of other
+        # class values change linearly with the points of the target
+        order = np.argsort(points[cls_index])
+        self.class_marker_values[attr_index] = [
+            np.interp(target_value, points[cls_index][order], p[order])
+            for p in points]
 
     def clear_scene(self):
         self.feature_items = {}

Orange/widgets/visualize/tests/test_ownomogram.py

@@ -5,7 +5,10 @@
 
 import numpy as np
 
-from AnyQt.QtCore import QPoint, QPropertyAnimation
+from AnyQt.QtCore import QPoint, QPropertyAnimation, Qt
+from AnyQt.QtTest import QTest
+
+from orangecanvas.gui.test import mouseMove
 
 from Orange.data import Table, Domain, ContinuousVariable, DiscreteVariable
 from Orange.classification import (
@@ -98,7 +101,129 @@ def test_nomogram_lr_multiclass(self):
         """Check probabilities for logistic regression classifier for various
         values of classes and radio buttons for multiclass data"""
         cls = LogisticRegressionLearner(max_iter=100)(self.lenses)
-        self._test_helper(cls, [18, 56, 78])
+        self._test_helper(cls, [4, 25, 70])
+
+    def _shown_probability(self):
+        dot = [item for item in self.widget.scene.items()
+               if isinstance(item, ProbabilitiesDotItem)][0]
+        ruler = dot.parentItem()
+        total = self.widget.scale_marker_values(
+            self.widget.feature_marker_values).sum()
+        # the dot is clipped to the ruler, so the ruler must cover the total
+        self.assertLessEqual(ruler.min_val, total)
+        self.assertGreaterEqual(ruler.max_val, total)
+        self.assertAlmostEqual(dot.value, total)
+        return dot.get_probabilities(dot.value)
+
+    def _check_lr_probabilities(self, cls, data, indices):
+        """Check that the nomogram shows the model's probabilities for all
+        target classes, both scales and any number of shown features"""
+        w = self.widget
+        # the ruler must be wide enough to show the totals
+        w.resize(1000, 600)
+        w.show()
+        self.send_signal(w.Inputs.classifier, cls)
+        # sort alphabetically, so that the most important feature is hidden
+        simulate.combobox_activate_index(w.sort_combo, SortBy.NAME)
+        n_attrs = len(data.domain.attributes)
+        for i in indices:
+            self.send_signal(w.Inputs.data, data[i:i + 1])
+            probs = cls(data[i], cls.Probs)
+            for scale in (0, 1):
+                w.controls.scale.buttons[scale].click()
+                for n in range(1, n_attrs + 1):
+                    w.n_spin.setValue(n)
+                    for cls_index, prob in enumerate(probs):
+                        simulate.combobox_activate_index(w.class_combo,
+                                                         cls_index)
+                        self.assertAlmostEqual(self._shown_probability(), prob)
+
+    def test_probabilities_lr_multiclass(self):
+        """Check that probabilities for multinomial logistic regression
+        match the model's predictions"""
+        data = Table("iris")
+        cls = LogisticRegressionLearner()(data)
+        self._check_lr_probabilities(cls, data, (0, 53, 120))
+        # softmax probabilities already sum to 1
+        self.assertTrue(self.widget.norm_check.isHidden())
+
+    def test_probabilities_lr_multiclass_zero_coefficients(self):
+        """Check probabilities for multinomial logistic regression when some
+        or all coefficients of class values are zero"""
+        data = Table("iris")
+        # C=0.05: some coefficients of setosa and virginica are zero, all
+        # coefficients of versicolor are zero; C=0.01: all coefficients of
+        # versicolor and virginica are zero; C=0.004: all coefficients are zero
+        for c in (0.05, 0.01, 0.004):
+            cls = LogisticRegressionLearner(penalty="l1", C=c)(data)
+            self._check_lr_probabilities(cls, data, (0, 53))
+
+    def _drag_dot(self, dot, value):
+        """Drag a feature marker with the mouse to the given value (in
+        displayed points)"""
+        viewport = self.widget.view.viewport()
+        start = self.widget.view.mapFromScene(dot.sceneBoundingRect().center())
+        end = start + QPoint(round((value - dot.value) * dot._scale), 0)
+        QTest.mousePress(viewport, Qt.LeftButton, Qt.NoModifier, start)
+        mouseMove(viewport, Qt.LeftButton, pos=end)
+        QTest.mouseRelease(viewport, Qt.LeftButton, Qt.NoModifier, end)
+
+    def test_probabilities_lr_multiclass_dragged_marker(self):
+        """Check that probabilities for multinomial logistic regression match
+        the model's predictions after a feature marker is dragged"""
+        w = self.widget
+        w.resize(1000, 600)
+        w.show()
+        data = Table("iris")
+        cls = LogisticRegressionLearner()(data)
+        self.send_signal(w.Inputs.classifier, cls)
+        self.send_signal(w.Inputs.data, data[:1])
+        w.controls.scale.buttons[0].click()
+        simulate.combobox_activate_index(w.class_combo, 1)
+
+        # drag the marker of petal length from 1.4 towards 5
+        attr_index = data.domain.index("petal length")
+        coef = w.log_reg_coeffs_orig[attr_index][1][0]
+        values = np.array(w.feature_marker_values, dtype=float)
+        start = w.scale_marker_values(values)[attr_index]
+        d = w.scale_marker_values(values + 1)[attr_index] - start
+        dot = w.feature_items[attr_index].dot
+        self._drag_dot(dot, start + (5 - 1.4) * coef * d)
+        # the value of petal length at the marker's new position
+        new_value = (values[attr_index] + (dot.value - start) / d) / coef
+        self.assertGreater(new_value, 4)
+
+        inst = data[:1].copy()
+        with inst.unlocked(inst.X):
+            inst.X[0, attr_index] = new_value
+        probs = cls(inst[0], cls.Probs)
+        self.assertAlmostEqual(self._shown_probability(), probs[1])
+
+        # the dragged marker is kept when the target or the scale change
+        for scale in (1, 0):
+            w.controls.scale.buttons[scale].click()
+            for cls_index, prob in enumerate(probs):
+                simulate.combobox_activate_index(w.class_combo, cls_index)
+                self.assertAlmostEqual(self._shown_probability(), prob)
+
+    def test_normalized_probabilities_hidden_features(self):
+        """Normalized probabilities must not depend on the number of shown
+        features"""
+        w = self.widget
+        w.resize(1000, 600)
+        w.show()
+        cls = NaiveBayesLearner()(self.lenses)
+        self.send_signal(w.Inputs.classifier, cls)
+        w.controls.normalize_probabilities.setChecked(True)
+        simulate.combobox_activate_index(w.sort_combo, SortBy.NAME)
+        w.controls.scale.buttons[0].click()
+        for cls_index in range(len(self.lenses.domain.class_var.values)):
+            simulate.combobox_activate_index(w.class_combo, cls_index)
+            w.n_spin.setValue(4)
+            expected = self._shown_probability()
+            for n in (1, 2, 3):
+                w.n_spin.setValue(n)
+                self.assertAlmostEqual(self._shown_probability(), expected)
 
     def test_nomogram_with_instance_nb(self):
         """Check initialized marker values and feature sorting for naive bayes