feat(report): group comparisons by reference key (#237)

Co-authored-by: Patil <[email protected]>
Co-authored-by: Simon Brugman <[email protected]>

popmon/analysis/comparison/comparisons.py

@@ -28,7 +28,7 @@
 
 @Comparisons.register(
     key="max_prob_diff",
-    description="The largest absolute difference between all bin pairs of two normalized histograms (one histogram in a time slot and one in {ref})",
+    description="The largest absolute difference between all bin pairs of two normalized histograms",
     htype="all",
 )
 def googl_test(bins_1, bins_2):
@@ -147,9 +147,9 @@ def ks_prob(testscore):
 @Comparisons.register(
     key=["ks", "ks_pvalue", "ks_zscore"],
     description=[
-        "Kolmogorov-Smirnov test statistic comparing each time slot to {ref}",
-        "p-value of the Kolmogorov-Smirnov test, comparing each time slot with {ref}",
-        "Z-score of the Kolmogorov-Smirnov test, comparing each time slot with {ref}",
+        "Kolmogorov-Smirnov test statistic",
+        "p-value of the Kolmogorov-Smirnov test",
+        "Z-score of the Kolmogorov-Smirnov test",
     ],
     dim=1,
     htype="num",
@@ -164,7 +164,7 @@ def ks(p, q, *args):
 
 @Comparisons.register(
     key="unknown_labels",
-    description="Are categories observed in a given time slot that are not present in {ref}?",
+    description="Are categories observed in a given time slot that are not present in the reference?",
     dim=1,
     htype="cat",
 )
@@ -178,7 +178,7 @@ def unknown_labels(hist1, hist2):
 
 @Comparisons.register(
     key="pearson",
-    description="Pearson correlation between each time slot and {ref}",
+    description="Pearson correlation coefficient",
     dim=(2,),
     htype="all",
 )
@@ -259,14 +259,12 @@ def _not_finite_to_zero(x):
         "chi2_spike_count",
     ],
     description=[
-        "Chi-squared test statistic, comparing each time slot with {ref}",
-        "Normalized chi-squared statistic, comparing each time slot with {ref}",
-        "Z-score of the chi-squared statistic, comparing each time slot with {ref}",
-        "p-value of the chi-squared statistic, comparing each time slot with {ref}",
-        "The largest absolute normalized residual (|chi|) observed in all bin pairs "
-        + "(one histogram in a time slot and one in {ref})",
-        "The number of normalized residuals of all bin pairs (one histogram in a time"
-        + " slot and one in {ref}) with absolute value bigger than a given threshold (default: 7).",
+        "Chi-squared test statistic",
+        "Normalized chi-squared statistic",
+        "Z-score of the chi-squared statistic",
+        "p-value of the chi-squared statistic",
+        "The largest absolute normalized residual (|chi|) observed in all bin pairs",
+        "The number of normalized residuals of all bin pairs with absolute value bigger than a given threshold (default: 7).",
     ],
     htype="all",
 )

popmon/notebooks/popmon_tutorial_reports.ipynb

@@ -355,7 +355,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "show_image(report.datastore[\"report_sections\"][4][\"features\"][0][\"plots\"][0])"
+    "show_image(report.datastore[\"report_sections\"][4][\"features\"][0][\"plots\"][\"prev1\"][0])"
    ]
   },
   {

popmon/visualization/histogram_section.py

@@ -159,14 +159,15 @@ def transform(self, data_obj: dict, sections: Optional[list] = None):
                 plot_type_layouts["heatmap"] = hplots[0]["layout"]
 
             plots = hplots + plots
-            # print(plot_types,layouts)
+
             features_w_metrics.append(
                 {
                     "name": feature,
                     "plot_type_layouts": plot_type_layouts,
                     "plots": plots,
                 }
             )
+
         sections.append(
             {
                 "section_title": self.section_name,

popmon/visualization/section_generator.py

@@ -16,8 +16,7 @@
 # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
 # IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
-
+from collections import defaultdict
 from typing import Optional
 
 import numpy as np
@@ -37,12 +36,22 @@
 comparisons = Comparisons.get_descriptions()
 
 
+group_titles = {
+    "prev1": "Previous Reference",
+    "rolling": "Rolling Reference",
+    "self": "Self-Reference",
+    "ref": "External Reference",
+    "expanding": "Expanding Reference",
+}
 references = {
     "ref": "the reference data",
     "roll": "a rolling window",
     "prev1": "the preceding time slot",
     "expanding": "all preceding time slots",
 }
+group_descriptions = {
+    key: f"Comparing each time slot to {value}." for key, value in references.items()
+}
 
 
 def get_stat_description(name: str):
@@ -65,7 +74,7 @@ def get_stat_description(name: str):
     tail = "_".join(tail)
 
     if tail in comparisons and head in references:
-        return comparisons[tail].format(ref=references[head])
+        return comparisons[tail]
 
     return ""
 
@@ -126,7 +135,6 @@ def __init__(
         self.skip_empty_plots = settings.skip_empty_plots
         self.description = description
         self.show_stats = settings.show_stats if not settings.extended_report else None
-
         self.primary_color = settings.primary_color
         self.tl_colors = settings.tl_colors
 
@@ -205,13 +213,32 @@ def transform(
                 if "range" in layouts["yaxis"]:
                     del layouts["yaxis"]["range"]
 
-            features_w_metrics.append(
-                {
-                    "name": feature,
-                    "plot_type_layouts": {"barplot": layouts},
-                    "plots": sorted(plots, key=lambda plot: plot["name"]),
-                }
-            )
+            # Group comparisons in Comparison section
+            if self.read_key == "comparisons":
+                grouped_metrics = defaultdict(list)
+                for plot in plots:
+                    prefix = plot["name"].split("_")[0]
+                    if prefix not in references:
+                        prefix = "Others"
+                    grouped_metrics[prefix].append(plot)
+
+                features_w_metrics.append(
+                    {
+                        "name": feature,
+                        "plot_type_layouts": {"barplot": layouts},
+                        "plots": grouped_metrics,
+                        "titles": group_titles,
+                        "descriptions": group_descriptions,
+                    }
+                )
+            else:
+                features_w_metrics.append(
+                    {
+                        "name": feature,
+                        "plot_type_layouts": {"barplot": layouts},
+                        "plots": plots,
+                    }
+                )
 
         sections.append(
             {

popmon/visualization/templates/section.html

@@ -29,13 +29,31 @@ <h2>{{ section_title }}</h2>
             feature{{ section_index }}{{ curr }}_layout["{{ plot_type }}"] = {{ layout | json_plot }};
         {%- endfor -%}
         </script>
-        <div class="row section_feature" data-section-feature="{{ feature.name }}">
+        <div class="section_feature" data-section-feature="{{ feature.name }}">
+          {%- if feature.plots is mapping -%}
+            {%- for ref,plots in feature.plots.items() -%}
+              {%- set count = (plots|length + 1)*(loop.index - 1) -%}
+              <h3> {{ feature.titles.get(ref, ref) }} </h3>
+              <div class="section-description">{{ feature.descriptions[ref] }}</div>
+              <div class="row">
+              {%- for metric in plots -%}
+                {%- set plt = count + loop.index  -%}
+                {%- with metric=metric -%}
+                  {%- include 'card.html' -%}
+                {%- endwith -%}
+              {%- endfor -%}
+              </div>
+            {%- endfor -%}
+          {%- else -%}
+            <div class="row">
             {%- for metric in feature.plots -%}
               {%- set plt = loop.index  -%}
               {%- with metric=metric -%}
                 {%- include 'card.html' -%}
               {%- endwith -%}
             {%- endfor -%}
+            </div>
+          {%- endif-%}
         </div>
       {%- endfor -%}
     {%- else -%}