Codec-Compare version 0.1.8 (#1)

Add CHANGELOG.md

CHANGELOG.md

@@ -0,0 +1,17 @@
+# Changelog
+
+## v0.1.8
+
+- Add matcher warning display capability.
+- Add metric warning display capability.
+- Replace the Settings page by a Settings section in the left menu.
+- Fix various scroll bars.
+- Hide the bits-per-pixel matcher.
+- Hide the bits-per-pixel metric.
+- Pick two objective quality metrics by default.
+- The two plot axis can be set directly.
+- Fit the image to the viewport in the visualizer.
+
+## v0.1.2
+
+Initial release of the Codec-Compare static front-end framework.

assets/demo_batch_some_codec_exp.json

@@ -25,31 +25,35 @@
     { "effort": "Compression effort parameter" },
     { "quality": "Compression quality parameter" },
     {
-      "psnr": "Quality metric Peak Signal-to-Noise Ratio. See https://en.wikipedia.org/wiki/Peak_signal-to-noise_ratio"
+      "psnr": "Quality metric Peak Signal-to-Noise Ratio. See https://en.wikipedia.org/wiki/Peak_signal-to-noise_ratio. Warning: There is no scientific consensus on which objective quality metric to use."
     },
     {
-      "ssim": "Quality metric Structural Similarity Index Measure. See https://en.wikipedia.org/wiki/Structural_similarity"
+      "ssim": "Quality metric Structural Similarity Index Measure. See https://en.wikipedia.org/wiki/Structural_similarity. Warning: There is no scientific consensus on which objective quality metric to use."
     },
     {
-      "msssim": "Quality metric Multi-Scale Structural Similarity Index Measure. See https://en.wikipedia.org/wiki/Structural_similarity#Multi-Scale_SSIM"
+      "msssim": "Quality metric Multi-Scale Structural Similarity Index Measure. See https://en.wikipedia.org/wiki/Structural_similarity#Multi-Scale_SSIM. Warning: There is no scientific consensus on which objective quality metric to use."
     },
     {
-      "butteraugli": "Quality metric Butteraugli. See https://en.wikipedia.org/wiki/Guetzli#Butteraugli"
+      "butteraugli": "Quality metric Butteraugli. See https://en.wikipedia.org/wiki/Guetzli#Butteraugli. Warning: There is no scientific consensus on which objective quality metric to use."
     },
     {
-      "ciede2000": "Quality metric CIEDE 2000. See https://en.wikipedia.org/wiki/Color_difference#CIEDE2000"
+      "ciede2000": "Quality metric CIEDE 2000. See https://en.wikipedia.org/wiki/Color_difference#CIEDE2000. Warning: There is no scientific consensus on which objective quality metric to use."
     },
-    { "encoding_time": "Encoding duration in seconds (approximate)" },
-    { "decoding_time": "Decoding duration in seconds (approximate)" },
     {
-      "ssimulacra": "Quality metric SSIMULACRA. See https://github.com/cloudinary/ssimulacra"
+      "encoding_time": "Encoding duration in seconds. Warning: Timings are environment-dependent and inaccurate."
+    },
+    {
+      "decoding_time": "Decoding duration in seconds. Warning: Timings are environment-dependent and inaccurate."
+    },
+    {
+      "ssimulacra": "Quality metric SSIMULACRA. See https://github.com/cloudinary/ssimulacra. Warning: There is no scientific consensus on which objective quality metric to use."
     },
     { "encoded_size": "Size of the encoded image file in bytes" },
     {
-      "lpips": "Learned Perceptual Image Patch Similarity. See https://arxiv.org/abs/1801.03924"
+      "lpips": "Learned Perceptual Image Patch Similarity. See https://arxiv.org/abs/1801.03924. Warning: There is no scientific consensus on which objective quality metric to use."
     },
     {
-      "flip": "Difference evaluator for alternating images. See https://research.nvidia.com/publication/2020-07_lip-difference-evaluator-alternating-images"
+      "flip": "Difference evaluator for alternating images. See https://research.nvidia.com/publication/2020-07_lip-difference-evaluator-alternating-images. Warning: There is no scientific consensus on which objective quality metric to use."
     }
   ],
   "field_values": [

package.json

@@ -1,6 +1,6 @@
 {
   "name": "codec_compare",
-  "version": "0.1.2",
+  "version": "0.1.8",
   "description": "Codec performance comparison tool",
   "publisher": "Google LLC",
   "author": "Yannis Guyon",

src/codec_compare.ts

@@ -110,12 +110,12 @@ export class CodecCompare extends LitElement {
             <mwc-icon>share</mwc-icon> Copy URL to clipboard
           </mwc-button-fit>
 
-          <mwc-button-fit @click=${() => {
-      this.settingsUi.onOpen();
-    }}>
+          <mwc-button-fit disabled>
             <mwc-icon>settings</mwc-icon> Settings
           </mwc-button-fit>
 
+          <settings-ui .state=${this.state}></settings-ui>
+
           <mwc-button-fit id="helpButton" @click=${() => {
       this.helpUi.onOpen();
     }}>
@@ -143,7 +143,7 @@ export class CodecCompare extends LitElement {
           </p>
 
           <p id="credits">
-            Codec Compare beta version 0.1.2<br>
+            Codec Compare beta version 0.1.8<br>
             <a href="https://github.com/webmproject/codec-compare">
               Sources on GitHub
             </a>
@@ -155,7 +155,6 @@ export class CodecCompare extends LitElement {
       <batch-selection-ui .state=${this.state}></batch-selection-ui>
       <matches-ui .state=${this.state}></matches-ui>
       <match-ui .state=${this.state}></match-ui>
-      <settings-ui .state=${this.state}></settings-ui>
       <help-ui></help-ui>
       ${this.isLoaded ? html`` : html`<loading-ui></loading-ui>`}
     `;
@@ -184,6 +183,8 @@ export class CodecCompare extends LitElement {
 
   private async onAllBatchesLoaded() {
     this.state.initialize();
+    this.urlState.setDefaultValues(this.state);
+    this.state.initializePostUrlStateDefaultValues();
     this.urlState.load(this.state);
     this.state.initializePostUrlStateLoad();
     listen(EventType.MATCHED_DATA_POINTS_CHANGED, () => {
@@ -259,7 +260,11 @@ export class CodecCompare extends LitElement {
       flex-direction: column;
       gap: 12px;
       padding: 0px 20px 0px 74px;
-      overflow: auto;
+      overflow-y: auto;
+      overflow-x: hidden;
+    }
+    batch-selections-ui {
+      overflow-x: auto;
     }
     #truncatedResults {
       background: orange;

src/entry.ts

@@ -108,13 +108,13 @@ const NAME_TO_FIELD_ID = new Map<string, FieldId>([
  * by decreasing preference of which matcher is enabled by default.
  */
 export const QUALITY_METRIC_FIELD_IDS = [
-  FieldId.SSIM, FieldId.MSSSIM, FieldId.PSNR, FieldId.BUTTERAUGLI,
-  FieldId.SSIMULACRA2, FieldId.SSIMULACRA, FieldId.CIEDE2000, FieldId.FLIP,
+  FieldId.SSIM, FieldId.SSIMULACRA2, FieldId.MSSSIM, FieldId.PSNR,
+  FieldId.BUTTERAUGLI, FieldId.SSIMULACRA, FieldId.CIEDE2000, FieldId.FLIP,
   FieldId.LPIPS, FieldId.P3NORM
 ];
 
 function fieldNameToFieldId(name: string): FieldId {
-  return NAME_TO_FIELD_ID.get(name.toLowerCase().replace('_', ' ')) ??
+  return NAME_TO_FIELD_ID.get(name.toLowerCase().replaceAll('_', ' ')) ??
       FieldId.CUSTOM;
 }
 
@@ -131,7 +131,7 @@ function fieldPrettyName(id: FieldId, name: string): string {
     }
   }
   // Custom properties are displayed too.
-  return name.replace('_', ' ');
+  return name.replaceAll('_', ' ');
 }
 
 /** Data column in a Batch. For example: source_image, encoding_time etc. */

src/help_ui.ts

@@ -151,13 +151,13 @@ export class HelpUi extends LitElement {
         others against.<br>
         The statistics relative to the reference batch for the fields selected
         as metrics are displayed in the right-most columns. The aggregation
-        method can be changed in the Settings page.
+        method can be changed in the Settings.
         </p>
       </div>
 
       <div class="descriptionHolder" id="graphDescription">
         <p>
-        The codecs are plotted on this graph as large circles, with the metric
+        The codecs are plotted on this graph as large disks, with the metric
         fields as axes. If any, batches sharing the same codec are linked with
         straight lines, usually to represent multiple encoding efforts.<br>
         If enabled in the Settings page, each matched pair is displayed as a

src/match_image_ui.ts

@@ -88,7 +88,7 @@ export class MatchImageUi extends LitElement {
 
   static override styles = css`
     :host {
-      overflow: auto;
+      overflow: hidden;
       display: flex;
       justify-content: center;
     }

src/match_ui.ts

@@ -193,7 +193,7 @@ export class MatchUi extends LitElement {
       /* Rely on margin:auto for distributing the space. */
       justify-content: flex-start;
       gap: 20px;
-      overflow: hidden;
+      overflow: auto;
     }
 
     #closeButton {
@@ -220,5 +220,8 @@ export class MatchUi extends LitElement {
     constants-table-ui {
       max-width: 50%;
     }
+    match-image-ui {
+      min-height: 78px;  /* 64px + 7px*2 margin as in match_image_ui.ts */
+    }
   `;
 }

src/matcher.ts

@@ -100,6 +100,9 @@ export function createMatchers(batches: Batch[]): FieldMatcher[] {
     if (field.id === FieldId.DECODED_IMAGE_NAME) continue;
     // Same source image, so these will always match. Remove them from the UI.
     if (field.id === FieldId.WIDTH || field.id === FieldId.HEIGHT) continue;
+    // If bpp values are available, encoded sizes probably are too.
+    // Skip the former which brings nothing as a matcher over the latter.
+    if (field.id === FieldId.ENCODED_BITS_PER_PIXEL) continue;
 
     // Try to guess a good default tolerance.
     let defaultTolerance = 0;
@@ -140,15 +143,18 @@ export function enableDefaultMatchers(
   }
 
   // Find any quality metric suggesting that it is not a lossless comparison and
-  // use it as a match criterion.
+  // use it as a match criterion. To be somewhat fairer, pick the first two in
+  // the QUALITY_METRIC_FIELD_IDS order, at the expense of fewer data points.
+  let numQualityMetrics = 0;
   for (const id of QUALITY_METRIC_FIELD_IDS) {
     const qualityMatcher = matchers.find(
         (matcher) => firstBatch.fields[matcher.fieldIndices[0]].id === id);
     if (qualityMatcher !== undefined) {
       const qualityField = firstBatch.fields[qualityMatcher.fieldIndices[0]];
       if (qualityField.isNumber && qualityField.uniqueValuesArray.length > 1) {
         qualityMatcher.enabled = true;
-        break;
+        ++numQualityMetrics;
+        if (numQualityMetrics === 2) break;
       }
     }
   }

src/matcher_ui.ts

@@ -14,6 +14,7 @@
 
 import '@material/mwc-menu';
 import './mwc_button_fit';
+import './tooltip_ui';
 
 import {Button} from '@material/mwc-button';
 import {Menu} from '@material/mwc-menu';
@@ -104,12 +105,18 @@ export class MatcherUi extends LitElement {
   }
 
   override render() {
+    const tokens = this.description.split('Warning:', /*limit=*/ 2);
+    const description = tokens[0].trim();
+    const tooltip = tokens.length === 1 ?
+        '' :
+        html` <tooltip-ui icon="warning" .text="${
+            tokens[1].trim()}"></tooltip-ui>`;
     return html`
       ${this.renderDeleteButton()}
       <p>
         ${this.isFirst ? 'for' : ''} the same
-        <strong title="${this.description}">${this.displayName}</strong>${
-        this.renderToleranceButton()}${this.isLast ? ',' : ' and'}
+        <strong title="${description}">${this.displayName}</strong>${
+        this.renderToleranceButton()}${tooltip}${this.isLast ? ',' : ' and'}
       </p>`;
   }
 

src/metric.ts

@@ -72,6 +72,9 @@ export function createMetrics(batches: Batch[]): FieldMetric[] {
     if (field.id === FieldId.WIDTH || field.id === FieldId.HEIGHT) continue;
     // Encoder settings should not be compared.
     if (field.id === FieldId.EFFORT || field.id === FieldId.QUALITY) continue;
+    // If bpp values are available, encoded sizes probably are too.
+    // Skip the former which brings nothing as a metric over the latter.
+    if (field.id === FieldId.ENCODED_BITS_PER_PIXEL) continue;
 
     metrics.push(new FieldMetric(fieldIndices));
   }

src/metric_ui.ts

@@ -14,6 +14,7 @@
 
 import '@material/mwc-icon';
 import './mwc_button_fit';
+import './tooltip_ui';
 
 import {css, html, LitElement} from 'lit';
 import {customElement, property} from 'lit/decorators.js';
@@ -53,13 +54,19 @@ export class MetricUi extends LitElement {
   }
 
   override render() {
-    const description = html`
+    const tokens = this.description.split('Warning:', /*limit=*/ 2);
+    const description = tokens[0].trim();
+    const tooltip = tokens.length === 1 ?
+        '' :
+        html` <tooltip-ui icon="warning" .text="${
+            tokens[1].trim()}"></tooltip-ui>`;
+    const text = html`
         the <span title="geometric mean">average</span>
-      <strong title="${this.description}">
+      <strong title="${description}">
         ${this.displayName}
-      </strong>${this.isLast ? '' : ' and'}`;
+      </strong>${tooltip}${this.isLast ? '' : ' and'}`;
 
-    return html`${this.renderDeleteButton()} <p>${description}</p>`;
+    return html`${this.renderDeleteButton()} <p>${text}</p>`;
   }
 
   static override styles = css`

src/settings_ui.ts

@@ -29,29 +29,10 @@ import {State} from './state';
 export class SettingsUi extends LitElement {
   @property({attribute: false}) state!: State;
 
-  /** Called when this component is made visible. */
-  onOpen() {
-    this.style.display = 'block';
-    this.requestUpdate();
-  }
-
   override render() {
     if (!this.state) return html``;
 
-    const onClose = () => {
-      this.style.display = 'none';
-      this.requestUpdate();
-    };
-
     return html`
-      <div id="background" @click=${onClose}>
-      </div>
-      <div id="dialog" @click=${(e: Event) => {
-      e.stopImmediatePropagation();
-    }}>
-        <mwc-fab id="closeButton" icon="close" title="Close" @click=${onClose}>
-        </mwc-fab>
-        <h2>Settings</h2>
         <div class="settingGroup">
           <span>Hide data points</span>
           <mwc-switch ?selected=${this.state.showEachMatch}
@@ -73,53 +54,20 @@ export class SettingsUi extends LitElement {
     }}>
           </mwc-switch>
           <span>Geometric mean</span>
-        </div>
-      </div>`;
+        </div>`;
   }
 
   static override styles = css`
     :host {
-      display: none;
-      position: absolute;
-      z-index: 8;
-      left: 0;
-      top: 0;
-      bottom: 0;
-      right: 0;
-    }
-
-    #background {
-      width: 100%;
-      height: 100%;
-      background-color: rgba(0, 0, 0, 0.6);
-    }
-
-    #dialog {
-      background-color: var(--mdc-theme-surface);
-      position: absolute;
-      left: 40px;
-      top: 40px;
-      bottom: 40px;
-      width: 520px;
-      padding: 20px;
-      box-shadow: 0 0 8px 0 rgba(0, 0, 0, 0.2);
-      border-radius: 16px;
+      margin-left: 70px;
       display: flex;
       flex-direction: column;
-      /* Rely on margin:auto for distributing the space. */
-      justify-content: flex-start;
       gap: 20px;
-      overflow: hidden;
-    }
-
-    #closeButton {
-      margin-left: auto;
     }
 
     .settingGroup {
       display: flex;
       gap: 20px;
-      justify-content: center;
     }
   `;
 }