Table component (#38)

* Basic table component with error instances

* Send index select indices to appropriate device

* Remove un-necessary move to device

* kRender instance image data within table component

* Add pagination to error instances table

* Update all the example Notebooks to use a dataset with associated ids

* Fix the dev environment and update the docstrings regarding the use of data instance ids

* Update docstrings for `get_error_instance_indices`

* Use user provied funtion to display images

* Fix pagination in error instances table

backwardcompatibilityml/helpers/http.py

@@ -0,0 +1,19 @@
+from flask.views import View, MethodView
+from flask import send_from_directory, abort, make_response, request
+import os.path as ospath
+from functools import wraps
+from werkzeug.exceptions import HTTPException
+from werkzeug.wrappers import Response
+import json
+
+
+def no_cache(f):
+    @wraps(f)
+    def decorated_function(*args, **kwargs):
+        response = make_response(f(*args, **kwargs))
+        response.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate'
+        response.headers['Pragma'] = 'no-cache'
+        response.headers['Expires'] = '-1'
+        return response
+
+    return decorated_function

backwardcompatibilityml/helpers/training.py

@@ -22,7 +22,7 @@ def train_epoch(epoch, network, optimizer, loss_function, training_set, batch_si
         network: The model which is undergoing training.
         optimizer: The optimizer instance to use for training.
         loss_function: An instance of the loss function to use for training.
-        training_set: The list of training samples as (input, target) pairs.
+        training_set: The list of training samples as (batch_ids, input, target).
         batch_size_train: An integer representing batch size of the training set.
         device: A string with values either "cpu" or "cuda" to indicate the
             device that Pytorch is performing training on. By default this
@@ -67,7 +67,7 @@ def test(network, loss_function, test_set, batch_size_test, device="cpu"):
     Args:
         network: The model which is undergoing testing.
         loss_function: An instance of the loss function to use for training.
-        test_set: The list of testing samples as (input, target) pairs.
+        test_set: The list of testing samples as (batch_ids, input, target).
         batch_size_test: An integer representing the batch size of the test set.
         device: A string with values either "cpu" or "cuda" to indicate the
             device that Pytorch is performing training on. By default this
@@ -114,8 +114,8 @@ def train(number_of_epochs, network, optimizer, loss_function,
         number_of_epochs: Number of epochs of training.
         optimizer: The optimizer instance to use for training.
         loss_function: An instance of the loss function to use for training.
-        training_set: The list of training samples as (input, target) pairs.
-        test_set: The list of testing samples as (input, target) pairs.
+        training_set: The list of training samples as (batch_ids, input, target).
+        test_set: The list of testing samples as (batch_ids, input, target).
         batch_size_train: An integer representing batch size of the training set.
         batch_size_test: An integer representing the batch size of the test set.
         device: A string with values either "cpu" or "cuda" to indicate the
@@ -162,7 +162,7 @@ def train_compatibility_epoch(epoch, h2, optimizer, loss_function, training_set,
         h2: The model which is undergoing training / updating.
         optimizer: The optimizer instance to use for training.
         loss_function: An instance of a compatibility loss function.
-        training_set: The list of training samples as (input, target) pairs.
+        training_set: The list of training samples as (batch_ids, input, target).
         batch_size_train: An integer representing batch size of the training set.
         device: A string with values either "cpu" or "cuda" to indicate the
             device that Pytorch is performing training on. By default this
@@ -203,7 +203,7 @@ def test_compatibility(h2, loss_function, test_set, batch_size_test, device="cpu
     Args:
         h2: The model which is undergoing training / updating.
         loss_function: An instance of a compatibility loss function.
-        test_set: The list of testing samples as (input, target) pairs.
+        test_set: The list of testing samples as (batch_ids, input, target).
         batch_size_test: An integer representing the batch size of the test set.
         device: A string with values either "cpu" or "cuda" to indicate the
             device that Pytorch is performing training on. By default this
@@ -248,8 +248,8 @@ def train_compatibility(number_of_epochs, h2, optimizer, loss_function,
         number_of_epochs: Number of epochs of training.
         loss_function: An instance of a compatibility loss function.
         optimizer: The optimizer instance to use for training.
-        training_set: The list of training samples as (input, target) pairs.
-        test_set: The list of testing samples as (input, target) pairs.
+        training_set: The list of training samples as (batch_ids, input, target).
+        test_set: The list of testing samples as (batch_ids, input, target).
         batch_size_train: An integer representing batch size of the training set.
         batch_size_test: An integer representing the batch size of the test set.
         device: A string with values either "cpu" or "cuda" to indicate the
@@ -313,6 +313,54 @@ def get_error_instance_indices(model, batched_evaluation_data, batched_evaluatio
         return model_diff.nonzero().view(-1).tolist()
 
 
+def get_all_error_instance_indices(h1, h2, batched_evaluation_data, batched_evaluation_target,
+                                   device="cpu"):
+    """
+    Return the list of indices of instances from batched_evaluation_data on which the
+    model prediction differs from the ground truth in batched_evaluation_target.
+
+    Args:
+        h1: The baseline model.
+        h2: The new updated model.
+        batched_evaluation_data: A single batch of input data to be passed to our model.
+        batched_evaluation_target: A single batch of the corresponding output targets.
+        device: A string with values either "cpu" or "cuda" to indicate the
+            device that Pytorch is performing training on. By default this
+            value is "cpu". But in case your models reside on the GPU, make sure
+            to set this to "cuda". This makes sure that the input and target
+            tensors are transferred to the GPU during training.
+    Returns:
+        A list of indices of the instances within the batched data, for which the
+        model did not match the expected target.
+    """
+    with torch.no_grad():
+        if device != "cpu":
+            batched_evaluation_data = batched_evaluation_data.to(device)
+            batched_evaluation_target = batched_evaluation_target.to(device)
+        _, _, h1_output_logsoftmax = h1(batched_evaluation_data)
+        _, _, h2_output_logsoftmax = h2(batched_evaluation_data)
+        h1_diff = (torch.argmax(h1_output_logsoftmax, 1) - batched_evaluation_target).float()
+        h2_diff = (torch.argmax(h2_output_logsoftmax, 1) - batched_evaluation_target).float()
+        h1_error_instance_ids = h1_diff.nonzero().view(-1).tolist()
+        h2_error_instance_ids = h2_diff.nonzero().view(-1).tolist()
+        error_instance_ids = list(set(h1_error_instance_ids).union(set(h2_error_instance_ids)))
+        h1_predictions = []
+        h2_predictions = []
+        instance_ground_truths = []
+        if len(error_instance_ids) > 0:
+            h1_predictions = torch.argmax(h1_output_logsoftmax, 1).index_select(
+                0, torch.tensor(error_instance_ids).to(device)).tolist()
+            h2_predictions = torch.argmax(h2_output_logsoftmax, 1).index_select(
+                0, torch.tensor(error_instance_ids).to(device)).tolist()
+            instance_ground_truths = batched_evaluation_target.index_select(
+                0, torch.tensor(error_instance_ids).to(device)).tolist()
+
+        return list(zip(error_instance_ids,
+                    h1_predictions,
+                    h2_predictions,
+                    instance_ground_truths))
+
+
 def get_model_error_overlap(h1, h2, batch_ids, batched_evaluation_data, batched_evaluation_target,
                             device="cpu"):
     """
@@ -384,15 +432,17 @@ def get_error_instance_ids_by_class(model, batch_ids, batched_evaluation_data, b
             batched_evaluation_data = batched_evaluation_data.to(device)
             batched_evaluation_target = batched_evaluation_target.to(device)
         _, _, model_output_logsoftmax = model(batched_evaluation_data)
+        class_error_instance_ids = {}
         model_diff = (torch.argmax(model_output_logsoftmax, 1) - batched_evaluation_target).float()
         model_errors = model_diff.nonzero().view(-1).tolist()
-        error_instance_indices = torch.tensor(batch_ids).index_select(0, torch.tensor(model_errors)).tolist()
-        target_error_classes = batched_evaluation_target[model_errors].view(-1).tolist()
-        class_error_instance_ids = {}
-        for class_label in set(batched_evaluation_target.view(-1).tolist()):
-            class_error_instance_ids[class_label] = []
-        for error_instance_index, error_class in zip(error_instance_indices, target_error_classes):
-            class_error_instance_ids[error_class].append(error_instance_index)
+        if len(model_errors) > 0:
+            error_instance_indices = torch.tensor(batch_ids).index_select(
+                0, torch.tensor(model_errors)).tolist()
+            target_error_classes = batched_evaluation_target[model_errors].view(-1).tolist()
+            for class_label in set(batched_evaluation_target.view(-1).tolist()):
+                class_error_instance_ids[class_label] = []
+            for error_instance_index, error_class in zip(error_instance_indices, target_error_classes):
+                class_error_instance_ids[error_class].append(error_instance_index)
 
         return class_error_instance_ids
 
@@ -460,12 +510,16 @@ def evaluate_model_performance_and_compatibility_on_dataset(h1, h2, dataset, per
     h1_and_h2_dataset_error_instance_ids = []
     h2_dataset_error_instance_ids_by_class = {}
     classes = set()
+    all_error_instances = []
     for batch_ids, data, target in dataset:
         classes = classes.union(target.tolist())
         h1_error_count_batch, h2_error_count_batch, h1_and_h2_error_count_batch =\
             get_model_error_overlap(h1, h2, batch_ids, data, target, device=device)
         h2_error_instance_ids_by_class =\
             get_error_instance_ids_by_class(h2, batch_ids, data, target, device=device)
+        all_errors = get_all_error_instance_indices(
+            h1, h2, data, target, device=device)
+        all_error_instances += all_errors
         h1_dataset_error_instance_ids += h1_error_count_batch
         h2_dataset_error_instance_ids += h2_error_count_batch
         h1_and_h2_dataset_error_instance_ids += h1_and_h2_error_count_batch
@@ -484,6 +538,15 @@ def evaluate_model_performance_and_compatibility_on_dataset(h1, h2, dataset, per
 
     h2_performance = performance_metric(h2, dataset, device)
 
+    all_error_instances_results = []
+    for error_instance_id, h1_prediction, h2_prediction, ground_truth in all_error_instances:
+        all_error_instances_results.append({
+            "instance_id": error_instance_id,
+            "h1_prediction": h1_prediction,
+            "h2_prediction": h2_prediction,
+            "ground_truth": ground_truth
+        })
+
     btc, bec = compatibility_scores(h1, h2, dataset, device=device)
 
     return {
@@ -497,7 +560,8 @@ def evaluate_model_performance_and_compatibility_on_dataset(h1, h2, dataset, per
         "sorted_classes": sorted(list(classes)),
         "h2_performance": h2_performance,
         "btc": btc,
-        "bec": bec
+        "bec": bec,
+        "error_instances": all_error_instances_results
     }
 
 
@@ -511,8 +575,8 @@ def evaluate_model_performance_and_compatibility(h1, h2, training_set, test_set,
         h1: The reference model being used.
         h2: The model being traind / updated.
         performance_metric: Performance metric to be used when evaluating the model.
-        training_set: The list of batched training samples as (input, target) pairs.
-        test_set: The list of batched testing samples as (input, target) pairs.
+        training_set: The list of batched training samples as (batch_ids, input, target).
+        test_set: The list of batched testing samples as (batch_ids, input, target).
         device: A string with values either "cpu" or "cuda" to indicate the
             device that Pytorch is performing training on. By default this
             value is "cpu". But in case your models reside on the GPU, make sure
@@ -547,8 +611,8 @@ def train_new_error(h1, h2, number_of_epochs,
         h1: Reference Pytorch model.
         h2: The model which is undergoing training / updating.
         number_of_epochs: Number of epochs of training.
-        training_set: The list of training samples as (input, target) pairs.
-        test_set: The list of testing samples as (input, target) pairs.
+        training_set: The list of training samples as (batch_ids, input, target).
+        test_set: The list of testing samples as (batch_ids, input, target).
         batch_size_train: An integer representing batch size of the training set.
         batch_size_test: An integer representing the batch size of the test set.
         OptimizerClass: The class to instantiate an optimizer from for training.
@@ -582,8 +646,8 @@ def train_strict_imitation(h1, h2, number_of_epochs,
         h1: Reference Pytorch model.
         h2: The model which is undergoing training / updating.
         number_of_epochs: Number of epochs of training.
-        training_set: The list of training samples as (input, target) pairs.
-        test_set: The list of testing samples as (input, target) pairs.
+        training_set: The list of training samples as (batch_ids, input, target).
+        test_set: The list of testing samples as (batch_ids, input, target).
         batch_size_train: An integer representing batch size of the training set.
         batch_size_test: An integer representing the batch size of the test set.
         OptimizerClass: The class to instantiate an optimizer from for training.
@@ -628,8 +692,8 @@ def compatibility_sweep(sweeps_folder_path, number_of_epochs, h1, h2,
         number_of_epochs: The number of training epochs to use on each sweep.
         h1: The reference model being used.
         h2: The new model being traind / updated.
-        training_set: The list of training samples as (input, target) pairs.
-        test_set: The list of testing samples as (input, target) pairs.
+        training_set: The list of training samples as (batch_ids, input, target).
+        test_set: The list of testing samples as (batch_ids, input, target).
         batch_size_train: An integer representing batch size of the training set.
         batch_size_test: An integer representing the batch size of the test set.
         OptimizerClass: The class to instantiate an optimizer from for training.
@@ -645,7 +709,7 @@ def compatibility_sweep(sweeps_folder_path, number_of_epochs, h1, h2,
             expected to have the following signature:
                 metric(model, dataset, device)
                     model: The model being evaluated
-                    dataset: The dataset as a list of (input, target) pairs
+                    dataset: The dataset as a list of (batch_ids, input, target)
                     device: The device Pytorch is using for training - "cpu" or "cuda"
             If unspecified, then accuracy is used.
         lambda_c_stepsize: The increments of lambda_c to use as we sweep the parameter

backwardcompatibilityml/sweep_management.py

@@ -4,6 +4,10 @@
 import os
 import json
 import threading
+import io
+import numpy as np
+from flask import send_file
+from PIL import Image
 from queue import Queue
 from backwardcompatibilityml.helpers import training
 from backwardcompatibilityml.metrics import model_accuracy
@@ -29,8 +33,8 @@ class SweepManager(object):
         number_of_epochs: The number of training epochs to use on each sweep.
         h1: The reference model being used.
         h2: The new model being traind / updated.
-        training_set: The list of training samples as (input, target) pairs.
-        test_set: The list of testing samples as (input, target) pairs.
+        training_set: The list of training samples as (batch_ids, input, target).
+        test_set: The list of testing samples as (batch_ids, input, target).
         batch_size_train: An integer representing batch size of the training set.
         batch_size_test: An integer representing the batch size of the test set.
         OptimizerClass: The class to instantiate an optimizer from for training.
@@ -146,7 +150,14 @@ class SweepManager(object):
         if get_instance_data_by_id is not None:
             return get_instance_data_by_id(instance_id)
 
-        return {}
+        # Generate a blank white PNG image as the default
+        data = np.uint8(np.zeros((30, 30)) + 255)
+        image = Image.fromarray(data)
+        img_bytes = io.BytesIO()
+        image.save(img_bytes, format="PNG")
+        img_bytes.seek(0)
+
+        return send_file(img_bytes, mimetype="image/png")
 
     def get_instance_label(self, instance_id):
         get_instance_label_by_id = self.get_instance_label_by_id

backwardcompatibilityml/widget/compatibility_analysis.py

@@ -18,6 +18,7 @@ from rai_core_flask.environments import (
     AzureNBEnvironment,
     DatabricksEnvironment,
     LocalIPythonEnvironment)
+from backwardcompatibilityml.helpers import http
 
 
 def build_environment_params(flask_service_env):
@@ -99,6 +100,7 @@ def init_app_routes(app, sweep_manager):
     """
 
     @app.route("/api/v1/start_sweep", methods=["POST"])
+    @http.no_cache
     def start_sweep():
         sweep_manager.start_sweep()
         return {
@@ -107,6 +109,7 @@ def init_app_routes(app, sweep_manager):
         }
 
     @app.route("/api/v1/sweep_status", methods=["GET"])
+    @http.no_cache
     def get_sweep_status():
         return {
             "running": sweep_manager.sweep_thread.is_alive(),
@@ -114,20 +117,24 @@ def init_app_routes(app, sweep_manager):
         }
 
     @app.route("/api/v1/sweep_summary", methods=["GET"])
+    @http.no_cache
     def get_data():
         return Response(
             json.dumps(sweep_manager.get_sweep_summary()),
             mimetype="application/json")
 
     @app.route("/api/v1/evaluation_data/<int:evaluation_id>")
+    @http.no_cache
     def get_evaluation(evaluation_id):
         return sweep_manager.get_evaluation(evaluation_id)
 
     @app.route("/api/v1/instance_data/<int:instance_id>")
+    @http.no_cache
     def get_instance_data(instance_id):
         return sweep_manager.get_instance_data(instance_id)
 
     @app.route("/api/v1/instance_label/<int:instance_id>")
+    @http.no_cache
     def get_instance_label(instance_id):
         return sweep_manager.get_instance_label(instance_id)
 
@@ -163,8 +170,8 @@ class CompatibilityAnalysis(object):
         number_of_epochs: The number of training epochs to use on each sweep.
         h1: The reference model being used.
         h2: The new model being traind / updated.
-        training_set: The list of training samples as (input, target) pairs.
-        test_set: The list of testing samples as (input, target) pairs.
+        training_set: The list of training samples as (batch_ids, input, target).
+        test_set: The list of testing samples as (batch_ids, input, target).
         batch_size_train: An integer representing batch size of the training set.
         batch_size_test: An integer representing the batch size of the test set.
         lambda_c_stepsize: The increments of lambda_c to use as we sweep the parameter

development/app.py

@@ -32,27 +32,33 @@ for i in range(len(data)):
     data[i] = row
 
 data = list(map(lambda r: (torch.tensor(r[:9], dtype=torch.float32), torch.tensor(r[9])), data))
+instance_ids = list(range(len(data)))
+dataset = []
+for (instance_id, (instance_data, instance_label)) in zip(instance_ids, data):
+    dataset.append([instance_id, instance_data, instance_label])
 
-random.shuffle(data)
+random.shuffle(dataset)
 
-training_set = data[:560]
-testing_set = data[560:]
+training_set = dataset[:560]
+testing_set = dataset[560:]
 
 training_set_torch = []
 prev = 0
 for i in range((batch_size_train - 1), len(training_set), batch_size_train):
-    training_data = list(map(lambda r: r[0], training_set[prev:(i + 1)]))
-    training_labels = list(map(lambda r: r[1], training_set[prev:(i + 1)]))
+    batch_ids = list(map(lambda r: r[0], training_set[prev:(i + 1)]))
+    training_data = list(map(lambda r: r[1], training_set[prev:(i + 1)]))
+    training_labels = list(map(lambda r: r[2], training_set[prev:(i + 1)]))
     prev = i
-    training_set_torch.append([torch.stack(training_data, dim=0), torch.stack(training_labels, dim=0)])
+    training_set_torch.append([batch_ids, torch.stack(training_data, dim=0), torch.stack(training_labels, dim=0)])
 
 testing_set_torch = []
 prev = 0
 for i in range((batch_size_test - 1), len(testing_set), batch_size_test):
-    testing_data = list(map(lambda r: r[0], testing_set[prev:(i + 1)]))
-    testing_labels = list(map(lambda r: r[1], testing_set[prev:(i + 1)]))
+    batch_ids = list(map(lambda r: r[0], testing_set[prev:(i + 1)]))
+    testing_data = list(map(lambda r: r[1], testing_set[prev:(i + 1)]))
+    testing_labels = list(map(lambda r: r[2], testing_set[prev:(i + 1)]))
     prev = i
-    testing_set_torch.append([torch.stack(testing_data, dim=0), torch.stack(testing_labels, dim=0)])
+    testing_set_torch.append([batch_ids, torch.stack(testing_data, dim=0), torch.stack(testing_labels, dim=0)])
 
 training_set = training_set_torch
 partial_training_set = training_set[:int(0.5 * (len(training_set)))]
@@ -66,7 +72,7 @@ train_counter, test_counter, train_losses, test_losses = training.train(
 h1.eval()
 
 with torch.no_grad():
-    _, _, output = h1(testing_set[0][0])
+    _, _, output = h1(testing_set[0][1])
 
 h2 = MLPClassifier(9, 2)
 

package.json

@@ -2,6 +2,7 @@
   "dependencies": {
     "axios": "^0.19.2",
     "d3": "^5.16.0",
+    "office-ui-fabric-react": "^7.117.0",
     "jquery": "^3.5.1",
     "lodash": "^4.17.11",
     "plotly.js-dist": "^1.54.7",

requirements.txt

@@ -3,3 +3,4 @@ Jinja2==2.11.2
 numpy==1.19.0
 scikit-learn==0.23.1
 rai_core_flask==0.0.2
+Pillow==7.2.0

widget/ErrorInstancesTable.tsx

@@ -3,37 +3,104 @@
 
 import React, { Component } from "react";
 import ReactDOM from "react-dom";
-import * as d3 from "d3";
+import {
+  DetailsList,
+  DetailsListLayoutMode,
+  Selection,
+  SelectionMode,
+  IColumn
+} from "office-ui-fabric-react/lib/DetailsList";
+import { DefaultButton } from 'office-ui-fabric-react';
+import { Fabric } from "office-ui-fabric-react/lib/Fabric";
+import { apiBaseUrl } from "./api.ts";
 
 
 type ErrorInstancesTableState = {
-  data: any
+  selecedDataPoint: any,
+  page: number
 }
 
 type ErrorInstancesTableProps = {
-  data: any
+  selectedDataPoint: any,
+  pageSize?: number
 }
 
 class ErrorInstancesTable extends Component<ErrorInstancesTableProps, ErrorInstancesTableState> {
+
+  public static defaultProps = {
+      pageSize: 5
+  };
+
   constructor(props) {
     super(props);
-
     this.state = {
-      data: this.props.data
-    };
+      selecedDataPoint: null,
+      page: 0
+    }
   }
 
   componentWillReceiveProps(nextProps) {
     this.setState({
-      data: nextProps.data
+      selecedDataPoint: nextProps.selecedDataPoint
     });
   }
 
   render() {
+
+    if (this.props.selectedDataPoint == null) {
+      return (
+        <React.Fragment />
+      );
+    }
+    var columns = [
+      {
+        key: 'instanceId',
+        name: 'Instance',
+        fieldName: 'instance_id',
+        minWidth: 100,
+        maxWidth: 100,
+        isResizable: false ,
+        onRender: (instance) => {
+          return (<img src={`${apiBaseUrl}/api/v1/instance_data/${instance.instance_id}`} />);
+        }
+      },
+      { key: 'h1Prediction', name: 'h1 Prediction', fieldName: 'h1_prediction', minWidth: 100, maxWidth: 100, isResizable: false },
+      { key: 'h2Prediction', name: 'h2 Prediction', fieldName: 'h2_prediction', minWidth: 100, maxWidth: 100, isResizable: false },
+      { key: 'groundTruth', name: 'Ground Truth', fieldName: 'ground_truth', minWidth: 100, maxWidth: 100, isResizable: false },
+    ];
+
+    var items = [];
+    for(var i=(this.state.page * this.props.pageSize);
+        i < Math.min((this.state.page * this.props.pageSize) + this.props.pageSize),
+                     this.props.selectedDataPoint.error_instances.length;
+        i++) {
+      items.push(this.props.selectedDataPoint.error_instances[i]);
+    }
+
     return (
-      <div className="table">
-        Error Instances Table goes here
-      </div>
+      <Fabric>
+        <DetailsList
+          selectionMode={SelectionMode.none}
+          items={items}
+          columns={columns}
+        />
+        <DefaultButton
+          text="Previous"
+          onClick={() => {
+            this.setState({
+              page: Math.max(0, this.state.page - 1)
+            })
+          }}
+        />
+        <DefaultButton
+          text="Next"
+          onClick={() => {
+            this.setState({
+              page: Math.min(this.state.page + 1, Math.ceil(this.props.selectedDataPoint.error_instances.length / this.props.pageSize))
+            })
+          }}
+        />
+      </Fabric>
     );
   }
 }

widget/MainContainer.tsx

@@ -133,10 +133,7 @@ function Container({
             <IncompatiblePointDistribution selectedDataPoint={selectedDataPoint} />
           </div>
           <div className="row">
-            <RawValues data={data.data} />
-          </div>
-          <div className="row">
-            <ErrorInstancesTable data={data.data} />
+            <ErrorInstancesTable selectedDataPoint={selectedDataPoint} />
           </div>
       </div>
     );

widget/api.ts

@@ -27,6 +27,7 @@ function makePostCall(endpoint: string, payload: any) {
 }
 
 export {
+  apiBaseUrl,
   makeGetCall,
   makePostCall
 };