Nested Exif Metadata support + Gate fixes for py 3.7 and keras version (#2545)

* test ckpt * gate updates * removed 3.7 * fridge test fixes * test fixes * other gate test fixes * added keras suffix * test fixes * tf keras save fix * pegged tf to avoid keras 3.0 install * tf keras update * removed .keras from path * keras version update * version updates * version updates * version updates * keras install gate * tf-keras install update * test updates * tf-keras support * removed py 3.7 * lint fixes * keras gate for nbk tests * removed py 3.7 from nbk tests * removed commented code
2024-04-02 09:49:19 -07:00 · 2024-04-02 09:49:19 -07:00 · 76e0de4bb0
--- a/.github/workflows/CI-e2e-notebooks-text-vision.yml
+++ b/.github/workflows/CI-e2e-notebooks-text-vision.yml
@ -56,7 +56,7 @@ jobs:
      matrix:
        # TODO: add macos
        operatingSystem: [ubuntu-latest, windows-latest]
-        pythonVersion: [3.7, 3.8, 3.9, "3.10"]
+        pythonVersion: [3.8, 3.9, "3.10"]
        flights: [""]
        notebookGroup: ["vis_nb_group_1", "text_nb_group_1"]

@ -99,6 +99,12 @@ jobs:
          pip install --upgrade setuptools
          pip install --upgrade "pip-tools<=7.1.0"

+      - name: Install backwards-compatible keras for transformers
+        shell: bash -l {0}
+        run: |
+          pip install tf-keras
+          pip install keras==2.15
+
      - name: Install dependencies
        shell: bash -l {0}
        run: |
--- a/.github/workflows/CI-notebook-text.yml
+++ b/.github/workflows/CI-notebook-text.yml
@ -20,7 +20,7 @@ jobs:
    strategy:
      matrix:
        operatingSystem: [ubuntu-latest, windows-latest]
-        pythonVersion: [3.7, 3.8, 3.9, "3.10"]
+        pythonVersion: [3.8, 3.9, "3.10"]

    runs-on: ${{ matrix.operatingSystem }}

@ -49,18 +49,12 @@ jobs:
        run: yarn buildall

      - if: ${{ matrix.operatingSystem != 'macos-latest' }}
-        name: Install pytorch on non-MacOS with python 3.7
+        name: Install pytorch on non-MacOS
        shell: bash -l {0}
        run: |
          conda install --yes --quiet "pytorch==1.13.1" "torchvision<0.15" captum cpuonly "numpy<1.24.0" -c pytorch

-      - if: ${{ matrix.operatingSystem == 'macos-latest' && matrix.pythonVersion == '3.7' }}
-        name: Install Anaconda packages on MacOS with python 3.7
-        shell: bash -l {0}
-        run: |
-          conda install --yes --quiet "pytorch==1.13.1" "torchvision<0.15" captum "numpy<1.24.0" -c pytorch
-
-      - if: ${{ matrix.operatingSystem == 'macos-latest' && matrix.pythonVersion != '3.7' }}
+      - if: ${{ matrix.operatingSystem == 'macos-latest' }}
        name: Install Anaconda packages on MacOS, which should not include cpuonly according to official docs
        shell: bash -l {0}
        run: |
@ -73,6 +67,12 @@ jobs:
          pip install --upgrade setuptools
          pip install --upgrade "pip-tools<=7.1.0"

+      - name: Install backwards-compatible keras for transformers
+        shell: bash -l {0}
+        run: |
+          pip install tf-keras
+          pip install keras==2.15
+
      - name: Install dependencies
        shell: bash -l {0}
        run: |
--- a/.github/workflows/CI-notebook-vision.yml
+++ b/.github/workflows/CI-notebook-vision.yml
@ -20,7 +20,7 @@ jobs:
    strategy:
      matrix:
        operatingSystem: [ubuntu-latest]
-        pythonVersion: [3.7, 3.8, 3.9, "3.10"]
+        pythonVersion: [3.8, 3.9, "3.10"]

    runs-on: ${{ matrix.operatingSystem }}

@ -48,25 +48,13 @@ jobs:
      - name: Build Typescript
        run: yarn buildall

-      - if: ${{ matrix.operatingSystem != 'macos-latest' && matrix.pythonVersion == '3.7' }}
-        name: Install pytorch on non-MacOS with python 3.7
-        shell: bash -l {0}
-        run: |
-          conda install --yes --quiet "pytorch==1.13.1" "torchvision<0.15" cpuonly "numpy<1.24.0" -c pytorch
-
-      - if: ${{ matrix.operatingSystem == 'macos-latest' && matrix.pythonVersion == '3.7' }}
-        name: Install Anaconda packages on MacOS with python 3.7
-        shell: bash -l {0}
-        run: |
-          conda install --yes --quiet "pytorch==1.13.1" "torchvision<0.15" "numpy<1.24.0" -c pytorch
-
-      - if: ${{ matrix.operatingSystem != 'macos-latest' && matrix.pythonVersion != '3.7' }}
+      - if: ${{ matrix.operatingSystem != 'macos-latest' }}
        name: Install pytorch on non-MacOS
        shell: bash -l {0}
        run: |
          conda install --yes --quiet "pytorch<2.1,>1.13.1" "torchvision<0.16" cpuonly "numpy<1.24.0" -c pytorch

-      - if: ${{ matrix.operatingSystem == 'macos-latest' && matrix.pythonVersion != '3.7' }}
+      - if: ${{ matrix.operatingSystem == 'macos-latest' }}
        name: Install Anaconda packages on MacOS, which should not include cpuonly according to official docs
        shell: bash -l {0}
        run: |
--- a/.github/workflows/CI-responsibleai-text-vision-pytest.yml
+++ b/.github/workflows/CI-responsibleai-text-vision-pytest.yml
@ -72,6 +72,12 @@ jobs:
        run: |
          conda install --yes --quiet matplotlib -c conda-forge

+      - name: Install backwards-compatible keras for transformers
+        shell: bash -l {0}
+        run: |
+          pip install tf-keras
+          pip install keras==2.15
+
      - name: Install dependencies
        shell: bash -l {0}
        run: |
--- a/libs/e2e/src/lib/describer/modelAssessment/datasets/FridgeObjectDetectionModelDebugging.ts
+++ b/libs/e2e/src/lib/describer/modelAssessment/datasets/FridgeObjectDetectionModelDebugging.ts
@ -3,7 +3,7 @@

 import { getOS } from "../../../../util/getOS";

-const FeatureCohorts = getOS() === "Linux" ? [3, 6] : 6;
+const FeatureCohorts = getOS() === "Linux" ? [2, 6] : 6;

 export const FridgeObjectDetectionModelDebugging = {
  causalAnalysisData: {
--- a/libs/e2e/src/lib/describer/modelAssessment/modelOverview/ensureAllModelOverviewFeatureCohortsViewElementsAfterSelectionArePresent.ts
+++ b/libs/e2e/src/lib/describer/modelAssessment/modelOverview/ensureAllModelOverviewFeatureCohortsViewElementsAfterSelectionArePresent.ts
@ -63,7 +63,7 @@ function assertNumberOfChartRowsEqual(
      .should("be.gte", expectedNumberOfCohorts[0])
      .and("be.lte", expectedNumberOfCohorts[1]);
  } else if (isVision) {
-    cy.get(getChartItems(chartIdentifier)).its("length").should("be.gt", 2);
+    cy.get(getChartItems(chartIdentifier)).its("length").should("be.gte", 2);
  } else {
    cy.get(getChartItems(chartIdentifier)).should(
      "have.length",
--- a/responsibleai_vision/requirements-dev.txt
+++ b/responsibleai_vision/requirements-dev.txt
@ -21,9 +21,9 @@ sphinx==3.1.1
 sphinx-gallery==0.8.1
 pydata-sphinx-theme==0.3.0

+tensorflow<=2.15
 transformers
 datasets
-tensorflow
 opencv-python

 fastai
--- a/responsibleai_vision/tests/common_vision_utils.py
+++ b/responsibleai_vision/tests/common_vision_utils.py
@ -318,6 +318,46 @@ def load_fridge_object_detection_dataset(automl_format=False):
    return data


+def load_clearsight_object_detection_dataset(automl_format=False):
+    # create data folder if it doesnt exist.
+    os.makedirs("data", exist_ok=True)
+
+    # download data
+    download_url = ("https://publictestdatasets.blob.core.windows.net/" +
+                    "computervision/clearsight_mini.zip")
+    data_file = "./data/clearsight_mini.zip"
+    retrieve_unzip_file(download_url, data_file)
+
+    # dummy function to load labels
+    labels = load_fridge_object_detection_dataset_labels(automl_format)[:2]
+    if automl_format:
+        image_details = load_image_details()
+        columns = [ImageColumns.IMAGE.value,
+                   ImageColumns.IMAGE_DETAILS.value,
+                   ImageColumns.LABEL.value]
+    else:
+        columns = [ImageColumns.IMAGE.value,
+                   ImageColumns.LABEL.value]
+    features = []
+    for i, file in enumerate(os.listdir("./data/clearsight_mini")):
+        image_path = "./data/clearsight_mini/" + file
+        if automl_format:
+            row = {
+                ImageColumns.IMAGE.value: image_path,
+                ImageColumns.IMAGE_DETAILS.value: image_details[i],
+                ImageColumns.LABEL.value: labels[i]
+            }
+        else:
+            row = {
+                ImageColumns.IMAGE.value: image_path,
+                ImageColumns.LABEL.value: labels[i]
+            }
+        features.append(row)
+
+    data = pd.DataFrame(features, columns=columns)
+    return data
+
+
 class ImageTransformEnum(Enum):
    '''
    Possible modifications to images
--- a/responsibleai_vision/tests/test_feature_extractors.py
+++ b/responsibleai_vision/tests/test_feature_extractors.py
@ -2,7 +2,8 @@
 # Licensed under the MIT License.

 import pytest
-from common_vision_utils import (load_flowers_dataset, load_fridge_dataset,
+from common_vision_utils import (load_clearsight_object_detection_dataset,
+                                 load_flowers_dataset, load_fridge_dataset,
                                 load_fridge_object_detection_dataset,
                                 load_imagenet_dataset)

@ -13,22 +14,42 @@ from responsibleai_vision.utils.feature_extractors import extract_features

 MEAN_PIXEL_VALUE = ExtractedFeatures.MEAN_PIXEL_VALUE.value
 FRIDGE_METADATA_FEATURES = [
-    'SensingMethod', 'GPSVersionID', 'ISOSpeedRatings', 'SceneType',
-    'SceneCaptureType', 'SubjectDistance', 'CustomRendered',
-    'SubjectDistanceRange', 'DigitalZoomRatio', 'ApertureValue',
-    'ImageWidth', 'GPSDOP', 'MaxApertureValue', 'ColorSpace',
-    'FocalLengthIn35mmFilm', 'ExposureMode', 'Saturation', 'ExposureTime',
-    'ExifImageHeight', 'FNumber', 'YCbCrPositioning', 'Make', 'MeteringMode',
-    'ExposureBiasValue', 'ExposureProgram', 'ComponentsConfiguration',
-    'ExifImageWidth', 'ExifInteroperabilityOffset', 'BrightnessValue',
-    'ImageLength', 'FlashPixVersion', 'SubsecTimeOriginal', 'Model',
-    'SubsecTimeDigitized', 'ResolutionUnit', 'DateTimeOriginal', 'XResolution',
-    'FocalLength', 'Sharpness', 'GPSLongitude', 'Contrast', 'Software',
-    'GPSLatitude', 'MakerNote', 'GPSDateStamp', 'GPSAltitude',
-    'GPSProcessingMethod', 'GPSTimeStamp', 'GPSLatitudeRef', 'WhiteBalance',
-    'GPSLongitudeRef', 'Flash', 'SubsecTime', 'YResolution',
-    'DateTimeDigitized', 'DateTime', 'GPSAltitudeRef', 'Orientation',
-    'ShutterSpeedValue', 'ExifVersion']
+    'FlashPixVersion', 'WhiteBalance', 'Make', 'ExifImageHeight',
+    'ExposureProgram', 'Software', 'CustomRendered', 'Contrast', 'XResolution',
+    'GPSTimeStamp', 'Orientation', 'SubjectDistanceRange', 'MakerNote',
+    'SubsecTimeDigitized', 'ExifInteroperabilityOffset', 'ColorSpace',
+    'BrightnessValue', 'ExifImageWidth', 'SubsecTimeOriginal',
+    'YCbCrPositioning', 'ISOSpeedRatings', 'SubsecTime', 'ExposureBiasValue',
+    'ComponentsConfiguration', 'ResolutionUnit', 'ImageLength',
+    'GPSLatitudeRef', 'ExifVersion', 'Flash', 'ExposureMode', 'ApertureValue',
+    'GPSDOP', 'ImageWidth', 'SubjectDistance', 'GPSLongitudeRef', 'FNumber',
+    'DigitalZoomRatio', 'FocalLength', 'SceneCaptureType',
+    'FocalLengthIn35mmFilm', 'ShutterSpeedValue', 'DateTimeOriginal',
+    'Sharpness', 'GPSAltitude', 'GPSLatitude', 'Saturation', 'ExposureTime',
+    'GPSLongitude', 'SceneType', 'GPSDateStamp', 'GPSVersionID',
+    'GPSAltitudeRef', 'YResolution', 'GPSProcessingMethod', 'DateTime',
+    'MeteringMode', 'SensingMethod', 'DateTimeDigitized', 'Model',
+    'MaxApertureValue'
+]
+CLEARSIGHT_METADATA_FEATURES = [
+    'FocalLength', 'Make', 'OffsetTimeOriginal', 'FocalPlaneYResolution',
+    'ExposureMode', 'LensSerialNumber', 'ApertureValue', 'GPSLatitudeRef',
+    'DateTimeDigitized', 'Orientation', 'OffsetTime',
+    'FocalPlaneResolutionUnit', 'ExposureProgram', 'ComponentsConfiguration',
+    'Flash', 'BodySerialNumber', 'SubsecTimeDigitized', 'CameraOwnerName',
+    'SubsecTime', 'ExifInteroperabilityOffset', 'ExposureBiasValue',
+    'ISOSpeedRatings', 'SubsecTimeOriginal', 'FNumber', 'WhiteBalance',
+    'GPSLatitude', 'XResolution', 'GPSSatellites', 'ShutterSpeedValue',
+    'GPSLongitudeRef', 'ExposureTime', 'DigitalZoomRatio', 'MeteringMode',
+    'ExifImageHeight', 'GPSTimeStamp', 'LensSpecification', 'Artist',
+    'FlashPixVersion', 'YCbCrPositioning', 'SceneCaptureType', 'DateTime',
+    'LensModel', 'OffsetTimeDigitized', 'GPSStatus', 'CustomRendered',
+    'GPSVersionID', 'SensitivityType', 'DateTimeOriginal', 'Copyright',
+    'ColorSpace', 'RecommendedExposureIndex', 'ResolutionUnit', 'GPSAltitude',
+    'Model', 'YResolution', 'ExifVersion', 'UserComment',
+    'FocalPlaneXResolution', 'GPSAltitudeRef', 'MakerNote', 'GPSMapDatum',
+    'GPSLongitude', 'GPSDateStamp', 'ExifImageWidth'
+]


 def validate_extracted_features(extracted_features, feature_names,
@ -95,3 +116,14 @@ class TestFeatureExtractors(object):
        validate_extracted_features(extracted_features, feature_names,
                                    expected_feature_names, data,
                                    feature_metadata)
+
+    def test_extract_features_clearsight_metadata(self):
+        data = load_clearsight_object_detection_dataset()
+        feature_metadata = FeatureMetadata()
+        extracted_features, feature_names = extract_dataset_features(
+            data, feature_metadata=feature_metadata)
+        expected_feature_names = [MEAN_PIXEL_VALUE]
+        expected_feature_names += CLEARSIGHT_METADATA_FEATURES
+        validate_extracted_features(extracted_features, feature_names,
+                                    expected_feature_names, data,
+                                    feature_metadata)
--- a/responsibleai_vision/tests/test_image_utils.py
+++ b/responsibleai_vision/tests/test_image_utils.py
@ -1,7 +1,6 @@
 # Copyright (c) Microsoft Corporation
 # Licensed under the MIT License.

-import platform
 from collections import Counter
 from http.client import HTTPMessage
 from math import isclose
@ -9,7 +8,8 @@ from unittest.mock import Mock, patch
 from urllib.parse import urlparse

 import numpy as np
-from common_vision_utils import load_fridge_object_detection_dataset
+from common_vision_utils import (load_clearsight_object_detection_dataset,
+                                 load_fridge_object_detection_dataset)

 from responsibleai_vision.common.constants import ImageColumns
 from responsibleai_vision.utils.image_reader import \
@ -96,8 +96,12 @@ class TestImageUtils(object):
    def test_get_all_exif_feature_names(self):
        image_dataset = load_fridge_object_detection_dataset().head(2)
        exif_feature_names = get_all_exif_feature_names(image_dataset)
-        num_features = 49 if platform.system() == "Linux" else 60
-        assert len(exif_feature_names) == num_features
+        assert len(exif_feature_names) == 60
+
+    def test_get_all_clearsight_feature_names(self):
+        image_dataset = load_clearsight_object_detection_dataset().head(2)
+        exif_feature_names = get_all_exif_feature_names(image_dataset)
+        assert len(exif_feature_names) == 64

    def test_generate_od_error_labels(self):
        true_y = np.array([[[3, 142, 257, 395, 463, 0]],