Fix continuous integration tests across incompatible editor versions (2022 and trunk) (#6079)

* Fix com.unity.ml-agents test 2022.3 on win by upgrading Unity versions Run all CI tests against 2023.2 (required by Sentis), not 2022.3 Sample failing sub-jobs before: https://unity-ci.cds.internal.unity3d.com/job/35022030/dependency-graph Sample passing sub-jobs after: https://unity-ci.cds.internal.unity3d.com/job/35033024/dependency-graph Note that `trunk` jobs are still failing after this fix. Those will be investigated separately since they've been failing since March 8: https://unity-ci.cds.internal.unity3d.com/job/34919178/dependency-graph * Ignore yamato-parser output files * Disable `trunk` tests, which break with Unity 6 Clean run of "Run All Combinations of Editors/Platforms Tests" after this change: https://unity-ci.cds.internal.unity3d.com/job/35037130/dependency-graph * Print full diffs when a sensor mismatch occurs in tests * Refactor tests to use positions instead of hardcoded numbers * Disable `RigidBodySensorTests.TestBodiesWithJoint` which fails in 2023.2 * Fix editor version in test_versions.metafile (use 2023.2)
2024-03-13 18:23:15 +01:00 · 2024-03-13 18:23:15 +01:00 · 93221d4cd9
--- a/.gitignore
+++ b/.gitignore
@ -95,3 +95,6 @@ coverage.xml
 /htmlcov/

 **/UserSettings/*
+
+# yamato-parser temporary output files:
+.yamato/unfolded/
--- a/.yamato/com.unity.ml-agents-coverage.yml
+++ b/.yamato/com.unity.ml-agents-coverage.yml
@ -5,8 +5,8 @@
 {% for platform in coverage_test_platforms %}
 {% capture coverageOptions %} --enable-code-coverage --code-coverage-options 'generateHtmlReport;assemblyFilters:+{{ package.assembly }}'{% endcapture %}

-test_coverage_{{ package.name }}_{{ platform.name }}_{{ editor.version }}:
-    name : Coverage {{ package.name }} test {{ editor.version }} on {{ platform.name }}
+test_coverage_{{ package.name }}_{{ platform.name }}_{{ editor.version }}_{{ editor.testProject }}:
+    name : Coverage {{ package.name }} test {{ editor.version }} on {{ platform.name }} in {{ editor.testProject }}
    agent:
        type: {{ platform.type }}
        image: {{ platform.image }}
--- a/.yamato/com.unity.ml-agents-performance.yml
+++ b/.yamato/com.unity.ml-agents-performance.yml
@ -1,5 +1,4 @@
 test_editors:
-  - version: 2022.3
  - version: 2023.2
 ---
 {% for editor in test_editors %}
--- a/.yamato/com.unity.ml-agents-promotion.yml
+++ b/.yamato/com.unity.ml-agents-promotion.yml
@ -1,5 +1,4 @@
 test_editors:
-  - version: 2022.3
  - version: 2023.2
 test_platforms:
  - name: win
--- a/.yamato/com.unity.ml-agents-test.yml
+++ b/.yamato/com.unity.ml-agents-test.yml
@ -1,6 +1,6 @@
 {% metadata_file .yamato/coverage_tests.metafile %}
 test_editors:
-    - version: 2022.3
+    - version: 2023.2
        # We want some scene tests to run in the DevProject, but packages there only support 2020+
      testProject: Project
      enableNoDefaultPackages: !!bool true
@ -9,9 +9,11 @@ test_editors:
      enableNoDefaultPackages: !!bool true

 trunk_editor:
-    - version: trunk
-        # Workaround for MLA-1596 - need to make sure we load the right results.
-      testProject: DevProject
+   # TODO: re-enable trunk tests after adding support for Unity 6. As of 12 Mar 2024, tests fail
+   # with "Error: '6000.0.0b12' is not a valid Unity version"
+   # - version: trunk
+   #     # Workaround for MLA-1596 - need to make sure we load the right results.
+   #   testProject: DevProject

 test_platforms:
    - name: win
@ -42,18 +44,18 @@ all_package_tests:
        {% for coverage_editor in coverage_test_editors %}
        {% for coverage_platform in coverage_test_platforms %}
        {% for coverage_package in coverage_test_packages %}
-        - .yamato/com.unity.ml-agents-coverage.yml#test_coverage_{{ coverage_package.name }}_{{ coverage_platform.name }}_{{ coverage_editor.version }}
+        - .yamato/com.unity.ml-agents-coverage.yml#test_coverage_{{ coverage_package.name }}_{{ coverage_platform.name }}_{{ coverage_editor.version }}_{{ coverage_editor.testProject }}
        {% endfor %}
        {% endfor %}
        {% endfor %}
        {% for editor in test_editors %}
        {% for platform in test_platforms %}
-        - .yamato/com.unity.ml-agents-test.yml#test_com.unity.ml-agents_{{ platform.name }}_{{ editor.version }}
+        - .yamato/com.unity.ml-agents-test.yml#test_com.unity.ml-agents_{{ platform.name }}_{{ editor.version }}_{{ editor.testProject }}
        {% endfor %}
        {% endfor %}
        {% for editor in trunk_editor %}
        {% for platform in test_platforms %}
-        - .yamato/com.unity.ml-agents-test.yml#test_com.unity.ml-agents_{{ platform.name }}_{{ editor.version }}
+        - .yamato/com.unity.ml-agents-test.yml#test_com.unity.ml-agents_{{ platform.name }}_{{ editor.version }}_{{ editor.testProject }}
        {% endfor %}
        {% endfor %}
    triggers:
@ -72,8 +74,8 @@ all_package_tests:
  {% assign noDefaultPackagesOptions = "" %}
  {% endif %}

-test_{{ package.name }}_{{ platform.name }}_{{ editor.version }}:
-    name : {{ package.name }} test {{ editor.version }} on {{ platform.name }}
+test_{{ package.name }}_{{ platform.name }}_{{ editor.version }}_{{ editor.testProject }}:
+    name : {{ package.name }} test {{ editor.version }} on {{ platform.name }} in {{ editor.testProject }}
    agent:
        type: {{ platform.type }}
        image: {{ platform.image }}
@ -90,7 +92,7 @@ test_{{ package.name }}_{{ platform.name }}_{{ editor.version }}:
        {% for coverage_editor in coverage_test_editors %}
        {% for coverage_platform in coverage_test_platforms %}
        {% for coverage_package in coverage_test_packages %}
-        - .yamato/com.unity.ml-agents-coverage.yml#test_coverage_{{ coverage_package.name }}_{{ coverage_platform.name }}_{{ coverage_editor.version }}
+        - .yamato/com.unity.ml-agents-coverage.yml#test_coverage_{{ coverage_package.name }}_{{ coverage_platform.name }}_{{ coverage_editor.version }}_{{ coverage_editor.testProject }}
        {% endfor %}
        {% endfor %}
        {% endfor %}
@ -117,8 +119,8 @@ test_{{ package.name }}_{{ platform.name }}_{{ editor.version }}:
  {% for editor in trunk_editor %}
  {% for platform in test_platforms %}

-test_{{ package.name }}_{{ platform.name }}_trunk:
-    name : {{ package.name }} test {{ editor.version }} on {{ platform.name }}
+test_{{ package.name }}_{{ platform.name }}_{{ editor.version}}_{{ editor.testProject }}:
+    name : {{ package.name }} test {{ editor.version }} on {{ platform.name }} in {{ editor.testProject }}
    agent:
        type: {{ platform.type }}
        image: {{ platform.image }}
@ -141,7 +143,7 @@ test_{{ package.name }}_{{ platform.name }}_trunk:
        {% for coverage_editor in coverage_test_editors %}
        {% for coverage_platform in coverage_test_platforms %}
        {% for coverage_package in coverage_test_packages %}
-        - .yamato/com.unity.ml-agents-coverage.yml#test_coverage_{{ coverage_package.name }}_{{ coverage_platform.name }}_{{ coverage_editor.version }}
+        - .yamato/com.unity.ml-agents-coverage.yml#test_coverage_{{ coverage_package.name }}_{{ coverage_platform.name }}_{{ coverage_editor.version }}_{{ coverage_editor.testProject }}
        {% endfor %}
        {% endfor %}
        {% endfor %}
--- a/.yamato/compressed-sensor-test.yml
+++ b/.yamato/compressed-sensor-test.yml
@ -1,8 +1,8 @@
 {% metadata_file .yamato/test_versions.metafile %}
 ---
 {% for editor in test_editors %}
-test_compressed_obs_{{ editor.version }}:
-  name: Test Compressed Sensor Observation {{ editor.version }}
+test_compressed_obs_{{ editor.version }}_{{ editor.extra_test }}:
+  name: Test Compressed Sensor Observation {{ editor.version }} {{ editor.extra_test }}
  agent:
    type: Unity::VM
    image: ml-agents/ml-agents-ubuntu-18.04:latest
@ -18,7 +18,7 @@ test_compressed_obs_{{ editor.version }}:
      python ml-agents/tests/yamato/scripts/run_compressed_sensor.py --env=artifacts/testPlayer-TestGridCompressed
      python ml-agents/tests/yamato/scripts/run_compressed_sensor.py --env=artifacts/testPlayer-TestTextureCompressed
  dependencies:
-    - .yamato/standalone-build-test.yml#test_linux_standalone_{{ editor.version }}
+    - .yamato/standalone-build-test.yml#test_linux_standalone_{{ editor.version }}_{{ editor.extra_test }}
  triggers:
    cancel_old_ci: true
    {% if editor.extra_test == "sensor" %}
--- a/.yamato/gym-interface-test.yml
+++ b/.yamato/gym-interface-test.yml
@ -1,8 +1,8 @@
 {% metadata_file .yamato/test_versions.metafile %}
 ---
 {% for editor in test_editors %}
-test_gym_interface_{{ editor.version }}:
-  name: Test Linux Gym Interface {{ editor.version }}
+test_gym_interface_{{ editor.version }}_{{ editor.extra_test }}:
+  name: Test Linux Gym Interface {{ editor.version }} {{ editor.extra_test }}
  agent:
    type: Unity::VM
    image: ml-agents/ml-agents-ubuntu-18.04:latest
@ -18,7 +18,7 @@ test_gym_interface_{{ editor.version }}:
      python -u -m ml-agents.tests.yamato.setup_venv
      python ml-agents/tests/yamato/scripts/run_gym.py --env=artifacts/testPlayer-Basic
  dependencies:
-    - .yamato/standalone-build-test.yml#test_linux_standalone_{{ editor.version }}
+    - .yamato/standalone-build-test.yml#test_linux_standalone_{{ editor.version }}_{{ editor.extra_test }}
  triggers:
    cancel_old_ci: true
    {% if editor.extra_test == "gym" %}
--- a/.yamato/python-ll-api-test.yml
+++ b/.yamato/python-ll-api-test.yml
@ -1,8 +1,8 @@
 {% metadata_file .yamato/test_versions.metafile %}
 ---
 {% for editor in test_editors %}
-test_linux_ll_api_{{ editor.version }}:
-  name: Test Linux LL-API {{ editor.version }}
+test_linux_ll_api_{{ editor.version }}_{{ editor.extra_test }}:
+  name: Test Linux LL-API {{ editor.version }} {{ editor.extra_test }}
  agent:
    type: Unity::VM
    image: ml-agents/ml-agents-ubuntu-18.04:latest
@ -20,7 +20,7 @@ test_linux_ll_api_{{ editor.version }}:
      python ml-agents/tests/yamato/scripts/run_llapi.py --env=artifacts/testPlayer-WallJump
      python ml-agents/tests/yamato/scripts/run_llapi.py --env=artifacts/testPlayer-Match3
  dependencies:
-    - .yamato/standalone-build-test.yml#test_linux_standalone_{{ editor.version }}
+    - .yamato/standalone-build-test.yml#test_linux_standalone_{{ editor.version }}_{{ editor.extra_test }}
  triggers:
    cancel_old_ci: true
    {% if editor.extra_test == "llapi" %}
--- a/.yamato/standalone-build-test.yml
+++ b/.yamato/standalone-build-test.yml
@ -1,8 +1,8 @@
 {% metadata_file .yamato/test_versions.metafile %}
 ---
 {% for editor in test_editors %}
-test_linux_standalone_{{ editor.version }}:
-  name: Test Linux Standalone {{ editor.version }}
+test_linux_standalone_{{ editor.version }}_{{ editor.extra_test }}:
+  name: Test Linux Standalone {{ editor.version }} {{ editor.extra_test }}
  agent:
    type: Unity::VM
    image: ml-agents/ml-agents-ubuntu-18.04:latest
--- a/.yamato/test_versions.metafile
+++ b/.yamato/test_versions.metafile
@ -3,9 +3,9 @@
 # For each "other" test, we only run it against a single version of the
 # editor to reduce the number of yamato jobs
 test_editors:
-  - version: 2022.3
+  - version: 2023.2
    extra_test: gym
-  - version: 2023.3
+  - version: 2023.2
    extra_test: sensor
  - version: trunk
    extra_test: llapi
--- a/.yamato/training-int-tests.yml
+++ b/.yamato/training-int-tests.yml
@ -1,8 +1,8 @@
 {% metadata_file .yamato/test_versions.metafile %}
 ---
 {% for editor in test_editors %}
-test_linux_training_int_{{ editor.version }}:
-  name: Test Linux Fast Training {{ editor.version }}
+test_linux_training_int_{{ editor.version }}_{{ editor.extra_test }}:
+  name: Test Linux Fast Training {{ editor.version }} {{ editor.extra_test }}
  agent:
    type: Unity::VM
    image: ml-agents/ml-agents-ubuntu-18.04:latest
@ -16,7 +16,7 @@ test_linux_training_int_{{ editor.version }}:
      python -m pip install pyyaml --index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple
      python -u -m ml-agents.tests.yamato.training_int_tests
  dependencies:
-    - .yamato/standalone-build-test.yml#test_linux_standalone_{{ editor.version }}
+    - .yamato/standalone-build-test.yml#test_linux_standalone_{{ editor.version }}_{{ editor.extra_test }}
  triggers:
    cancel_old_ci: true
    expression: |
--- a/com.unity.ml-agents.extensions/Tests/Runtime/Sensors/RigidBodySensorTests.cs
+++ b/com.unity.ml-agents.extensions/Tests/Runtime/Sensors/RigidBodySensorTests.cs
@ -60,6 +60,9 @@ namespace Unity.MLAgents.Extensions.Tests.Sensors
            SensorTestHelper.CompareObservation(sensor, expected);
        }

+        // TODO: figure out why this fails with position mismatches when upgrading from
+        // Unity 2022.3 to Unity 2023.2.
+#if !UNITY_2023_2_OR_NEWER
        [Test]
        public void TestBodiesWithJoint()
        {
@ -104,13 +107,13 @@ namespace Unity.MLAgents.Extensions.Tests.Sensors
            {
                // Model space
                0f, 0f, 0f, // Root pos
-                13.37f, 0f, 0f, // Middle pos
+                middleGamObj.transform.position.x, 0f, 0f, // Middle pos
                leafGameObj.transform.position.x, 0f, 0f, // Leaf pos

                // Local space
                0f, 0f, 0f, // Root pos
-                13.37f, 0f, 0f, // Attached pos
-                4.2f, 0f, 0f, // Leaf pos
+                middleGamObj.transform.localPosition.x, 0f, 0f, // Attached pos
+                leafGameObj.transform.localPosition.x, 0f, 0f, // Leaf pos

                1f, 0f, 0f, // Root vel (relative to virtual root)
                -1f, 1f, 0f, // Attached vel
@ -139,5 +142,6 @@ namespace Unity.MLAgents.Extensions.Tests.Sensors
            SensorTestHelper.CompareObservation(sensor, expected);
            Assert.AreEqual(expected.Length, sensor.GetObservationSpec().Shape[0]);
        }
+#endif
    }
 }
--- a/com.unity.ml-agents/Runtime/SensorHelper.cs
+++ b/com.unity.ml-agents/Runtime/SensorHelper.cs
@ -51,16 +51,22 @@ namespace Unity.MLAgents.Sensors
            }

            sensor.Write(writer);
+            bool mismatch = false;
+            errorMessage = null;
            for (var i = 0; i < output.Length; i++)
            {
                if (expected[i] != output[i])
                {
-                    errorMessage = $"Expected and actual differed in position {i}. Expected: {expected[i]}  Actual: {output[i]} ";
-                    return false;
+                    string error = $"Expected and actual differed in position {i}. Expected: {expected[i]}  Actual: {output[i]} ";
+                    errorMessage = !mismatch ? error : $"{errorMessage}\n{error}";
+                    mismatch = true;
                }
            }
+            if (mismatch)
+            {
+                return false;
+            }

-            errorMessage = null;
            return true;
        }

--- a/docs/Migrating.md
+++ b/docs/Migrating.md
@ -20,7 +20,7 @@ from mlagents_envs.envs.unity_gym_env import UnityToGymWrapper

 ## Migrating the package to version 3.x
 - The official version of Unity ML-Agents supports is now 2023.2. If you run
-  into issues, please consider deleting your project's Library folder and reponening your
+  into issues, please consider deleting your project's Library folder and reopening your
  project.