Add Launchable into CI

.github/workflows/macos.yml

@@ -13,6 +13,21 @@ on:
     # https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/collaborating-on-repositories-with-code-quality-features/troubleshooting-required-status-checks#handling-skipped-but-required-checks
   merge_group:
 
+env:
+  # GITHUB_PULL_REQUEST_URL are used for commenting test reports in Launchable Github App.
+  # https://github.com/launchableinc/cli/blob/v1.80.1/launchable/utils/link.py#L42
+  GITHUB_PULL_REQUEST_URL: ${{ github.event.pull_request.html_url }}
+  # The following envs are necessary in Launchable tokenless authentication.
+  # https://github.com/launchableinc/cli/blob/v1.80.1/launchable/utils/authentication.py#L20
+  LAUNCHABLE_ORGANIZATION: ${{ github.repository_owner }}
+  LAUNCHABLE_WORKSPACE: ${{ github.event.repository.name }}
+  # https://github.com/launchableinc/cli/blob/v1.80.1/launchable/utils/authentication.py#L71
+  GITHUB_PR_HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
+  # This secret setting is needed if you want to run Launchable on your forked
+  # repository.
+  # See https://github.com/ruby/ruby/wiki/CI-Servers#launchable-ci for details.
+  LAUNCHABLE_TOKEN: ${{ secrets.LAUNCHABLE_TOKEN }}
+
 concurrency:
   group: ${{ github.workflow }} / ${{ startsWith(github.event_name, 'pull') && github.ref_name || github.sha }}
   cancel-in-progress: ${{ startsWith(github.event_name, 'pull') }}
@@ -25,12 +40,14 @@ jobs:
     strategy:
       matrix:
         test_task: ['check']
+        test_opts: ['']
         os:
           - macos-12
           - macos-13
           - ${{ github.repository == 'ruby/ruby' && 'macos-arm-oss' || 'macos-14' }}
         include:
-          - test_task: test-all TESTS=--repeat-count=2
+          - test_task: test-all
+            test_opts: --repeat-count=2
             os: ${{ github.repository == 'ruby/ruby' && 'macos-arm-oss' || 'macos-14' }}
           - test_task: test-bundled-gems
             os: ${{ github.repository == 'ruby/ruby' && 'macos-arm-oss' || 'macos-14' }}
@@ -50,10 +67,23 @@ jobs:
       )}}
 
     steps:
+      - name: Enable Launchable conditionally
+        id: enable_launchable
+        run: echo "enable_launchable=true" >> $GITHUB_OUTPUT
+        working-directory:
+        if: >-
+          ${{
+          (github.repository == 'ruby/ruby' ||
+          (github.repository != 'ruby/ruby' && env.LAUNCHABLE_TOKEN)) &&
+          (matrix.test_task == 'check' || matrix.test_task == 'test-all')
+          }}
+
       - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
         with:
           sparse-checkout-cone-mode: false
           sparse-checkout: /.github
+          # Set fetch-depth: 0 so that Launchable can receive commits information.
+          fetch-depth: 0
 
       - name: Install libraries
         uses: ./.github/actions/setup/macos
@@ -80,6 +110,41 @@ jobs:
           echo "TESTS=${TESTS}" >> $GITHUB_ENV
         if: ${{ matrix.test_task == 'check' && matrix.skipped_tests }}
 
+      # Launchable CLI requires Python and Java
+      # https://www.launchableinc.com/docs/resources/cli-reference/
+      - name: Set up Python
+        uses: actions/setup-python@871daa956ca9ea99f3c3e30acb424b7960676734 # v5.0.0
+        with:
+          python-version: "3.10"
+        if: steps.enable_launchable.outputs.enable_launchable
+
+      - name: Set up Java
+        uses: actions/setup-java@7a445ee88d4e23b52c33fdc7601e40278616c7f8 # v4.0.0
+        with:
+          distribution: 'temurin'
+          java-version: '17'
+        if: steps.enable_launchable.outputs.enable_launchable
+
+      - name: Set up Launchable
+        run: |
+          set -x
+          pip install launchable
+          launchable verify
+          : # The build name cannot include a slash, so we replace the string here.
+          github_ref="$(echo ${{ github.ref }} | sed 's/\//_/g')"
+          : # With the --name option, we need to configure a unique identifier for this build.
+          : # To avoid setting the same build name as the CI which runs on other branches, we use the branch name here.
+          : #
+          : # FIXME: Need to fix `WARNING: Failed to process a change to a file`.
+          : # https://github.com/launchableinc/cli/issues/786
+          launchable record build --name ${github_ref}_${GITHUB_PR_HEAD_SHA}
+          echo "TESTS=${TESTS} --launchable-test-reports=launchable_reports.json" >> $GITHUB_ENV
+        if: steps.enable_launchable.outputs.enable_launchable
+
+      - name: Set extra test options
+        run: echo "TESTS=$TESTS ${{ matrix.test_opts }}" >> $GITHUB_ENV
+        if: matrix.test_opts
+
       - name: make ${{ matrix.test_task }}
         run: |
           make -s ${{ matrix.test_task }} ${TESTS:+TESTS="$TESTS"}
@@ -99,6 +164,10 @@ jobs:
         if: ${{ matrix.test_task == 'check' && matrix.skipped_tests }}
         continue-on-error: ${{ matrix.continue-on-skipped_tests || false }}
 
+      - name: Launchable - record tests
+        run: launchable record tests --flavor os=${{ matrix.os }} --flavor test_task=${{ matrix.test_task }} raw launchable_reports.json
+        if: ${{ always() && steps.enable_launchable.outputs.enable_launchable }}
+
       - uses: ./.github/actions/slack
         with:
           label: ${{ matrix.os }} / ${{ matrix.test_task }}

tool/lib/test/unit.rb

@@ -842,7 +842,7 @@ module Test
         end
       end
 
-      def record(suite, method, assertions, time, error)
+      def record(suite, method, assertions, time, error, source_location = nil)
         if @options.values_at(:longest, :most_asserted).any?
           @tops ||= {}
           rec = [suite.name, method, assertions, time, error]
@@ -854,38 +854,6 @@ module Test
           end
         end
         # (((@record ||= {})[suite] ||= {})[method]) = [assertions, time, error]
-        if writer = @options[:launchable_test_reports]
-          location = suite.instance_method(method).source_location
-          if location && path = location.first
-            # Launchable JSON schema is defined at
-            # https://github.com/search?q=repo%3Alaunchableinc%2Fcli+https%3A%2F%2Flaunchableinc.com%2Fschema%2FRecordTestInput&type=code.
-            e = case error
-                when nil
-                  status = 'TEST_PASSED'
-                  nil
-                when Test::Unit::PendedError
-                  status = 'TEST_SKIPPED'
-                  "Skipped:\n#{klass}##{meth} [#{location e}]:\n#{e.message}\n"
-                when Test::Unit::AssertionFailedError
-                  status = 'TEST_FAILED'
-                  "Failure:\n#{klass}##{meth} [#{location e}]:\n#{e.message}\n"
-                when Timeout::Error
-                  status = 'TEST_FAILED'
-                  "Timeout:\n#{klass}##{meth}\n"
-                else
-                  status = 'TEST_FAILED'
-                  bt = Test::filter_backtrace(e.backtrace).join "\n    "
-                  "Error:\n#{klass}##{meth}:\n#{e.class}: #{e.message.b}\n    #{bt}\n"
-                end
-            writer.write_object do
-              writer.write_key_value('testPath', "file=#{path}#class=#{suite.name}#testcase=#{method}",)
-              writer.write_key_value('status', status)
-              writer.write_key_value('duration', time)
-              writer.write_key_value('createdAt', Time.now)
-              writer.write_key_value('stderr', e) if e
-            end
-          end
-        end
         super
       end
 
@@ -914,104 +882,6 @@ module Test
         opts.on '--most-asserted=N', Integer, 'Show most asserted N tests' do |n|
           options[:most_asserted] = n
         end
-        opts.on '--launchable-test-reports=PATH', String, 'Report test results in Launchable JSON format' do |path|
-          require 'json'
-          options[:launchable_test_reports] = writer = JsonStreamWriter.new(path)
-          writer.write_array('testCases')
-          at_exit{ writer.close }
-        end
-      end
-      ##
-      # JsonStreamWriter writes a JSON file using a stream.
-      # By utilizing a stream, we can minimize memory usage, especially for large files.
-      class JsonStreamWriter
-        def initialize(path)
-          @file = File.open(path, "w")
-          @file.write("{")
-          @indent_level = 0
-          @is_first_key_val = true
-          @is_first_obj = true
-          write_new_line
-        end
-
-        def write_object
-          if @is_first_obj
-            @is_first_obj = false
-          else
-            write_comma
-            write_new_line
-          end
-          @indent_level += 1
-          write_indent
-          @file.write("{")
-          write_new_line
-          @indent_level += 1
-          yield
-          @indent_level -= 1
-          write_new_line
-          write_indent
-          @file.write("}")
-          @indent_level -= 1
-          @is_first_key_val = true
-        end
-
-        def write_array(key)
-          @indent_level += 1
-          write_indent
-          @file.write(to_json_str(key))
-          write_colon
-          @file.write(" ", "[")
-          write_new_line
-        end
-
-        def write_key_value(key, value)
-          if @is_first_key_val
-            @is_first_key_val = false
-          else
-            write_comma
-            write_new_line
-          end
-          write_indent
-          @file.write(to_json_str(key))
-          write_colon
-          @file.write(" ")
-          @file.write(to_json_str(value))
-        end
-
-        def close
-          close_array
-          @indent_level -= 1
-          write_new_line
-          @file.write("}")
-        end
-
-        private
-        def to_json_str(obj)
-          JSON.dump(obj)
-        end
-
-        def write_indent
-          @file.write(" " * 2 * @indent_level)
-        end
-
-        def write_new_line
-          @file.write("\n")
-        end
-
-        def write_comma
-          @file.write(',')
-        end
-
-        def write_colon
-          @file.write(":")
-        end
-
-        def close_array
-          write_new_line
-          write_indent
-          @file.write("]")
-          @indent_level -= 1
-        end
       end
     end
 
@@ -1483,6 +1353,198 @@ module Test
       end
     end
 
+    module LaunchableOption
+      module Nothing
+        private
+        def setup_options(opts, options)
+          super
+          opts.define_tail 'Launchable options:'
+          # This is expected to be called by Test::Unit::Worker.
+          opts.on_tail '--launchable-test-reports=PATH', String, 'Do nothing'
+        end
+      end
+
+      def record(suite, method, assertions, time, error, source_location = nil)
+        if writer = @options[:launchable_test_reports]
+          if path = (source_location || suite.instance_method(method).source_location).first
+            # Launchable JSON schema is defined at
+            # https://github.com/search?q=repo%3Alaunchableinc%2Fcli+https%3A%2F%2Flaunchableinc.com%2Fschema%2FRecordTestInput&type=code.
+            e = case error
+                when nil
+                  status = 'TEST_PASSED'
+                  nil
+                when Test::Unit::PendedError
+                  status = 'TEST_SKIPPED'
+                  "Skipped:\n#{suite.name}##{method} [#{location error}]:\n#{error.message}\n"
+                when Test::Unit::AssertionFailedError
+                  status = 'TEST_FAILED'
+                  "Failure:\n#{suite.name}##{method} [#{location error}]:\n#{error.message}\n"
+                when Timeout::Error
+                  status = 'TEST_FAILED'
+                  "Timeout:\n#{suite.name}##{method}\n"
+                else
+                  status = 'TEST_FAILED'
+                  bt = Test::filter_backtrace(error.backtrace).join "\n    "
+                  "Error:\n#{suite.name}##{method}:\n#{error.class}: #{error.message.b}\n    #{bt}\n"
+                end
+            repo_path = File.expand_path("#{__dir__}/../../../")
+            relative_path = path.delete_prefix("#{repo_path}/")
+            # The test path is a URL-encoded representation.
+            # https://github.com/launchableinc/cli/blob/v1.81.0/launchable/testpath.py#L18
+            test_path = {file: relative_path, class: suite.name, testcase: method}.map{|key, val|
+              "#{encode_test_path_component(key)}=#{encode_test_path_component(val)}"
+            }.join('#')
+          end
+        end
+        super
+      ensure
+        if writer && test_path && status
+          # Occasionally, the file writing operation may be paused, especially when `--repeat-count` is specified.
+          # In such cases, we proceed to execute the operation here.
+          writer.write_object do
+            writer.write_key_value('testPath', test_path)
+            writer.write_key_value('status', status)
+            writer.write_key_value('duration', time)
+            writer.write_key_value('createdAt', Time.now.to_s)
+            writer.write_key_value('stderr', e)
+            writer.write_key_value('stdout', nil)
+          end
+        end
+      end
+
+      private
+      def setup_options(opts, options)
+        super
+        opts.on_tail '--launchable-test-reports=PATH', String, 'Report test results in Launchable JSON format' do |path|
+          require 'json'
+          require 'uri'
+          options[:launchable_test_reports] = writer = JsonStreamWriter.new(path)
+          writer.write_array('testCases')
+          main_pid = Process.pid
+          at_exit {
+            # This block is executed when the fork block in a test is completed.
+            # Therefore, we need to verify whether all tests have been completed.
+            stack = caller
+            if stack.size == 0 && main_pid == Process.pid && $!.is_a?(SystemExit)
+              writer.close
+            end
+          }
+        end
+
+        def encode_test_path_component component
+          component.to_s.gsub('%', '%25').gsub('=', '%3D').gsub('#', '%23').gsub('&', '%26')
+        end
+      end
+
+      ##
+      # JsonStreamWriter writes a JSON file using a stream.
+      # By utilizing a stream, we can minimize memory usage, especially for large files.
+      class JsonStreamWriter
+        def initialize(path)
+          @file = File.open(path, "w")
+          @file.write("{")
+          @indent_level = 0
+          @is_first_key_val = true
+          @is_first_obj = true
+          write_new_line
+        end
+
+        def write_object
+          if @is_first_obj
+            @is_first_obj = false
+          else
+            write_comma
+            write_new_line
+          end
+          @indent_level += 1
+          write_indent
+          @file.write("{")
+          write_new_line
+          @indent_level += 1
+          yield
+          @indent_level -= 1
+          write_new_line
+          write_indent
+          @file.write("}")
+          @indent_level -= 1
+          @is_first_key_val = true
+          # Occasionally, invalid JSON will be created as shown below, especially when `--repeat-count` is specified.
+          # {
+          #   "testPath": "file=test%2Ftest_timeout.rb&class=TestTimeout&testcase=test_allows_zero_seconds",
+          #   "status": "TEST_PASSED",
+          #   "duration": 2.7e-05,
+          #   "createdAt": "2024-02-09 12:21:07 +0000",
+          #   "stderr": null,
+          #   "stdout": null
+          # }: null <- here
+          # },
+          # To prevent this, IO#flush is called here.
+          @file.flush
+        end
+
+        def write_array(key)
+          @indent_level += 1
+          write_indent
+          @file.write(to_json_str(key))
+          write_colon
+          @file.write(" ", "[")
+          write_new_line
+        end
+
+        def write_key_value(key, value)
+          if @is_first_key_val
+            @is_first_key_val = false
+          else
+            write_comma
+            write_new_line
+          end
+          write_indent
+          @file.write(to_json_str(key))
+          write_colon
+          @file.write(" ")
+          @file.write(to_json_str(value))
+        end
+
+        def close
+          return if @file.closed?
+          close_array
+          @indent_level -= 1
+          write_new_line
+          @file.write("}")
+          @file.flush
+          @file.close
+        end
+
+        private
+        def to_json_str(obj)
+          JSON.dump(obj)
+        end
+
+        def write_indent
+          @file.write(" " * 2 * @indent_level)
+        end
+
+        def write_new_line
+          @file.write("\n")
+        end
+
+        def write_comma
+          @file.write(',')
+        end
+
+        def write_colon
+          @file.write(":")
+        end
+
+        def close_array
+          write_new_line
+          write_indent
+          @file.write("]")
+          @indent_level -= 1
+        end
+      end
+    end
+
     class Runner # :nodoc: all
 
       attr_accessor :report, :failures, :errors, :skips # :nodoc:
@@ -1720,13 +1782,13 @@ module Test
       # failure or error in teardown, it will be sent again with the
       # error or failure.
 
-      def record suite, method, assertions, time, error
+      def record suite, method, assertions, time, error, source_location = nil
       end
 
       def location e # :nodoc:
         last_before_assertion = ""
 
-        return '<empty>' unless e.backtrace # SystemStackError can return nil.
+        return '<empty>' unless e&.backtrace # SystemStackError can return nil.
 
         e.backtrace.reverse_each do |s|
           break if s =~ /in .(?:Test::Unit::(?:Core)?Assertions#)?(assert|refute|flunk|pass|fail|raise|must|wont)/
@@ -1811,6 +1873,7 @@ module Test
       prepend Test::Unit::ExcludesOption
       prepend Test::Unit::TimeoutOption
       prepend Test::Unit::RunCount
+      prepend Test::Unit::LaunchableOption::Nothing
 
       ##
       # Begins the full test run. Delegates to +runner+'s #_run method.
@@ -1867,6 +1930,7 @@ module Test
     class AutoRunner # :nodoc: all
       class Runner < Test::Unit::Runner
         include Test::Unit::RequireFiles
+        include Test::Unit::LaunchableOption
       end
 
       attr_accessor :to_run, :options

tool/lib/test/unit/parallel.rb

@@ -180,7 +180,7 @@ module Test
         else
           error = ProxyError.new(error)
         end
-        _report "record", Marshal.dump([suite.name, method, assertions, time, error])
+        _report "record", Marshal.dump([suite.name, method, assertions, time, error, suite.instance_method(method).source_location])
         super
       end
     end