@wip Add a CI job to retag MRI tests

ci.jsonnet

@@ -393,6 +393,11 @@ local part_definitions = {
       is_after+:: ["$.run.generate_native_config"],
       run+: jt(["check_native_configuration"]) + jt(["check_config_header"]),
     },
+
+    retag_mri_tests: {
+      run+: jt(["retag", "test/mri/tests/**/*test*.rb"]) +
+            [["git", "commit", "-a", "-m", "Retag MRI tests"], ["git", "push"]]
+    },
   },
 
   benchmark: {
@@ -698,6 +703,8 @@ local composition_environment = utils.add_inclusion_tracking(part_definitions, "
     "ruby-generate-native-config-linux-aarch64":  $.platform.linux_aarch64 + $.jdk.latest + shared + native_config,
     "ruby-generate-native-config-darwin-amd64":   $.platform.darwin_amd64 + $.jdk.latest + shared + native_config,
     "ruby-generate-native-config-darwin-aarch64": $.platform.darwin_aarch64 + $.jdk.latest + shared + native_config,
+
+    "ruby-retag-mri-tests-linux-amd64":           $.platform.linux + $.jdk.latest + shared + $.env.native + $.use.build + $.run.retag_mri_tests,
   },
 
   builds:

test/mri/failing.exclude

@@ -75,6 +75,8 @@ dtrace/test_singleton_function.rb
 dtrace/test_string.rb
 misc/test_ruby_mode.rb
 ruby/enc/test_windows_1251.rb
+ruby/test_ast.rb # relies on RubyVM::AbstractSyntaxTree
+ruby/test_compile_prism.rb # relies on RubyVM::InstructionSequence
 ruby/test_iseq.rb # RubyVM
 ruby/test_rubyvm.rb
 ruby/test_bignum.rb # relies on Bignums on Fixnum range

tool/jt.rb

@@ -1386,7 +1386,12 @@ def retag(*args)
     files_to_retag.each do |test_file|
       puts '', test_file
       test_classes = File.read(test_file).scrub.scan(/class\s+([\w:]+)\s*<.+Test/).map(&:first) # see test/mri/tests/mkmf/test_config.rb, test/mri/tests/rdoc/test_rdoc_alias.rb...
-      raise "Could not find class inheriting from TestCase in #{test_file}" if test_classes.empty?
+
+      if test_classes.empty?
+        puts "\nCould not find class inheriting from TestCase in #{test_file}"
+        next
+      end
+
       found_excludes = false
       test_classes.each do |test_class|
         prefix = "test/mri/excludes/#{test_class.gsub('::', '/')}"
@@ -1420,11 +1425,31 @@ def retag(*args)
       end
 
       process_tests = true
+      forced_exit_count = 0
       while process_tests
         puts '1. Tagging tests'
         output_file = 'mri_tests.txt'
         run_mri_tests(options, [test_file], [], [:err, :out] => output_file, continue_on_failure: true)
 
+        puts "*"*80
+        puts "*** mri_tests.txt:"
+        puts File.read(output_file)
+        puts "*"*80
+
+        # handle tests timeout on CI - it could be transient and we try to restart it several times
+        # exit status after SIGKILL/SIGQUIT (e.g. by timeout on CI) is 128 + 9 or 128 + 3
+        if $LAST_SH_STATUS.termsig
+          puts "Child process was terminated with signal #{$LAST_SH_STATUS.termsig}"
+          forced_exit_count += 1
+          if forced_exit_count <= 3
+            puts "Restart tagging a #{test_file} file because of forced_exit_count = #{forced_exit_count}"
+            process_tests = true
+            next
+          else
+            puts "Stop restarting tagging a #{test_file} file because forced_exit_count has reached the limit"
+          end
+        end
+
         puts '2. Parsing errors'
         sh 'ruby', 'tool/parse_mri_errors.rb', output_file, continue_on_failure: true
 

tool/parse_mri_errors.rb

@@ -42,6 +42,12 @@ def platform_info
 def exclude_test!(class_name, test_method, error_display, platform = nil)
   file = EXCLUDES_DIR + "/" + class_name.split("::").join('/') + ".rb"
 
+  # If a method name breaks the rules for method naming, e.g. it's defined dynamically with #define_method,
+  # in the tests output it's wrapped into "...". The patterns may not remove `""` so do this explicitly.
+  if test_method[0] == '"' && test_method[-1] == '"'
+    test_method = test_method[1..-2]
+  end
+
   # A method name can contain escape sequences e.g. \t, \v etc (when a method is created
   # with #define_method method call). Such method name is escaped when being printed in
   # the test output. So it should be unescaped to match an actual Ruby method name.
@@ -131,15 +137,27 @@ module Patterns
   #
   # Sample: [1/3] TestBignum_Big2str#test_big2str_generictest/mri/tests/runner.rb: TestBignum_Big2str#test_big2str_generic: symbol lookup error: /home/nirvdrum/dev/workspaces/truffleruby-ws/truffleruby/.ext/c/bignum.so: undefined symbol: rb_big2str_generic
   # Extracts: ['TestBignum_Big2str', 'test_big2str', 'rb_big2str_generic']
+  #
+  # Sample: test/mri/tests/runner.rb: TestRDocParserRuby#test_read_directive_one_liner: symbol lookup error: /b/b/e/main/mxbuild/truffleruby-native/lib/mri/ripper.so: undefined symbol: rb_parser_st_locale_insensitive_strncasecmp
+  # Extracts: ['TestRDocParserRuby', 'test_read_directive_one_liner', 'rb_parser_st_locale_insensitive_strncasecmp']
   MISSING_SYMBOL = / ((?:\w+::)*\w+)#(\w+): symbol lookup error.*? undefined symbol: (\w+)/
 
   # Sample:  [1/4] TestThreadInstrumentation#test_join_counters/home/nirvdrum/dev/workspaces/truffleruby-ws/truffleruby/mxbuild/truffleruby-jvm-ce/bin/ruby: symbol lookup error: /home/nirvdrum/dev/workspaces/truffleruby-ws/truffleruby/.ext/c/thread/instrumentation.so: undefined symbol: rb_internal_thread_add_event_hook
   # Extracts: ['TestThreadInstrumentation', 'test_join', 'rb_internal_thread_add_event_hook']
-  SYMBOL_LOOKUP_ERROR = / ((?:\w+::)*\w+)#(\w+?)(?:\/.*?)?: symbol lookup error.*? undefined symbol: (\w+)/
+  #
+  # Sample:  [1/4] TestRDocAnyMethod#test_has_call_seq?/home/nirvdrum/dev/workspaces/truffleruby-ws/truffleruby/mxbuild/truffleruby-jvm-ce/bin/ruby: symbol lookup error: /home/nirvdrum/dev/workspaces/truffleruby-ws/truffleruby/.ext/c/thread/instrumentation.so: undefined symbol: rb_internal_thread_add_event_hook'
+  # Extracts: ['TestRDocAnyMethod', 'test_has_call_seq?', 'rb_internal_thread_add_event_hook']
+  #
+  # Sample:  [1/4] TestRDocAnyMethod#"test_resolve_method:!"/home/nirvdrum/dev/workspaces/truffleruby-ws/truffleruby/mxbuild/truffleruby-jvm-ce/bin/ruby: symbol lookup error: /home/nirvdrum/dev/workspaces/truffleruby-ws/truffleruby/.ext/c/thread/instrumentation.so: undefined symbol: rb_internal_thread_add_event_hook'
+  # Extracts: ['TestRDocAnyMethod', 'test_resolve_method:!', 'rb_internal_thread_add_event_hook']
+  #
+  # Sample:  [1/4] TestRDocAnyMethod#"test_resolve_method:/"/home/nirvdrum/dev/workspaces/truffleruby-ws/truffleruby/mxbuild/truffleruby-jvm-ce/bin/ruby: symbol lookup error: /home/nirvdrum/dev/workspaces/truffleruby-ws/truffleruby/.ext/c/thread/instrumentation.so: undefined symbol: rb_internal_thread_add_event_hook'
+  # Extracts: ['TestRDocAnyMethod', 'test_resolve_method:/', 'rb_internal_thread_add_event_hook']
+  SYMBOL_LOOKUP_ERROR = / ((?:\w+::)*\w+)#(\w+[?!]?|"\w+[^"\n]*?")(?:\/.*?)?: symbol lookup error.*? undefined symbol: (\w+)/
 
   # Sample: [ 35/123] TestFileExhaustive#test_expand_path_hfsdyld[32447]: missing symbol called
   # Extracts: ['TestFileExhaustive', 'test_expand_path_hfs', 'missing symbol called']
-  DYLD_MISSING_SYMBOL = / ((?:\w+::)*\w+)#"?(\w+?[^"\n]*)"?dyld\[\d+\]: (.*)/
+  DYLD_MISSING_SYMBOL = / ((?:\w+::)*\w+)#"?(\w+?[^"\n]*?)"?dyld\[\d+\]: (.*)/
 
   # Sample: [29/39] TestDir#test_instance_chdircannot return the original directory: /Users/andrykonchin/projects/truffleruby-ws/truffleruby
   # Extracts: ['TestDir', 'test_instance_chdircannot', 'cannot return the original directory']