From 3a99c76761c6e0aaddabf5471c14347f687292c5 Mon Sep 17 00:00:00 2001 From: nick evans Date: Fri, 24 Jul 2026 13:34:34 -0400 Subject: [PATCH 1/4] =?UTF-8?q?=E2=9C=85=20continue-on-error=20when=20cove?= =?UTF-8?q?rage=20metrics=20fail?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 32579ec1..d7de17cb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -42,3 +42,4 @@ jobs: !startsWith(matrix.ruby, 'truffleruby') && !startsWith(matrix.ruby, 'jruby') }} run: bundle exec rake coverage:report timeout-minutes: 1 # _should_ finish in under a second + continue-on-error: true From 935510f16d436a02feff8eaba09d4d03692a6e95 Mon Sep 17 00:00:00 2001 From: nick evans Date: Thu, 23 Jul 2026 14:16:33 -0400 Subject: [PATCH 2/4] =?UTF-8?q?=E2=9C=85=20Ratchet=20up=20the=20minimum=20?= =?UTF-8?q?coverage=20requirements?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since simplecov v1.0 makes it easier to specify more detailed coverage requirements, I've raised the requirements to be just below the current coverage. --- Rakefile | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/Rakefile b/Rakefile index d17e1ceb..807c4203 100644 --- a/Rakefile +++ b/Rakefile @@ -17,8 +17,56 @@ task "coverage:report" do require "simplecov" SimpleCov.collate "coverage/.resultset.json" do coverage(:line) do - minimum 90 - minimum_per_file 40 + minimum 95 + + minimum_per_group 98, only: "Config" + minimum_per_group 97, only: "StringPrep" + minimum_per_group 97, only: "SASL" + minimum_per_group 95, only: "Data Types" + minimum_per_group 94, only: "Parser" + minimum_per_group 92, only: "Client" + + minimum_per_file 90 + minimum_per_file 87, only: "lib/net/imap/sasl/authenticators.rb" + minimum_per_file 86, only: "lib/net/imap/config/attr_type_coercion.rb" + minimum_per_file 84, only: "lib/net/imap/authenticators.rb" + minimum_per_file 80, only: "lib/net/imap/response_data.rb" + minimum_per_file 55, only: "lib/net/imap/search_result.rb" + end + + coverage(:branch) do + minimum 80 + + minimum_per_group 92, only: "Data Types" + minimum_per_group 87, only: "Config" + minimum_per_group 83, only: "Client" + minimum_per_group 81, only: "Parser" + minimum_per_group 76, only: "SASL" + minimum_per_group 73, only: "StringPrep" + + minimum_per_file 65 + minimum_per_file 62, only: "lib/net/imap/sasl/scram_authenticator.rb" + minimum_per_file 50, only: "lib/net/imap/sasl/authenticators.rb" + minimum_per_file 50, only: "lib/net/imap/config/attr_accessors.rb" + end + + coverage(:method) do + minimum 88 + + minimum_per_group 100, only: "Config" + minimum_per_group 92, only: "Data Types" + minimum_per_group 90, only: "StringPrep" + minimum_per_group 88, only: "Client" + minimum_per_group 83, only: "Parser" + minimum_per_group 82, only: "SASL" + + minimum_per_file 66 + minimum_per_file 64, only: "lib/net/imap/response_parser/parser_utils.rb" + minimum_per_file 57, only: "lib/net/imap/sasl/authenticators.rb" + minimum_per_file 50, only: "lib/net/imap/authenticators.rb" + minimum_per_file 50, only: "lib/net/imap/sasl/anonymous_authenticator.rb" + minimum_per_file 36, only: "lib/net/imap/sasl/protocol_adapters.rb" + minimum_per_file 20, only: "lib/net/imap/response_data.rb" end end end From d069d129520f05d48207fbec5021a7c4f664f332 Mon Sep 17 00:00:00 2001 From: nick evans Date: Thu, 23 Jul 2026 13:21:07 -0400 Subject: [PATCH 3/4] =?UTF-8?q?=E2=9C=85=20Use=20custom=20SimpleCov=20form?= =?UTF-8?q?atter=20by=20default?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is much faster than the HTML formatter, _and_ its stderr output is more informative. The HTML formatter is still available via `rake coverage:report`. --- .simplecov | 2 - Rakefile | 4 +- test/lib/helper.rb | 4 + .../lib/simplecov_markdown_table_formatter.rb | 88 +++++++++++++++++++ 4 files changed, 95 insertions(+), 3 deletions(-) create mode 100644 test/lib/simplecov_markdown_table_formatter.rb diff --git a/.simplecov b/.simplecov index c58b248a..b6051e90 100644 --- a/.simplecov +++ b/.simplecov @@ -1,8 +1,6 @@ # frozen_string_literal: true SimpleCov.configure do - formatter SimpleCov::Formatter::HTMLFormatter - enable_coverage :branch enable_coverage :method enable_coverage :eval diff --git a/Rakefile b/Rakefile index 807c4203..8332e8ad 100644 --- a/Rakefile +++ b/Rakefile @@ -12,10 +12,12 @@ end task :default => :test -desc "Output coverage data report, and error when threshholds aren't met" +desc "Output HTML coverage data report, and error when threshholds aren't met" task "coverage:report" do require "simplecov" SimpleCov.collate "coverage/.resultset.json" do + formatter SimpleCov::Formatter::HTMLFormatter + coverage(:line) do minimum 95 diff --git a/test/lib/helper.rb b/test/lib/helper.rb index e37a1b0f..8aefab2b 100644 --- a/test/lib/helper.rb +++ b/test/lib/helper.rb @@ -3,6 +3,10 @@ require "simplecov" + # run `rake coverage:report` for HTML/JSON formatter + require_relative "simplecov_markdown_table_formatter" + SimpleCov.formatter = SimpleCovMarkdownTableFormatter + SimpleCov.start do command_name "Net::IMAP tests" end diff --git a/test/lib/simplecov_markdown_table_formatter.rb b/test/lib/simplecov_markdown_table_formatter.rb new file mode 100644 index 00000000..b3909ed7 --- /dev/null +++ b/test/lib/simplecov_markdown_table_formatter.rb @@ -0,0 +1,88 @@ +# frozen_string_literal: true + +require "simplecov" + +# This formatter is much faster than the HTML formatter, and it prints much more +# useful info to stderr than any of the bundled formatters. +# +# TODO: extract this to its own gem? it needs some basic tests +# TODO: output to ENV["GITHUB_STEP_OUTPUT"] +class SimpleCovMarkdownTableFormatter < SimpleCov::Formatter::Base + def format(result) + return if @silent + output = "### Coverage report for #{result.command_name}\n" + output << format_markdown_table(result) + $stderr.puts output + output + end + + def format_markdown_table(result) + groups = { "All files" => result }.merge(result.groups) + .transform_values(&:coverage_statistics) + + name_size = groups.keys.map(&:length).max + criteria_sizes = groups + .values.map { _1.transform_values(&:total) } #=> Array[Hash[name, total]] + .reduce { _1.merge(_2) {|_, a, b| [a, b].max } } #=> Hash[name, max] + .transform_values { _1.to_s.length } #=> Hash[name, strlen] + + rows = format_markdown_table_header(name_size, criteria_sizes) + + rows.concat groups.map {|name, stats| + format_row_cells(name, stats, name_size, criteria_sizes) + } + .map { format_markdown_table_row _1 } + + rows.join("\n") + end + + private + + def format_markdown_table_header(name_size, criteria_sizes) + heading_cells = format_markdown_table_row([ + "Group".center(name_size), + *criteria_sizes.map {|name, size| + "#{name.to_s.capitalize} coverage".center(column_width(size)) + } + ]) + border_line = format_row_cells("", name_size, criteria_sizes) + .then { format_markdown_table_row _1 } + .then { _1.tr(" ", "-") } + [heading_cells, border_line] + end + + def format_markdown_table_row(cells) = "| #{cells.join(" | ")} |" + + def format_row_cells(name, stats = {}, name_size, criteria_sizes) + name = name.ljust(name_size) + cols = criteria_sizes.map {|criterion, size| + format_stat_column(stats[criterion], size) + } + [name, *cols] + end + + def column_width(size) = 10 + size * 2 + 1 # "000.00% = " + 2*size + "/" + + def colorize_percent(percent) + formatted = "%6.2f%%" % [percent] + color = SimpleCov::Color.for_percent(percent) + SimpleCov::Color.colorize(formatted, color) + end + + def format_stat_column(stat, size) + if stat + "%%s = %%%{size}s/%%%{size}s" % {size:} % stat_values(stat) + else + "" + end + .ljust(column_width(size)) + end + + # converts SimpleCov::CoverageStatistics to [String, String, String] + def stat_values(stat) + return ["0", "0", colorize_percent(0.0)] unless stat + percent = colorize_percent SimpleCov.round_coverage(stat.percent) + [percent, stat.covered, stat.total] + end + +end From 5776679e0d30d53a6314c19a1346327ddc660daf Mon Sep 17 00:00:00 2001 From: nick evans Date: Fri, 24 Jul 2026 11:48:45 -0400 Subject: [PATCH 4/4] =?UTF-8?q?=E2=9C=85=20Don't=20require=20minimum=20cov?= =?UTF-8?q?erage=20on=20ruby-head?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These numbers are _very_ different, for some reason (maybe a bug?). --- .github/workflows/test.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d7de17cb..980787fc 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -38,8 +38,11 @@ jobs: timeout-minutes: 5 # _should_ finish in under a minute - name: Report coverage - if: ${{ matrix.os == 'ubuntu-latest' && github.event_name != 'pull_request' && - !startsWith(matrix.ruby, 'truffleruby') && !startsWith(matrix.ruby, 'jruby') }} + if: ${{ matrix.os == 'ubuntu-latest' && + github.event_name != 'pull_request' && + matrix.ruby != "ruby-head" && + !startsWith(matrix.ruby, 'truffleruby') && + !startsWith(matrix.ruby, 'jruby') }} run: bundle exec rake coverage:report timeout-minutes: 1 # _should_ finish in under a second continue-on-error: true