diff --git a/.github/workflows/rspec.yml b/.github/workflows/rspec.yml new file mode 100644 index 0000000000..b3b936d3ba --- /dev/null +++ b/.github/workflows/rspec.yml @@ -0,0 +1,34 @@ +name: Run all page tests + +# Run when a PR is opened, any branch is pushed +# Also allow manually triggering workflows, and running if a release is created. +on: + - pull_request + - push + - workflow_dispatch + - deployment + - release + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: ruby/setup-ruby@v1 + with: + # The Ruby version is determined by either a `.ruby-version` or a `.tool-versions` file in root of the repo. + bundler-cache: true + - name: Run a11y tests + run: | + bundle exec rspec + - name: summary + if: failure() + run: ruby spec/support/spec_summary.rb + - name: Keep screenshots from failed tests + uses: actions/upload-artifact@v4 + if: failure() + with: + name: screenshots + path: ${{ github.workspace }}/tmp/capybara + if-no-files-found: ignore + retention-days: 7 diff --git a/.gitignore b/.gitignore index c3a1190d0d..a0ba544a69 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,6 @@ node_modules *.gem .bundle .ruby-version + +# Used for automated test results and metadata. +tmp/ diff --git a/.rspec b/.rspec new file mode 100644 index 0000000000..1841350fcf --- /dev/null +++ b/.rspec @@ -0,0 +1 @@ +-f documentation -f documentation --out tmp/rspec_output.txt -f json --out tmp/rspec_output.json -f html --out tmp/rspec_output.html diff --git a/Gemfile b/Gemfile index 76bd13913f..5c2a530db9 100644 --- a/Gemfile +++ b/Gemfile @@ -4,5 +4,21 @@ gemspec gem "jekyll-github-metadata", ">= 2.15" gem "jekyll-include-cache", group: :jekyll_plugins +gem "jekyll-sitemap", group: :jekyll_plugins -gem "html-proofer", "~> 5.0", :group => :development +group :development, :test do + gem "html-proofer", "~> 5.0" + + # Test Infrastructure + gem 'rack' + gem 'rackup' + gem 'rspec' + gem 'webrick' + + # Frontend a11y tests + gem 'axe-core-capybara' + gem 'axe-core-rspec' + gem 'capybara' + gem 'capybara-screenshot' + gem 'selenium-webdriver' +end diff --git a/Gemfile.lock b/Gemfile.lock index a2be9b3b84..10ed89ff61 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -19,13 +19,48 @@ GEM fiber-annotation io-event (~> 1.5, >= 1.5.1) timers (~> 4.1) + axe-core-api (4.9.1) + dumb_delegator + virtus + axe-core-capybara (4.9.1) + axe-core-api (= 4.9.1) + dumb_delegator + axe-core-rspec (4.9.1) + axe-core-api (= 4.9.1) + dumb_delegator + virtus + axiom-types (0.1.1) + descendants_tracker (~> 0.0.4) + ice_nine (~> 0.11.0) + thread_safe (~> 0.3, >= 0.3.1) + base64 (0.2.0) bigdecimal (3.1.8) + capybara (3.40.0) + addressable + matrix + mini_mime (>= 0.1.3) + nokogiri (~> 1.11) + rack (>= 1.6.0) + rack-test (>= 0.6.3) + regexp_parser (>= 1.5, < 3.0) + xpath (~> 3.2) + capybara-screenshot (1.0.26) + capybara (>= 1.0, < 4) + launchy + childprocess (5.1.0) + logger (~> 1.5) + coercible (1.0.0) + descendants_tracker (~> 0.0.1) colorator (1.1.0) concurrent-ruby (1.2.3) console (1.25.2) fiber-annotation fiber-local (~> 1.1) json + descendants_tracker (0.0.4) + thread_safe (~> 0.3, >= 0.3.1) + diff-lcs (1.5.1) + dumb_delegator (1.0.0) em-websocket (0.5.3) eventmachine (>= 0.12.9) http_parser.rb (~> 0) @@ -59,6 +94,7 @@ GEM http_parser.rb (0.8.0) i18n (1.14.4) concurrent-ruby (~> 1.0) + ice_nine (0.11.2) io-event (1.5.1) jekyll (4.3.3) addressable (~> 2.4) @@ -85,6 +121,8 @@ GEM sass-embedded (~> 1.54) jekyll-seo-tag (2.8.0) jekyll (>= 3.8, < 5.0) + jekyll-sitemap (1.4.0) + jekyll (>= 3.7, < 5.0) jekyll-watch (2.2.1) listen (~> 3.0) json (2.7.2) @@ -92,11 +130,17 @@ GEM rexml kramdown-parser-gfm (1.1.0) kramdown (~> 2.0) + launchy (3.0.1) + addressable (~> 2.8) + childprocess (~> 5.0) liquid (4.0.4) listen (3.9.0) rb-fsevent (~> 0.10, >= 0.10.3) rb-inotify (~> 0.9, >= 0.9.10) + logger (1.6.0) + matrix (0.4.2) mercenary (0.4.0) + mini_mime (1.1.5) net-http (0.4.1) uri nokogiri (1.16.5-arm64-darwin) @@ -116,15 +160,36 @@ GEM ttfunk public_suffix (5.0.5) racc (1.7.3) + rack (3.1.7) + rack-test (2.1.0) + rack (>= 1.3) + rackup (2.1.0) + rack (>= 3) + webrick (~> 1.8) rainbow (3.1.1) rake (13.2.1) rb-fsevent (0.11.2) rb-inotify (0.10.1) ffi (~> 1.0) + regexp_parser (2.9.2) rexml (3.3.3) strscan rouge (4.2.1) + rspec (3.13.0) + rspec-core (~> 3.13.0) + rspec-expectations (~> 3.13.0) + rspec-mocks (~> 3.13.0) + rspec-core (3.13.0) + rspec-support (~> 3.13.0) + rspec-expectations (3.13.1) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.13.0) + rspec-mocks (3.13.1) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.13.0) + rspec-support (3.13.1) ruby-rc4 (0.1.5) + rubyzip (2.3.2) safe_yaml (1.0.5) sass-embedded (1.75.0-arm64-darwin) google-protobuf (>= 3.25, < 5.0) @@ -133,9 +198,16 @@ GEM sawyer (0.9.2) addressable (>= 2.3.5) faraday (>= 0.17.3, < 3) + selenium-webdriver (4.23.0) + base64 (~> 0.2) + logger (~> 1.4) + rexml (~> 3.2, >= 3.2.5) + rubyzip (>= 1.2.2, < 3.0) + websocket (~> 1.0) strscan (3.1.0) terminal-table (3.0.2) unicode-display_width (>= 1.1.1, < 3) + thread_safe (0.3.6) timers (4.3.5) ttfunk (1.8.0) bigdecimal (~> 3.1) @@ -143,7 +215,14 @@ GEM ethon (>= 0.9.0) unicode-display_width (2.5.0) uri (0.13.0) + virtus (2.0.0) + axiom-types (~> 0.1) + coercible (~> 1.0) + descendants_tracker (~> 0.0, >= 0.0.3) webrick (1.8.1) + websocket (1.2.11) + xpath (3.2.0) + nokogiri (~> 1.8) yell (2.2.2) zeitwerk (2.6.13) @@ -152,11 +231,21 @@ PLATFORMS x86_64-linux-gnu DEPENDENCIES + axe-core-capybara + axe-core-rspec bundler (>= 2.3.5) + capybara + capybara-screenshot html-proofer (~> 5.0) jekyll-github-metadata (>= 2.15) jekyll-include-cache + jekyll-sitemap just-the-docs! + rack + rackup + rspec + selenium-webdriver + webrick BUNDLED WITH 2.5.9 diff --git a/spec/accessibility_spec.rb b/spec/accessibility_spec.rb new file mode 100644 index 0000000000..67e69869ce --- /dev/null +++ b/spec/accessibility_spec.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +# Run accessibility specs for all pages in the webiste. +# This runs the axe accessibility checker on each page in a headless browser. + +# spec_helper ensures the webiste is built and can be served locally +require 'spec_helper' + +# Axe-core test standards groups +# See https://github.com/dequelabs/axe-core/blob/develop/doc/API.md#axe-core-tags +# Tests are segmented in 2.0, 2.1 and 2.2+ +# In most places WCAG 2.1AA is the minimum requirement, but 2.2 is the current WCAG Standard. +required_a11y_standards = %i[wcag2a wcag2aa wcag21a wcag21aa] +complete_a11y_standards = %i[wcag22aa best-practice secion508] + +# axe-core rules that are not required to be accessible / do not apply +# You may temporarily want to add rules here during development. +# See: https://github.com/dequelabs/axe-core/blob/develop/doc/rule-descriptions.md +skipped_rules = [] +# These are elements that are not required to be accessible +# It should be rare to add to this list. This disables all rules for an element. +# e.g. would pass even though it's missing alt text. +excluded_elements = [ + '[data-a11y-errors="true"]' +] + +# We must call this to ensure the build it up-to-date. +build_jekyll_site! +ALL_PAGES = load_sitemap +puts "Running tests on #{ALL_PAGES.count} pages." +puts " - #{ALL_PAGES.join("\n - ")}\n\n" + +ALL_PAGES.each do |path| + describe "#{path} is accessible", :js, type: :feature do + before do + visit(path) + end + + it 'meets WCAG 2.1' do + expect(page).to be_axe_clean + .according_to(*required_a11y_standards) + .skipping(*skipped_rules) + .excluding(*excluded_elements) + end + + it 'meets WCAG 2.2' do + expect(page).to be_axe_clean + .according_to(*complete_a11y_standards) + .skipping(*skipped_rules) + .excluding(*excluded_elements) + end + end +end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb new file mode 100644 index 0000000000..f7c94ab4b5 --- /dev/null +++ b/spec/spec_helper.rb @@ -0,0 +1,109 @@ +# frozen_string_literal: true + +# This file was generated by the `rspec --init` command. Conventionally, all +# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`. +# The generated `.rspec` file contains `--require spec_helper` which will cause +# this file to always be loaded, without a need to explicitly require it in any +# files. +# +# Given that it is always loaded, you are encouraged to keep this file as +# light-weight as possible. Requiring heavyweight dependencies from this file +# will add to the boot time of your test suite on EVERY test run, even for an +# individual file that may not need all of that loaded. Instead, consider making +# a separate helper file that requires the additional dependencies and performs +# the additional setup, and require it from the spec files that actually need +# it. +# +# See https://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration + +require 'rspec' +require 'rack' +require 'yaml' +require 'webrick' + +require 'capybara/rspec' +require 'capybara/dsl' +require 'capybara-screenshot/rspec' +require 'capybara/session' + +require 'rack/test' +require 'axe-rspec' +require 'axe-capybara' + +require_relative 'support/jekyll' + +# Used to set the path for a local webserver. +# Update this if you move this file. +REPO_ROOT = File.expand_path('../', __dir__) + +Capybara.register_driver :chrome_headless do |app| + options = Selenium::WebDriver::Chrome::Options.new + options.add_argument('--headless') + options.add_argument('--no-sandbox') + options.add_argument('--disable-dev-shm-usage') + # MacBook Air ~13" screen size, with an absurd height to capture more content. + options.add_argument('--window-size=1280,4000') + + Capybara::Selenium::Driver.new(app, browser: :chrome, options:) +end + +# Change default_driver to :selenium_chrome if you want to actually see the tests running in a browser locally. +# Should be :chrome_headless in CI though. +Capybara.default_driver = :chrome_headless +Capybara.javascript_driver = :chrome_headless + +Capybara::Screenshot.register_driver(:chrome_headless) do |driver, path| + driver.save_screenshot(path) +end + +Capybara::Screenshot.register_filename_prefix_formatter(:rspec) do |example| + page = example.example_group.top_level_description.gsub(%r{^/}, '').gsub('/', '-').gsub(' is accessible', '') + standards = example.description.split.last + "tmp/capybara/screenshot_#{page}_#{standards}" +end + +Capybara::Screenshot.autosave_on_failure = true +Capybara::Screenshot.append_timestamp = false +Capybara::Screenshot.prune_strategy = :keep_last_run + +# Setup for Capybara to serve static files served by Rack +Capybara.server = :webrick +Capybara.app = Rack::Builder.new do + use Rack::Lint + run StaticSite.new(REPO_ROOT) +end.to_app + +RSpec.configure do |config| + # Allow rspec to use `--only-failures` and `--next-failure` flags + # Ensure that `tmp` is in your `.gitignore` file + config.example_status_persistence_file_path = 'tmp/rspec-failures.txt' + + # rspec-expectations config goes here. You can use an alternate + # assertion/expectation library such as wrong or the stdlib/minitest + # assertions if you prefer. + config.expect_with :rspec do |expectations| + # This option will default to `true` in RSpec 4. It makes the `description` + # and `failure_message` of custom matchers include text for helper methods + # defined using `chain`, e.g.: + # be_bigger_than(2).and_smaller_than(4).description + # # => "be bigger than 2 and smaller than 4" + # ...rather than: + # # => "be bigger than 2" + expectations.include_chain_clauses_in_custom_matcher_descriptions = true + end + + # rspec-mocks config goes here. You can use an alternate test double + # library (such as bogus or mocha) by changing the `mock_with` option here. + config.mock_with :rspec do |mocks| + mocks.verify_partial_doubles = true + end + + # This option will default to `:apply_to_host_groups` in RSpec 4 (and will + # have no way to turn it off -- the option exists only for backwards + # compatibility in RSpec 3). It causes shared context metadata to be + # inherited by the metadata hash of host groups and examples, rather than + # triggering implicit auto-inclusion in groups with matching metadata. + config.shared_context_metadata_behavior = :apply_to_host_groups + + config.include Capybara::DSL +end diff --git a/spec/support/jekyll.rb b/spec/support/jekyll.rb new file mode 100644 index 0000000000..ab83f13c85 --- /dev/null +++ b/spec/support/jekyll.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +require 'jekyll' + +# Tools to build / compile the Jekyll site and extract the sitemap +def site_config + # Consider simplifying baseurl / url + # Consider forcing the desination folder to not clobber _site? + # Consider appending jekyll-sitemap to the plugins here, instead of in _config. + # Note: Config keys must be strings and thus use => style hashes. + @site_config ||= Jekyll.configuration({ 'sass' => { 'quiet_deps' => true } }) +end + +def build_jekyll_site! + puts 'Building site...' + @site = Jekyll::Site.new(site_config) + @site.process + puts 'Site build complete.' +end + +def load_sitemap + # Ensure that you have called build_jekyll_site! first. + sitemap_text = File.read('_site/sitemap.xml') + sitemap_links = sitemap_text.scan(%r{.+}) + sitemap_links.filter_map do |link| + link = link.gsub("#{site_config['url']}", '').gsub('', '') + # Skip non-html pages + # (FUTURE?) Are there other pages that should be audited for accessibility? + # (e.g. PDFs, documents. They'd need a different checker.) + next unless link.end_with?('.html') || link.end_with?('/') + + link + end.sort +end + +# Start a local Rack server +# adapted from https://nts.strzibny.name/how-to-test-static-sites-with-rspec-capybara-and-webkit/ +class StaticSite + attr_reader :root, :server + + def initialize(root) + @root = root + @server = Rack::Files.new(root) + end + + def call(env) + # Remove the /baseurl prefix, which is present in all URLs, but not in the file system. + path = "_site#{env['PATH_INFO'].gsub(site_config['baseurl'], '/')}" + + env['PATH_INFO'] = if path.end_with?('/') && exists?("#{path}index.html") + "#{path}index.html" + elsif !exists?(path) && exists?("#{path}.html") + "#{path}.html" + else + path + end + + server.call(env) + end + + def exists?(path) + File.exist?(File.join(root, path)) + end +end diff --git a/spec/support/spec_summary.rb b/spec/support/spec_summary.rb new file mode 100644 index 0000000000..0db441326d --- /dev/null +++ b/spec/support/spec_summary.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +# Summarize the axe rspec failures into aggregate counts +# TODO: This should be an RSpec formatter + +require 'json' + +RESULTS_PATH = File.join(File.dirname(__FILE__), '..', '..', 'tmp/rspec_output.json') +AXE_CASE_TITLE = /\n\s*\n\s*\d+\)\s+([-\w]+):/ + +def failing_specs(results_data) + results_data['examples'].filter do |ex| + ex['status'] == 'failed' + end +end + +def summarize_results(results) + failing_specs(results).map do |ex| + ex['exception']['message'].scan(AXE_CASE_TITLE) + end.flatten.tally +end + +# rubocop:disable Metrics/AbcSize +# rubocop:disable Metrics/MethodLength +def group_results(results) + all_cases_list = failing_specs(results).map do |ex| + msg = ex['exception']['message'] + msg.gsub!(/\nInvocation:.*;/, '') + cases = msg.split(AXE_CASE_TITLE) + cases.delete_at(0) + Hash[*cases].transform_values { |v| { page: ex['full_description'], message: v } } + end + results = {} + results.default = [] + all_cases_list.each do |test_hash| + test_hash.each do |axe_name, failure| + if results.key?(axe_name) + results[axe_name] << failure + else + results[axe_name] = [failure] + end + end + end + results +end +# rubocop:enable Metrics/AbcSize +# rubocop:enable Metrics/MethodLength + +def test_failures_with_pages(summary_group) + summary_group.transform_values { |list| list.map { |h| h[:page] } } +end + +def nicely_print(hash) + hash.each do |key, values| + puts "#{key}:" + values.each { |item| puts("\t#{item}") } + end +end + +def print_summary + results_data = JSON.parse(File.read(RESULTS_PATH)) + failing_tests_by_type = summarize_results(results_data) + total_failures = failing_tests_by_type.values.sum + puts "#{total_failures} total a11y failures." + return if total_failures == 0 + + pp(failing_tests_by_type) + puts "Failing Pages:\n#{'-' * 16}" + summary_group = group_results(results_data) + nicely_print(test_failures_with_pages(summary_group)) +end + +print_summary