diff --git a/cmd/screentest/main.go b/cmd/screentest/main.go index 4838c6e156..2cd55f063a 100644 --- a/cmd/screentest/main.go +++ b/cmd/screentest/main.go @@ -3,55 +3,166 @@ // license that can be found in the LICENSE file. /* -Command screentest runs the screentest check for a set of scripts. +Screentest compares images of rendered web pages. +It compares images obtained from two sources, one to test and one for the expected result. +The comparisons are driven by a script file in a format described below. - Usage: screentest [flags] [glob] +# Usage + + screentest [flags] [glob] The flags are: - -test - URL to test against - -want - URL for expected results + -test URL + URL or path being tested. Required. + -want URL + URL or path for expected results. Required. + -c + Number of test cases to run concurrently. + -d + URL of a Chrome websocket debugger. If omitted, screentest tries to find the + Chrome executable on the system and starts a new instance. -headers - HTTP headers to send + HTTP(S) headers to send with each request, as a comma-separated list of name:value. + -run REGEXP + Run only tests matching regexp. -o - URL for output + URL or path for output files. If omitted, files are written to a subdirectory of the + user's cache directory. -u - update cached screenshots + Update cached screenshots. -v - variables provided to script templates as comma separated KEY:VALUE pairs - -c - number of testcases to run concurrently - -d - chrome debugger url - -run - run only tests matching regexp + Variables provided to script templates as comma separated KEY:VALUE pairs. + +# Scripts + +A script file contains one or more test cases described as a sequence of lines. The +file is first processed as Go template using the text/template package, with a map +of the variables given by the -v flag provided as `.`. + +The script format is line-oriented. +Lines beginning with # characters are ignored as comments. +Each non-blank, non-comment line is a directive, listed below. + +Each test case begins with the 'test' directive and ends with a blank line. +A test case describes actions to take on a page, along +with the dimensions of the screenshots to be compared. For example, here is +a trivial script: + + test about + pathname /about + capture fullscreen + +This script has a single test case. The first line names the test. +The second line sets the page to visit at each origin. The last line +captures full-page screenshots of the pages and generates a diff image if they +do not match. + +# Directives + +Use windowsize WIDTHxHEIGHT to set the default window size for all test cases +that follow. + + windowsize 540x1080 + +Use block URL ... to set URL patterns to block. Wildcards ('*') are allowed. + + block https://codecov.io/* https://travis-ci.com/* + +The directives above apply to all test cases that follow. +The ones below must appear inside a test case and apply only to that case. + +Use test NAME to create a name for the test case. + + test about page + +Use pathname PATH to set the page to visit at each origin. + + pathname /about + +Use status CODE to set an expected HTTP status code. The default is 200. + + status 404 + +Use click SELECTOR to add a click an element on the page. + + click button.submit + +Use wait SELECTOR to wait for an element to appear. + + wait [role="treeitem"][aria-expanded="true"] + +Use capture [SIZE] [ARG] to create a test case with the properties +defined in the test case. If present, the first argument to capture should be one of +'fullscreen', 'viewport' or 'element'. + + capture fullscreen 540x1080 + +When taking an element screenshot provide a selector. + + capture element header + +Use eval JS to evaluate JavaScript snippets to hide elements or prepare the page in +some other way. + + eval 'document.querySelector(".selector").remove();' + eval 'window.scrollTo({top: 0});' + +Each capture command to creates a new test case for a single page. + + windowsize 1536x960 + + test homepage + pathname / + capture viewport + capture viewport 540x1080 + capture viewport 400x1000 + + test about page + pathname /about + capture viewport + capture viewport 540x1080 + capture viewport 400x1000 */ package main import ( + "context" "flag" "fmt" "log" "os" "path/filepath" - "regexp" "runtime" - "strings" ) -var ( - testURL = flag.String("test", "", "URL or file path to test") - wantURL = flag.String("want", "", "URL or file path with expected results") - update = flag.Bool("u", false, "update cached screenshots") - vars = flag.String("v", "", "variables provided to script templates as comma separated KEY:VALUE pairs") - concurrency = flag.Int("c", (runtime.NumCPU()+1)/2, "number of testcases to run concurrently") - debuggerURL = flag.String("d", "", "chrome debugger url") - run = flag.String("run", "", "regexp to match test") - outputURL = flag.String("o", "", "path for output: file path or URL with 'file' or 'gs' scheme") - headers = flag.String("headers", "", "HTTP headers: comma-separated list of name:value") -) +var flags options + +func init() { + flag.StringVar(&flags.testURL, "test", "", "URL or file path to test") + flag.StringVar(&flags.wantURL, "want", "", "URL or file path with expected results") + flag.BoolVar(&flags.update, "u", false, "update cached screenshots") + flag.StringVar(&flags.vars, "v", "", "variables provided to script templates as comma separated KEY:VALUE pairs") + flag.IntVar(&flags.maxConcurrency, "c", (runtime.NumCPU()+1)/2, "number of test cases to run concurrently") + flag.StringVar(&flags.debuggerURL, "d", "", "chrome debugger URL") + flag.StringVar(&flags.outputURL, "o", "", "path for output: file path or URL with 'file' or 'gs' scheme") + flag.StringVar(&flags.headers, "headers", "", "HTTP headers: comma-separated list of name:value") + flag.StringVar(&flags.run, "run", "", "regexp to match test") +} + +// options are the options for the program. +// See the top command and the flag.XXXVar calls above for documentation. +type options struct { + testURL string + wantURL string + update bool + vars string + maxConcurrency int + debuggerURL string + run string + outputURL string + headers string +} func main() { flag.Usage = func() { @@ -69,39 +180,8 @@ func main() { if len(args) == 1 { glob = args[0] } - parsedVars := make(map[string]string) - if *vars != "" { - for _, pair := range strings.Split(*vars, ",") { - parts := strings.SplitN(pair, ":", 2) - if len(parts) != 2 { - log.Fatal(fmt.Errorf("invalid key value pair, %q", pair)) - } - parsedVars[strings.TrimSpace(parts[0])] = strings.TrimSpace(parts[1]) - } - } - var splitHeaders []string - if len(*headers) > 0 { - splitHeaders = strings.Split(*headers, ",") - } - opts := CheckOptions{ - TestURL: *testURL, - WantURL: *wantURL, - Update: *update, - MaxConcurrency: *concurrency, - Vars: parsedVars, - DebuggerURL: *debuggerURL, - OutputURL: *outputURL, - Headers: splitHeaders, - } - if *run != "" { - re, err := regexp.Compile(*run) - if err != nil { - log.Fatal(err) - } - opts.Filter = re.MatchString - } - if err := CheckHandler(glob, opts); err != nil { + if err := run(context.Background(), glob, flags); err != nil { log.Fatal(err) } } diff --git a/cmd/screentest/screentest.go b/cmd/screentest/screentest.go index 177de0e8b7..8882faf6f5 100644 --- a/cmd/screentest/screentest.go +++ b/cmd/screentest/screentest.go @@ -2,96 +2,6 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// TODO(jba): incorporate the following comment into the top comment in main.go - -// Package screentest implements script-based visual diff testing -// for webpages. -// -// # Scripts -// -// A script is a template file containing a sequence of testcases, separated by -// blank lines. Lines beginning with # characters are ignored as comments. A -// testcase is a sequence of lines describing actions to take on a page, along -// with the dimensions of the screenshots to be compared. For example, here is -// a trivial script: -// -// test about -// pathname /about -// capture fullscreen -// -// This script has a single testcase. The first line names the test. -// The second line sets the page to visit at each origin. The last line -// captures fullpage screenshots of the pages and generates a diff image if they -// do not match. -// -// # Keywords -// -// Use windowsize WIDTHxHEIGHT to set the default window size for all testcases -// that follow. -// -// windowsize 540x1080 -// -// Use block URL ... to set URL patterns to block. Wildcards ('*') are allowed. -// -// block https://codecov.io/* https://travis-ci.com/* -// -// Values set with the keywords above apply to all testcases that follow. Values set with -// the keywords below reset each time the test keyword is used. -// -// Use test NAME to create a name for the testcase. -// -// test about page -// -// Use pathname PATH to set the page to visit at each origin. If no -// test name is set, PATH will be used as the name for the test. -// -// pathname /about -// -// Use status CODE to set an expected HTTP status code. The default is 200. -// -// status 404 -// -// Use click SELECTOR to add a click an element on the page. -// -// click button.submit -// -// Use wait SELECTOR to wait for an element to appear. -// -// wait [role="treeitem"][aria-expanded="true"] -// -// Use capture [SIZE] [ARG] to create a testcase with the properties -// defined above. -// -// capture fullscreen 540x1080 -// -// When taking an element screenshot provide a selector. -// -// capture element header -// -// Evaluate JavaScript snippets to hide elements or prepare the page in -// some other way. -// -// eval 'document.querySelector(".selector").remove();' -// eval 'window.scrollTo({top: 0});' -// -// Chain capture commands to create multiple testcases for a single page. -// -// windowsize 1536x960 -// compare https://go.dev::cache http://localhost:6060 -// output testdata/snapshots -// -// test homepage -// pathname / -// capture viewport -// capture viewport 540x1080 -// capture viewport 400x1000 -// -// test about page -// pathname /about -// capture viewport -// capture viewport 540x1080 -// capture viewport 400x1000 - package main import ( @@ -126,48 +36,15 @@ import ( "google.golang.org/api/iterator" ) -type CheckOptions struct { - // TestURL is the URL or path that is being tested. - TestURL string - - // WantURL is the URL or path that the test is being compared with; the "goldens." - WantURL string - - // Update is true if cached screenshots should be updated. - Update bool - - // MaxConcurrency is the maximum number of testcases to run in parallel. - MaxConcurrency int - - // Vars are accessible as values in the test script templates. - Vars map[string]string - - // DebuggerURL is the URL to a chrome websocket debugger. If left empty - // screentest tries to find the Chrome executable on the system and starts - // a new instance. - DebuggerURL string - - // If set, only tests for which Filter returns true are included. - // Filter is called on the test name. - Filter func(string) bool - - // If set, where cached files and diffs are written to. - // May be a file: or gs: URL, or a file path. - OutputURL string - - // Headers to add to HTTP(S) requests. - // Each header should be of the form "name:value". - Headers []string -} - -// CheckHandler runs the test scripts matched by glob. If any errors are -// encountered, CheckHandler returns an error listing the problems. -func CheckHandler(glob string, opts CheckOptions) error { - if opts.MaxConcurrency < 1 { - opts.MaxConcurrency = 1 +// run runs the test scripts matched by glob. If any errors are +// encountered, run returns an error listing the problems. +func run(ctx context.Context, glob string, opts options) error { + if opts.maxConcurrency < 1 { + opts.maxConcurrency = 1 } + now := time.Now() - ctx := context.Background() + files, err := filepath.Glob(glob) if err != nil { return fmt.Errorf("filepath.Glob(%q): %w", glob, err) @@ -176,8 +53,8 @@ func CheckHandler(glob string, opts CheckOptions) error { return fmt.Errorf("no files match %q", glob) } var cancel context.CancelFunc - if opts.DebuggerURL != "" { - ctx, cancel = chromedp.NewRemoteAllocator(ctx, opts.DebuggerURL) + if opts.debuggerURL != "" { + ctx, cancel = chromedp.NewRemoteAllocator(ctx, opts.debuggerURL) } else { ctx, cancel = chromedp.NewExecAllocator(ctx, append( chromedp.DefaultExecAllocatorOptions[:], @@ -191,7 +68,7 @@ func CheckHandler(glob string, opts CheckOptions) error { if err != nil { return fmt.Errorf("readTestdata(%q): %w", file, err) } - if len(tests) == 0 && opts.Filter == nil { + if len(tests) == 0 && opts.run == "" { return fmt.Errorf("no tests found in %q", file) } if err := cleanOutput(ctx, tests); err != nil { @@ -200,9 +77,9 @@ func CheckHandler(glob string, opts CheckOptions) error { ctx, cancel = chromedp.NewContext(ctx, chromedp.WithLogf(log.Printf)) defer cancel() var hdr bool - runConcurrently(len(tests), opts.MaxConcurrency, func(i int) { + runConcurrently(len(tests), opts.maxConcurrency, func(i int) { tc := tests[i] - if err := tc.run(ctx, opts.Update); err != nil { + if err := tc.run(ctx, opts.update); err != nil { if !hdr { fmt.Fprintf(&buf, "%s\n\n", file) hdr = true @@ -220,17 +97,6 @@ func CheckHandler(glob string, opts CheckOptions) error { return nil } -type TestOpts struct { - // Update is true if cached screenshots should be updated. - Update bool - - // Parallel runs t.Parallel for each testcase. - Parallel bool - - // Vars are accessible as values in the test script templates. - Vars map[string]string -} - // cleanOutput clears the output locations of images not cached // as part of a testcase, including diff output from previous test // runs and obsolete screenshots. It ensures local directories exist @@ -374,7 +240,7 @@ func (t *testcase) String() string { } // readTests parses the testcases from a text file. -func readTests(file string, opts CheckOptions) ([]*testcase, error) { +func readTests(file string, opts options) ([]*testcase, error) { tmpl := template.New(filepath.Base(file)).Funcs(template.FuncMap{ "ints": func(start, end int) []int { var out []int @@ -389,8 +255,14 @@ func readTests(file string, opts CheckOptions) ([]*testcase, error) { if err != nil { return nil, fmt.Errorf("template.ParseFiles(%q): %w", file, err) } + + parsedVars, err := splitList(opts.vars) + if err != nil { + return nil, err + } + var tmplout bytes.Buffer - if err := tmpl.Execute(&tmplout, opts.Vars); err != nil { + if err := tmpl.Execute(&tmplout, parsedVars); err != nil { return nil, fmt.Errorf("tmpl.Execute(...): %w", err) } var tests []*testcase @@ -409,31 +281,29 @@ func readTests(file string, opts CheckOptions) ([]*testcase, error) { if err != nil { return nil, fmt.Errorf("os.UserCacheDir(): %w", err) } - if opts.TestURL != "" { - originA = opts.TestURL + if opts.testURL != "" { + originA = opts.testURL if strings.HasSuffix(originA, cacheSuffix) { originA = strings.TrimSuffix(originA, cacheSuffix) cacheA = true } } - if opts.WantURL != "" { - originB = opts.WantURL + if opts.wantURL != "" { + originB = opts.wantURL if strings.HasSuffix(originB, cacheSuffix) { originB = strings.TrimSuffix(originB, cacheSuffix) cacheB = true } } - headers := map[string]any{} // any to match chromedp's arg - for _, h := range opts.Headers { - name, value, ok := strings.Cut(h, ":") - name = strings.TrimSpace(name) - value = strings.TrimSpace(value) - if !ok || name == "" || value == "" { - return nil, fmt.Errorf("invalid header %q", h) - } - headers[name] = value + headers := map[string]any{} + hs, err := splitList(opts.headers) + if err != nil { + return nil, err + } + for k, v := range hs { + headers[k] = v } - dir := cmp.Or(opts.OutputURL, filepath.Join(cache, "screentest")) + dir := cmp.Or(opts.outputURL, filepath.Join(cache, "screentest")) out, err := outDir(dir, file) if err != nil { return nil, err @@ -441,6 +311,16 @@ func readTests(file string, opts CheckOptions) ([]*testcase, error) { if strings.HasPrefix(out, gcsScheme) { gcsBucket = true } + + filter := func(string) bool { return true } + if opts.run != "" { + re, err := regexp.Compile(opts.run) + if err != nil { + return nil, err + } + filter = re.MatchString + } + scan := bufio.NewScanner(&tmplout) for scan.Scan() { lineNo++ @@ -461,7 +341,7 @@ func readTests(file string, opts CheckOptions) ([]*testcase, error) { tasks = nil status = http.StatusOK case "COMPARE": - log.Printf("%s:%d: DEPRECATED: instead of 'compare', set the -test and -want flags, or the TestURL and WantURL options", file, lineNo) + log.Printf("%s:%d: DEPRECATED: instead of 'compare', set the -test and -want flags", file, lineNo) if originA != "" || originB != "" { log.Printf("%s:%d: DEPRECATED: multiple 'compare's", file, lineNo) } @@ -483,7 +363,7 @@ func readTests(file string, opts CheckOptions) ([]*testcase, error) { return nil, fmt.Errorf("url.Parse(%q): %w", originB, err) } case "HEADER": - log.Printf("%s:%d: DEPRECATED: instead of 'header', set the -headers flag, or the CheckOptions.Headers option", file, lineNo) + log.Printf("%s:%d: DEPRECATED: instead of 'header', set the -headers flag", file, lineNo) parts := strings.SplitN(args, ":", 2) if len(parts) != 2 { return nil, fmt.Errorf("invalid header %s on line %d", args, lineNo) @@ -495,7 +375,7 @@ func readTests(file string, opts CheckOptions) ([]*testcase, error) { return nil, fmt.Errorf("strconv.Atoi(%q): %w", args, err) } case "OUTPUT": - log.Printf("DEPRECATED: 'output': set CheckOptions.OutputURL, or provide -o on the command line") + log.Printf("DEPRECATED: 'output': provide -o on the command line") if strings.HasPrefix(args, gcsScheme) { gcsBucket = true } @@ -555,7 +435,7 @@ func readTests(file string, opts CheckOptions) ([]*testcase, error) { if err != nil { return nil, fmt.Errorf("url.Parse(%q): %w", originB+pathname, err) } - if opts.Filter != nil && !opts.Filter(testName) { + if !filter(testName) { continue } test := &testcase{ @@ -614,6 +494,27 @@ func readTests(file string, opts CheckOptions) ([]*testcase, error) { return tests, nil } +// splitList splits a list of key:value pairs separated by commas. +// Whitespace is trimmed around comma-separated elements, keys, and values. +// Empty names are an error; empty values are OK. +func splitList(s string) (map[string]string, error) { + s = strings.TrimSpace(s) + if len(s) == 0 { + return nil, nil + } + m := map[string]string{} + for _, h := range strings.Split(s, ",") { + name, value, ok := strings.Cut(h, ":") + if !ok || name == "" { + return nil, fmt.Errorf("invalid name:value pair: %q", h) + } + name = strings.TrimSpace(name) + value = strings.TrimSpace(value) + m[name] = value + } + return m, nil +} + // outDir gets a diff output directory for a given testfile. // If dir points to a GCS bucket or testfile is empty it just // returns dir. diff --git a/cmd/screentest/screentest_test.go b/cmd/screentest/screentest_test.go index 3bb05440fa..e4baba020d 100644 --- a/cmd/screentest/screentest_test.go +++ b/cmd/screentest/screentest_test.go @@ -30,7 +30,7 @@ func TestReadTests(t *testing.T) { tests := []struct { name string args args - opts CheckOptions + opts options want any wantErr bool }{ @@ -39,10 +39,10 @@ func TestReadTests(t *testing.T) { args: args{ filename: "testdata/readtests.txt", }, - opts: CheckOptions{ - Vars: map[string]string{"Authorization": "Bearer token"}, - TestURL: "https://go.dev", - WantURL: "http://localhost:6060", + opts: options{ + vars: "Authorization:Bearer token", + testURL: "https://go.dev", + wantURL: "http://localhost:6060", }, want: []*testcase{ { @@ -141,11 +141,11 @@ func TestReadTests(t *testing.T) { args: args{ filename: "testdata/readtests2.txt", }, - opts: CheckOptions{ - TestURL: "https://pkg.go.dev::cache", - WantURL: "http://localhost:8080", - Headers: []string{"Authorization:Bearer token"}, - OutputURL: "gs://bucket/prefix", + opts: options{ + testURL: "https://pkg.go.dev::cache", + wantURL: "http://localhost:8080", + headers: "Authorization:Bearer token", + outputURL: "gs://bucket/prefix", }, want: []*testcase{ { @@ -203,7 +203,7 @@ func TestReadTests(t *testing.T) { } } -func TestCheckHandler(t *testing.T) { +func TestRun(t *testing.T) { // Skip this test if Google Chrome is not installed. _, err := exec.LookPath("google-chrome") if err != nil { @@ -221,7 +221,7 @@ func TestCheckHandler(t *testing.T) { var tests = []struct { name string args args - opts CheckOptions + opts options wantErr bool wantFiles []string }{ @@ -230,9 +230,9 @@ func TestCheckHandler(t *testing.T) { args: args{ glob: "testdata/pass.txt", }, - opts: CheckOptions{ - TestURL: "https://go.dev", - WantURL: "https://go.dev", + opts: options{ + testURL: "https://go.dev", + wantURL: "https://go.dev", }, wantErr: false, }, @@ -255,10 +255,10 @@ func TestCheckHandler(t *testing.T) { output: "testdata/screenshots/cached", glob: "testdata/cached.txt", }, - opts: CheckOptions{ - TestURL: "https://go.dev::cache", - WantURL: "https://go.dev::cache", - OutputURL: "testdata/screenshots/cached", + opts: options{ + testURL: "https://go.dev::cache", + wantURL: "https://go.dev::cache", + outputURL: "testdata/screenshots/cached", }, wantFiles: []string{ filepath.Join("testdata", "screenshots", "cached", "homepage.a.png"), @@ -268,7 +268,7 @@ func TestCheckHandler(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if err := CheckHandler(tt.args.glob, tt.opts); (err != nil) != tt.wantErr { + if err := run(context.Background(), tt.args.glob, tt.opts); (err != nil) != tt.wantErr { t.Fatalf("CheckHandler() error = %v, wantErr %v", err, tt.wantErr) } if len(tt.wantFiles) != 0 {