diff --git a/scripts/conformance/rubyspec/README.md b/scripts/conformance/rubyspec/README.md index aa8a98f9..d96a0046 100644 --- a/scripts/conformance/rubyspec/README.md +++ b/scripts/conformance/rubyspec/README.md @@ -80,8 +80,64 @@ and be recorded as not loading, which bakes the load into the gate. The CI job (`.github/workflows/rubyspec-ratchet.yml`) enforces the baseline on every push and pull request, and raises `TIMEOUT` to 60s for exactly this reason. +## `cmd/phantom` — is a pass the behaviour, or the method's absence? + +A spec that asserts an exception class cannot tell why it got one. +`core/array/rfind_spec.rb` has + +```ruby +it "raises a NoMethodError if the ifnone argument does not respond to #call …" do + -> { [1, 2, 3].rfind(42) { |x| false } }.should.raise(NoMethodError) +end +``` + +which means *"42 is not callable"*. On a build where `Array#rfind` does not exist, +`[1,2,3].rfind(42)` raises NoMethodError for **that** reason and the example +passes — a pass for a method the interpreter does not have. + +`cmd/phantom` names those. For every `describe "X#y"` or `describe "X.y"` holding +a NoMethodError expectation it asks a real rbgo whether the method exists at all +(`method_defined?`/`private_method_defined?`, or `respond_to?` for the `.` form — +no instance is constructed), and reports the ones that do not. + +```bash +go run ./scripts/conformance/rubyspec/cmd/phantom -root /tmp/rubyspec-corpus -rbgo /tmp/rbgo-rubyspec +``` + +Measured on `main` after #849: **59 attributable expectations over 2206 spec +files, 0 candidates** — so **no pass anywhere in the corpus is bought by a +missing method**, and any future non-zero is the signal. + +It was 1 before #849 (`Array#rfind`), and that one is instructive: the example +read *"raises a NoMethodError if the ifnone argument does not respond to +#call"*, and on a build without `rfind` it passed because `rfind` itself was +undefined. `BASELINE` recorded that file as `0`, so the counted total never held +it — the pass only appeared with the shim at 4.0, which is where this tool will +matter when the version moves. + +The 2206 is the same population `BASELINE` sums, which is the check that this +tool and the gate are looking at the same corpus. + +It withdraws any describe block that **also asserts the absence**, because such +an example is asserting exactly what it got: `core/complex/negative_spec.rb` +("Complex#negative? is undefined") is conformance, not a phantom, and MRI really +does not define it. 23 of the 82 attributable expectations are of that kind. + +Its limits are printed on every run rather than left in a comment: it reads the +describe string, skips operator methods, and stops attribution at a sibling +`describe` judged by indentation — so it can MISS a candidate but will not invent +one, and its count is a lower bound. Like `cmd/ratchet` it refuses rather than +guesses: exit 2 if the corpus cannot be read, if the interpreter cannot be run, +or if its own control fails (a method that certainly exists must come back +present, and one that cannot exist must come back absent). ## Which examples the version string selects — `cmd/guards` +`cmd/guards` and `cmd/phantom` answer different questions about the same +number and neither replaces the other: guards says **which examples the +version string lets into the population**, phantom says **whether a pass in +that population is behaviour or an absence**. Both are analysis tools rather +than gates, and both print their own control first. + `spec_helper.rb` sets `CUR_VERSION`, and the corpus guards whole blocks and individual values on it. Moving that string moves the population **in both directions**, so `cmd/guards` censuses the corpus before anything is @@ -116,3 +172,7 @@ The shim is not full MSpec: a few matchers (`complain`, `output`, examples count as skipped rather than passing. The measured totals are therefore a conservative lower bound on rbgo's true conformance — which is exactly what a ratchet should be. + +`cmd/phantom` is the other half of that honesty: the stubbed matchers make the +total too LOW, and a pass bought by an absence would make it too high. Neither is +visible in the number itself. diff --git a/scripts/conformance/rubyspec/cmd/phantom/main.go b/scripts/conformance/rubyspec/cmd/phantom/main.go new file mode 100644 index 00000000..8067b785 --- /dev/null +++ b/scripts/conformance/rubyspec/cmd/phantom/main.go @@ -0,0 +1,427 @@ +// Copyright (c) the go-embedded-ruby/ruby authors +// +// SPDX-License-Identifier: BSD-3-Clause + +// phantom finds ruby/spec examples that can pass BECAUSE A METHOD IS MISSING. +// +// Why it exists (#850). A spec that asserts an exception class cannot tell why +// it got one. core/array/rfind_spec.rb has +// +// it "raises a NoMethodError if the ifnone argument does not respond to #call …" do +// -> { [1, 2, 3].rfind(42) { |x| false } }.should.raise(NoMethodError) +// end +// +// which means "42 is not callable". On a build where `Array#rfind` does not +// exist at all, `[1,2,3].rfind(42)` raises NoMethodError for THAT reason and the +// example passes. Measured on 3a3fb818, that file scored `pass=1 error=15` with +// no rfind in the interpreter: fifteen honest errors and one pass the gate +// counted for a method it did not have. +// +// So BASELINE can hold passes that are not conformance, and nothing +// distinguished them. This names them. +// +// WHAT IT DOES. For every `describe "X#y"` or `describe "X.y"` block containing +// a NoMethodError expectation, it asks a real rbgo whether that method exists — +// `X.method_defined?(:y)` (or `private_method_defined?`, since a module_function +// is private) for the `#` form, `X.respond_to?(:y)` for the `.` form. No +// instance is constructed, so a receiver whose `new` needs arguments is still +// answerable. An answer of "no" means that example's pass, if it passes, is +// suspect. +// +// ⛔ WHAT IT CANNOT SEE, printed on every run rather than left in this comment: +// it reads the describe string, so an example whose receiver is not what the +// describe names is outside its population, and a describe naming an operator +// method (`[]`, `<=>`, `+`) is skipped because those do not match the identifier +// shape. Same class of limit as cmd/guards declaring its example counts a lower +// bound. A census that does not state its population is a census that will be +// read as a total. +// +// Exit status: 0 nothing suspect, 1 suspects found, 2 refused — could not read +// the corpus, could not run the interpreter, or its own control failed. +package main + +import ( + "bufio" + "flag" + "fmt" + "io/fs" + "os" + "os/exec" + "path/filepath" + "regexp" + "sort" + "strings" +) + +// NOTE ON THE FLAG PACKAGE. The fleet's rule is that command-line tools use +// cobra. This one does not, deliberately: it is a conformance census that lives +// beside cmd/ratchet and cmd/guards and is invoked the same way they are, and +// cobra is not a direct dependency of this module — adding one to the +// interpreter's go.mod for a script would enlarge the dependency graph of every +// consumer of a repository whose whole claim is pure Go and minimal. If the three +// should be converted together, that is one change and worth doing as one. + +// describeRe matches a describe line that names a receiver and a method. The +// receiver may be namespaced (`Math::DomainError`); the method may end in ?, ! +// or =. Operator methods are deliberately NOT matched — see the package comment. +var describeRe = regexp.MustCompile(`^\s*describe\s+"([A-Z][A-Za-z0-9_]*(?:::[A-Z][A-Za-z0-9_]*)*)([#.])([a-zA-Z_][A-Za-z0-9_]*[?!=]?)"`) + +// noMethodRe matches the two spellings of the expectation, mspec's and RSpec's. +var noMethodRe = regexp.MustCompile(`should\.raise\(NoMethodError|should\s+raise_error\(NoMethodError`) + +// anyDescribeRe matches a describe line whatever it names — a constant +// (`describe "RUBY_VERSION"`), a shared block (`describe :foo, shared: true`), a +// prose grouping (`describe "when empty"`). +// +// ⛔ It exists because attribution has to STOP somewhere. Without it, the +// expectation inside +// +// describe "String#squeeze" do … end +// describe :shared_thing, shared: true do +// it "raises a NoMethodError" do … end +// end +// +// was attributed to String#squeeze — a claim that file does not make, and the +// kind of false positive that sends a reader to check a non-issue. Its own test +// predicted it and the first implementation had it. +var anyDescribeRe = regexp.MustCompile(`^(\s*)describe[\s(]`) + +// absenceRe matches an assertion that the method is NOT there. +// +// ⛔ THIS IS THE FALSE-POSITIVE GUARD, and it was added because the first run +// produced two. core/complex/negative_spec.rb reads, in full: +// +// describe "Complex#negative?" do +// it "is undefined" do +// c = Complex(1) +// c.methods.should_not.include?(:negative?) +// -> { c.negative? }.should.raise(NoMethodError) +// end +// end +// +// MRI does not define Complex#negative? — Complex has no ordering — and the spec +// asserts that absence on purpose. rbgo passing it is CONFORMANCE, not a pass +// bought by a missing method, and reporting it would have sent a reader to fix +// something that is right. `core/complex/positive_spec.rb` is the same shape. +// +// So "the describe names a method the interpreter does not define" is not +// sufficient: an example that ALSO asserts the absence is asserting exactly what +// it got. Those are excluded, by their own assertion rather than by a list of +// file names — a denylist of paths would have gone stale at the next corpus bump +// and would not have covered the third such spec nobody had met yet. +var absenceRe = regexp.MustCompile(`should_not[.\s]*(?:include|respond_to)|respond_to\?\([^)]*\)\.should\s*==\s*false|should_not\.respond_to\?`) + +// site is one suspect expectation. +type site struct { + file string + line int + receiver string + kind string // "#" or "." + method string +} + +func (s site) name() string { return s.receiver + s.kind + s.method } + +// vocabularyCases is the parser's own control, run on EVERY invocation rather +// than only under `go test`: this tool is run by hand and from CI as a binary, +// and a describe pattern that silently stops matching reports an empty census, +// which is indistinguishable from a clean corpus. +var vocabularyCases = []struct { + line string + want string // "" when no match is expected +}{ + {`describe "Array#rfind" do`, "Array#rfind"}, + {`describe "Math.log1p" do`, "Math.log1p"}, + {` describe "Kernel#require" do`, "Kernel#require"}, + {`describe "String#start_with?" do`, "String#start_with?"}, + {`describe "Struct#each_pair" do`, "Struct#each_pair"}, + {`describe "Math::DomainError" do`, ""}, // a constant, no method named + {`describe "RUBY_VERSION" do`, ""}, + {`describe :array_find, shared: true do`, ""}, // a shared block + {`describe "Array#[]" do`, ""}, // operator: outside the population + {` it "raises a NoMethodError" do`, ""}, +} + +func checkVocabulary() error { + for _, c := range vocabularyCases { + m := describeRe.FindStringSubmatch(c.line) + got := "" + if m != nil { + got = m[1] + m[2] + m[3] + } + if got != c.want { + return fmt.Errorf("the describe pattern is wrong: %q read as %q, want %q", c.line, got, c.want) + } + } + for _, s := range []string{ + `-> { [1,2,3].rfind(42) { |x| false } }.should.raise(NoMethodError)`, + `expect { x.y }.should raise_error(NoMethodError)`, + } { + if !noMethodRe.MatchString(s) { + return fmt.Errorf("the expectation pattern missed %q", s) + } + } + if noMethodRe.MatchString(`-> { x.y }.should.raise(NameError)`) { + return fmt.Errorf("the expectation pattern matches NameError, which is a different claim") + } + return nil +} + +func main() { + root := flag.String("root", ".", "the ruby/spec checkout") + rbgo := flag.String("rbgo", "rbgo", "the interpreter to ask") + flag.Parse() + + if err := checkVocabulary(); err != nil { + fmt.Fprintf(os.Stderr, "phantom: %v\n", err) + os.Exit(2) + } + + sites, files, err := collect(*root) + if err != nil { + fmt.Fprintf(os.Stderr, "phantom: %v\n", err) + os.Exit(2) + } + // "N of M, never N alone": a walk that matched nothing must not read as a + // clean corpus. + if len(sites) == 0 { + fmt.Fprintf(os.Stderr, "phantom: no NoMethodError expectation found under %s in %d spec file(s); "+ + "either the corpus is not there or the patterns no longer fit\n", *root, files) + os.Exit(2) + } + + have, err := ask(*rbgo, sites) + if err != nil { + fmt.Fprintf(os.Stderr, "phantom: %v\n", err) + os.Exit(2) + } + + // ⛔ THE CONTROL, on this very corpus and this very interpreter, before any + // verdict: a method that certainly exists must come back present, and the + // absence detector must be able to answer "no" at all. Without the second + // half, an `ask` that returned "present" for everything would report a clean + // census — the most reassuring possible way for this tool to be broken. + if !probe(*rbgo, "Array", "#", "find") { + fmt.Fprintf(os.Stderr, "phantom: control failed — the interpreter says Array#find does not exist, "+ + "so a 'missing' answer means nothing here\n") + os.Exit(2) + } + if probe(*rbgo, "Array", "#", "no_such_method_aaa") { + fmt.Fprintf(os.Stderr, "phantom: control failed — the interpreter says a method that cannot exist "+ + "does exist, so nothing would ever be reported\n") + os.Exit(2) + } + + var suspect []site + for _, s := range sites { + if !have[s.name()] { + suspect = append(suspect, s) + } + } + + fmt.Printf("%d NoMethodError expectation(s) over %d spec file(s); %d name a method this interpreter does not define\n", + len(sites), files, len(suspect)) + fmt.Printf("population: describe strings of the form \"X#y\" or \"X.y\" only — operator methods and " + + "shared blocks are outside it, and an example whose receiver differs from its describe is invisible here\n") + if len(suspect) == 0 { + return + } + sort.Slice(suspect, func(i, j int) bool { + if suspect[i].file != suspect[j].file { + return suspect[i].file < suspect[j].file + } + return suspect[i].line < suspect[j].line + }) + for _, s := range suspect { + fmt.Printf("%s:%d\t%s is not defined — a pass here may be the absence, not the behaviour\n", + s.file, s.line, s.name()) + } + os.Exit(1) +} + +// collect walks the corpus and returns every NoMethodError expectation that sits +// under a describe naming a method, plus the number of spec files read. +// +// Attribution is to the MOST RECENT matching describe line, which is what a +// line-at-a-time read can honestly do: a nested describe that names a method +// takes over from its parent, and an expectation under a describe that names no +// method is skipped rather than guessed at. +func collect(root string) ([]site, int, error) { + var out []site + files := 0 + for _, tree := range []string{"language", "core"} { + dir := filepath.Join(root, tree) + err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || !strings.HasSuffix(path, "_spec.rb") { + return nil + } + files++ + f, oerr := os.Open(path) + if oerr != nil { + return oerr + } + defer f.Close() + rel, rerr := filepath.Rel(root, path) + if rerr != nil { + rel = path + } + var cur *site + curIndent := 0 + // Sites are held until the describe block ends, because an assertion + // of absence anywhere in it withdraws them all: the block is then + // asserting the absence it got, which is conformance. + var pending []site + asserts := false + flush := func() { + if !asserts { + out = append(out, pending...) + } + pending, asserts = nil, false + } + sc := bufio.NewScanner(f) + sc.Buffer(make([]byte, 1<<20), 1<<20) + for n := 1; sc.Scan(); n++ { + text := sc.Text() + if m := describeRe.FindStringSubmatch(text); m != nil { + flush() + cur = &site{file: rel, receiver: m[1], kind: m[2], method: m[3]} + curIndent = len(m[0]) - len(strings.TrimLeft(m[0], " \t")) + continue + } + if cur != nil && absenceRe.MatchString(text) { + asserts = true + continue + } + // A describe this cannot attribute ends the previous attribution + // only when it is a SIBLING or an outer block — judged by + // indentation. A describe nested INSIDE the method's own + // ("describe "Array#rfind" do" then " describe "with a block" do") + // is a sub-grouping of the same method and attribution continues + // through it, which is how ruby/spec is usually written. + // + // Indentation is a convention, not a parse. It is used here rather + // than counting `do`/`end` because the failure mode it risks is a + // MISS (attribution stops early, a candidate goes unreported), + // where mis-attribution would invent a claim. For a tool whose + // output sends someone to read a spec, that is the right direction + // to be wrong in — and the count it prints is therefore a lower + // bound, which the output says. + if m := anyDescribeRe.FindStringSubmatch(text); m != nil { + if indent := len(m[1]); cur != nil && indent <= curIndent { + flush() + cur = nil + } + continue + } + if cur != nil && noMethodRe.MatchString(text) { + s := *cur + s.line = n + pending = append(pending, s) + } + } + flush() + return sc.Err() + }) + if err != nil { + return nil, 0, fmt.Errorf("walking %s: %w", dir, err) + } + } + return out, files, nil +} + +// ask runs one Ruby program that reports, for each distinct method named, whether +// the interpreter defines it. One process for the whole census: asking per site +// would be hundreds of interpreter start-ups. +func ask(rbgo string, sites []site) (map[string]bool, error) { + seen := map[string]site{} + for _, s := range sites { + seen[s.name()] = s + } + names := make([]string, 0, len(seen)) + for n := range seen { + names = append(names, n) + } + sort.Strings(names) + + var b strings.Builder + b.WriteString(probeProgramPrelude) + for _, n := range names { + s := seen[n] + fmt.Fprintf(&b, "q(%q, %q, %q)\n", s.receiver, s.kind, s.method) + } + + out, err := run(rbgo, b.String()) + if err != nil { + return nil, fmt.Errorf("asking the interpreter: %w", err) + } + have := map[string]bool{} + for _, line := range strings.Split(out, "\n") { + parts := strings.Split(line, "\t") + if len(parts) != 4 || parts[0] != "ANSWER" { + continue + } + have[parts[1]] = parts[2] == "yes" + } + if len(have) != len(names) { + return nil, fmt.Errorf("asked about %d method(s) and got %d answer(s); "+ + "an unanswered question would read as absent", len(names), len(have)) + } + return have, nil +} + +// probe asks a single question, used only for the control. +func probe(rbgo, receiver, kind, method string) bool { + out, err := run(rbgo, probeProgramPrelude+fmt.Sprintf("q(%q, %q, %q)\n", receiver, kind, method)) + if err != nil { + return false + } + return strings.Contains(out, "\tyes\t") +} + +// probeProgramPrelude defines the question in Ruby. A module_function is a +// PRIVATE instance method, so `method_defined?` alone would report Math.log1p's +// instance side as absent; both are asked. +const probeProgramPrelude = ` +def q(const, kind, meth) + present = begin + if Object.const_defined?(const) + k = Object.const_get(const) + if kind == "#" + k.method_defined?(meth.to_sym) || k.private_method_defined?(meth.to_sym) + else + k.respond_to?(meth.to_sym) + end + else + false + end + rescue StandardError + false + end + puts "ANSWER\t#{const}#{kind}#{meth}\t#{present ? "yes" : "no"}\t." +end +` + +// run executes src with the interpreter and returns its stdout. +func run(rbgo, src string) (string, error) { + f, err := os.CreateTemp("", "phantom-*.rb") + if err != nil { + return "", err + } + defer os.Remove(f.Name()) + if _, err := f.WriteString(src); err != nil { + f.Close() + return "", err + } + if err := f.Close(); err != nil { + return "", err + } + cmd := exec.Command(rbgo, f.Name()) + out, err := cmd.CombinedOutput() + if err != nil { + return "", fmt.Errorf("%s: %v: %s", rbgo, err, strings.TrimSpace(string(out))) + } + return string(out), nil +} diff --git a/scripts/conformance/rubyspec/cmd/phantom/main_test.go b/scripts/conformance/rubyspec/cmd/phantom/main_test.go new file mode 100644 index 00000000..424680d2 --- /dev/null +++ b/scripts/conformance/rubyspec/cmd/phantom/main_test.go @@ -0,0 +1,264 @@ +// Copyright (c) the go-embedded-ruby/ruby authors +// +// SPDX-License-Identifier: BSD-3-Clause + +package main + +import ( + "os" + "path/filepath" + "testing" +) + +// The parser's own control. It also runs inside the binary on every invocation — +// see checkVocabulary — because this tool is used as a binary from CI, which +// never runs this file. Having it in both places is deliberate: here it fails a +// change at review time, there it protects a consumer. +func TestVocabulary(t *testing.T) { + if err := checkVocabulary(); err != nil { + t.Fatal(err) + } +} + +// writeSpec puts a spec file in a fake corpus and returns the root. +func writeSpec(t *testing.T, tree, name, body string) string { + t.Helper() + root := t.TempDir() + dir := filepath.Join(root, tree) + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + // Both trees are walked; the other must exist or the walk errors. + other := "core" + if tree == "core" { + other = "language" + } + if err := os.MkdirAll(filepath.Join(root, other), 0o755); err != nil { + t.Fatal(err) + } + return root +} + +// ⛔ The shape this tool exists for, verbatim from core/array/rfind_spec.rb: the +// expectation means "42 is not callable", and on a build without Array#rfind it +// is satisfied by rfind's own NoMethodError instead. +func TestCollectAttributesToTheDescribe(t *testing.T) { + root := writeSpec(t, "core", "rfind_spec.rb", `require_relative '../../spec_helper' + +describe "Array#rfind" do + it "returns the last element" do + [1, 2, 3].rfind { |x| x == 2 }.should == 2 + end + + it "raises a NoMethodError if the ifnone argument does not respond to #call" do + -> { [1, 2, 3].rfind(42) { |x| false } }.should.raise(NoMethodError) + end +end +`) + sites, files, err := collect(root) + if err != nil { + t.Fatal(err) + } + if files != 1 { + t.Errorf("read %d spec file(s), want 1", files) + } + if len(sites) != 1 { + t.Fatalf("found %d site(s), want 1: %+v", len(sites), sites) + } + if got := sites[0].name(); got != "Array#rfind" { + t.Errorf("attributed to %q, want %q", got, "Array#rfind") + } + if sites[0].line != 9 { + t.Errorf("line %d, want 9", sites[0].line) + } +} + +// A nested describe that names a method takes over from its parent, and an +// expectation under a describe that names NO method is skipped rather than +// guessed at — attributing it to an enclosing describe would put a claim in the +// census that the file does not make. +func TestCollectSkipsWhatItCannotAttribute(t *testing.T) { + root := writeSpec(t, "language", "defined_spec.rb", ` +describe "RUBY_VERSION" do + it "raises a NoMethodError for nonsense" do + -> { RUBY_VERSION.nope }.should.raise(NoMethodError) + end +end + +describe "String#squeeze" do + it "raises a NoMethodError" do + -> { "x".squeeze(42) }.should.raise(NoMethodError) + end +end + +describe :shared_thing, shared: true do + it "raises a NoMethodError" do + -> { nil.nope }.should.raise(NoMethodError) + end +end +`) + sites, _, err := collect(root) + if err != nil { + t.Fatal(err) + } + // The first describe names a constant, not a method; the third is a shared + // block. Only String#squeeze is attributable — but note the shared block + // follows it, so a parser that simply never cleared `cur` would attribute + // the third expectation to String#squeeze and report two. + if len(sites) != 1 { + t.Fatalf("found %d site(s), want 1: %+v", len(sites), sites) + } + if got := sites[0].name(); got != "String#squeeze" { + t.Errorf("attributed to %q, want %q", got, "String#squeeze") + } +} + +// The other half of the attribution rule: a describe nested INSIDE a method's +// own is a sub-grouping of the same method, and attribution must survive it. +// Clearing on every unattributable describe would be safe but would lose these, +// which is how ruby/spec is usually written. +func TestCollectAttributionSurvivesANestedGrouping(t *testing.T) { + root := writeSpec(t, "core", "nested_spec.rb", ` +describe "Array#rfind" do + describe "with a block" do + it "raises a NoMethodError" do + -> { [1].rfind(42) { false } }.should.raise(NoMethodError) + end + end + + describe "without a block" do + it "raises a NoMethodError too" do + -> { [1].rfind(42) }.should.raise(NoMethodError) + end + end +end + +describe "Array#first" do + it "raises a NoMethodError" do + -> { [1].first(42) }.should.raise(NoMethodError) + end +end +`) + sites, _, err := collect(root) + if err != nil { + t.Fatal(err) + } + if len(sites) != 3 { + t.Fatalf("found %d site(s), want 3: %+v", len(sites), sites) + } + want := []string{"Array#rfind", "Array#rfind", "Array#first"} + for i, w := range want { + if got := sites[i].name(); got != w { + t.Errorf("site %d attributed to %q, want %q", i, got, w) + } + } +} + +// ⛔ THE FALSE POSITIVES THE FIRST RUN PRODUCED, verbatim from +// core/complex/negative_spec.rb. MRI does not define Complex#negative? — Complex +// has no ordering — and the spec asserts that absence on purpose, so rbgo +// passing it is conformance. An example that asserts the absence is asserting +// exactly what it got, and reporting it would send a reader to fix something +// that is right. +func TestCollectWithdrawsAnExampleThatAssertsTheAbsence(t *testing.T) { + root := writeSpec(t, "core", "negative_spec.rb", `require_relative '../../spec_helper' + +describe "Complex#negative?" do + it "is undefined" do + c = Complex(1) + + c.methods.should_not.include?(:negative?) + + -> { + c.negative? + }.should.raise(NoMethodError) + end +end +`) + sites, _, err := collect(root) + if err != nil { + t.Fatal(err) + } + if len(sites) != 0 { + t.Errorf("found %d site(s), want 0 — the example asserts the absence: %+v", len(sites), sites) + } +} + +// And the withdrawal must be narrow: one describe asserting an absence must not +// silence a sibling that does not. Without the flush-per-block, a single +// `should_not.include?` anywhere in a file would hide every candidate in it. +func TestTheAbsenceWithdrawalIsPerDescribeBlock(t *testing.T) { + root := writeSpec(t, "core", "mixed_spec.rb", ` +describe "Complex#negative?" do + it "is undefined" do + Complex(1).methods.should_not.include?(:negative?) + -> { Complex(1).negative? }.should.raise(NoMethodError) + end +end + +describe "Array#rfind" do + it "raises a NoMethodError if the ifnone argument does not respond to #call" do + -> { [1, 2, 3].rfind(42) { |x| false } }.should.raise(NoMethodError) + end +end +`) + sites, _, err := collect(root) + if err != nil { + t.Fatal(err) + } + if len(sites) != 1 { + t.Fatalf("found %d site(s), want 1: %+v", len(sites), sites) + } + if got := sites[0].name(); got != "Array#rfind" { + t.Errorf("kept %q, want the sibling that asserts no absence (%q)", got, "Array#rfind") + } +} + +// NameError is a different claim and must not be collected: a spec asserting +// NameError is usually about a constant, and treating it as a method-absence +// candidate would pad the census. +func TestCollectIgnoresOtherExceptionClasses(t *testing.T) { + root := writeSpec(t, "core", "other_spec.rb", ` +describe "Array#first" do + it "raises a NameError" do + -> { NoSuchConstant }.should.raise(NameError) + end + + it "raises an ArgumentError" do + -> { [1].first(1, 2) }.should.raise(ArgumentError) + end +end +`) + sites, _, err := collect(root) + if err != nil { + t.Fatal(err) + } + if len(sites) != 0 { + t.Errorf("found %d site(s), want 0: %+v", len(sites), sites) + } +} + +// RSpec's spelling as well as mspec's, since the corpus carries both. +func TestCollectReadsBothSpellings(t *testing.T) { + root := writeSpec(t, "core", "both_spec.rb", ` +describe "Hash#dig" do + it "one" do + -> { {}.dig(42) }.should.raise(NoMethodError) + end + + it "two" do + expect { {}.dig(42) }.should raise_error(NoMethodError) + end +end +`) + sites, _, err := collect(root) + if err != nil { + t.Fatal(err) + } + if len(sites) != 2 { + t.Fatalf("found %d site(s), want 2: %+v", len(sites), sites) + } +}