How to Fail With

100% Test Coverage

Stuart Halloway
stu@thinkrelevance.com

1
 What Kind of
Coverage are You?

2
lining up

3
module Math
def max(a, b)
if a > b then a else b end
end
end

require "test/unit"
require "math"

class MathTest < Test::Unit::TestCase

include Math

def test_max
assert_equal 2, max(2, 1)
end
end

4
5
module Math
def max(a, b)
if a > b then
a
else
b
end
end
end

6
7
require "test/unit"
require "math"

class MathTest < Test::Unit::TestCase

include Math

def test_first_parameter_as_max
assert_equal 2, max(2, 1)
end

def test_second_parameter_as_max
assert_equal 2, max(1, 2)
end
end

8
9
branching out

10
// Math.java
public class Math {
public static int max(int a, int b) {
return a > b ? a : b;
}
}

// MathTest.java
import junit.framework.TestCase;

public class MathTest extends TestCase {

public void testMax() {
assertEquals(2, Math.max(2, 1));
}
}

11
12
import junit.framework.TestCase;

public class MathTest extends TestCase {

public void testFirstParameterAsMax() {
assertEquals(2, Math.max(2, 1));
}

public void testSecondParameterAsMax() {

assertEquals(2, Math.max(1, 2));
}
}

13
14
 101 coverage measures

kaner.com/coverage.htm

15
8 Flavors of Fail

16
Underspecification

17
Underspecification

http://www.flickr.com/photos/abuzavi/92491999
18
“While it's possible to specify a
product ambiguously, it is not
possible to build a product
ambiguously.”
Tom DeMarco and Timothy Lister

19
# code under test
#############################
MIN_FREE_SHIPPING_PRICE = 25.0

def free_shipping?(total_order_price)
total_order_price > MIN_FREE_SHIPPING_PRICE
end

20
# code under test
#############################
MIN_FREE_SHIPPING_PRICE = 25.0

def free_shipping?(total_order_price)
total_order_price > MIN_FREE_SHIPPING_PRICE
end

# test suite?
#############################
def test_free_shipping_returns_true_for_order_above_min_price
assert free_shipping?(MIN_FREE_SHIPPING_PRICE + 1)
end

def test_free_shipping_returns_false_for_order_below_min_price
assert !free_shipping?(MIN_FREE_SHIPPING_PRICE - 1)
end

21
what about my
$25.00 order!

22
# code under test
#############################
MIN_FREE_SHIPPING_PRICE = 25.0

def free_shipping?(total_order_price)
total_order_price >=
= MIN_FREE_SHIPPING_PRICE
end

# test suite
#############################
def test_free_shipping_returns_true_for_order_above_min_price
assert free_shipping?(MIN_FREE_SHIPPING_PRICE + 1)
end

def test_free_shipping_returns_false_for_order_below_min_price
assert !free_shipping?(MIN_FREE_SHIPPING_PRICE - 1)
end

def test_free_shipping_returns_true_for_order_equal_to_min_price
assert free_shipping?(MIN_FREE_SHIPPING_PRICE)
end

23
edge cases matter

24
 “Code coverage can only tell you
how much of your code is being
tested. It cannot tell you how
much code you still need to write.”
Eric Sink

25
Incidental Coverage

26
class ProductsController < ApplicationController
def index
@products = Product.find(:all)

respond_to do |format|
format.html # index.html.erb
format.xml { render :xml => @products }
end
end
end

27
class ProductsControllerTest < ActionController::TestCase
def test_should_get_index
get :index
end
end

28
29
failure to assert

30
testing != System.out.println()

31
class ProductsControllerTest < ActionController::TestCase
def test_should_get_index
get :index
assert_response :success
assert_not_nil assigns(:products)
end
end

32
100% covered

50% tested

33
def index
@products = Product.find(:all)

respond_to do |format|
format.html # index.html.erb
format.xml { render :xml => @products }
end
end

34
def index
@products = Product.find(:all)

respond_to do |format|
format.html # index.html.erb
format.xml { raise :fatal_error }
end
end

Still no failing tests?

35
def test_should_get_index_formatted_for_html
get :index
assert_response :success
assert_not_nil assigns(:products)
end

def test_should_get_index_formatted_for_xml
@request.env['HTTP_ACCEPT'] = 'application/xml'
get :index
assert_response :success
assert_not_nil assigns(:products)
end

36
Invisible Code

37
<% if (@article.published_at + @article.comment_age.days) > Time.now %>
<% form_for(@comment) do |f| %>
<!-- capture witty comment here (maybe) -->
<% end %>
<% end %>

38
<% if (@article.published_at + @article.comment_age.days) > Time.now %>
<% form_for(@comment) do |f| %>
<!-- capture witty comment here (maybe) -->
<% end %>
<% end %>

39
<% if @article.accept_comments? %>
<% form_for(@comment) do |f| %>
<!-- capture witty comment here -->
<% end %>
<% end %>

This (more respectable) solution inspired by Mephisto. See http://github.com/technoweenie/mephisto.

40
class Article < ActiveRecord::Base
def accept_comments?
comment_age == 0 || comments_expired_at > Time.now.utc
end

def comments_expired_at
published_at + comment_age.days
end
end

41
class ArticleTest < ActiveSupport::TestCase
def test_comments_always_accepted_when_comment_days_is_zero
a = Article.new(:comment_age => 0, :published_at => 5.years.ago.utc)
assert a.accept_comments?
end

def test_comments_accepted_prior_to_expiration
a = Article.new(:comment_age => 5, :published_at => 2.days.ago.utc)
assert a.accept_comments?
end

def test_comments_not_accepted_past_expiration
a = Article.new(:comment_age => 10, :published_at => 12.days.ago.utc)
assert !a.accept_comments?
end
end

42
Overspecification

43
class ProductsControllerTest < ActionController::TestCase
def test_something
product = Product.create(:name => "Frisbee", :price => 5.00)
get :show, :id => product.id
assert_response :success
product = assigns(:product)
assert_not_nil product
assert product.valid?
assert product.name == "Frisbee"
assert product.price == 5.00
end
end

44
class ProductsController < ApplicationController
def show
@product = Product.find(params[:id])
end
end

class ProductsControllerTest < ActionController::TestCase

def test_something
product = Product.create(:name => "Frisbee", :price => 5.00)
get :show, :id => product.id
assert_response :success
product = assigns(:product)
assert_not_nil product
assert product.valid?
assert product.name == "Frisbee"
assert product.price == 5.00
end
end

45
class ProductsController < ApplicationController
def show
@product = Product.find(params[:id])
end
end

class ProductsControllerTest < ActionController::TestCase

def test_should_show_product
product = create_product
get :show, :id => product.id
assert_response :success
assert_equal product, assigns(:product)
end
end

46
communicate essence

47
Fragile Mocking

48
http://www.flickr.com/photos/mrbill/96827834
49
# code under test - round 1
#############################
def create
@product = Product.create(params[:product])
if !@product.new_record?
flash[:notice] = 'Product was successfully created.'
redirect_to(@product)
else
render :action => "new"
end
end

50
# fragile mocking - round 1
#############################
def test_should_create_product
product = Product.new
product.stubs(:new_record?).returns(false)
Product.expects(:create).returns(product)
post :create, :product => {}
end

# code under test - round 1

#############################
def create
@product = Product.create(params[:product])
if !@product.new_record?
flash[:notice] = 'Product was successfully created.'
redirect_to(@product)
else
render :action => "new"
end
end

51
# code under test - round 2
#############################
def create
@product = Product.new(params[:product])
if @product.save
flash[:notice] = 'Product was successfully created.'
redirect_to(@product)
else
render :action => "new"
end
end

52
# fragile mocking - round 2
#############################
def test_should_create_product
Product.any_instance.expects(:save).returns(true)
post :create, :product => {}
end

# code under test - round 2

#############################
def create
@product = Product.new(params[:product])
if @product.save
flash[:notice] = 'Product was successfully created.'
redirect_to(@product)
else
render :action => "new"
end
end

53
# fragile mocking - round 1
#############################

Fail!
def test_should_create_product
product = Product.new
product.stubs(:new_record?).returns(false)
Product.expects(:create).returns(product)
post :create, :product => {}
end

# code under test - round 2

#############################
def create
@product = Product.new(params[:product])
if @product.save
flash[:notice] = 'Product was successfully created.'
redirect_to(@product)
else
render :action => "new"
end
end

54
def test_should_create_product
assert_difference('Product.count') do
post :create, :product => {"name" => "Frisbee", "price" => 5.00}
end
end

55
 "Well written tests specify as
little implementation detail as
possible."
Jay Fields

56
The Ugly Mirror

57
# code under test
#############################
User = Struct.new(:first_name, :last_name, :email) do
def to_s
"#{last_name}, #{first_name} <#{email}>"
end
end

58
# code under test
#############################
User = Struct.new(:first_name, :last_name, :email) do
def to_s
"#{last_name}, #{first_name} <#{email}>"
end
end

# test suite?
#############################
def test_to_s_includes_name_and_email
user = User.new("John", "Smith", "jsmith@example.com")
assert_equal "#{user.last_name}, #{user.first_name} <#{user.email}>",
user.to_s
end

59
# code under test
#############################
User = Struct.new(:first_name, :last_name, :email) do
def to_s
"#{last_name}, #{first_name} <#{email}>"
end
end

# test suite
#############################
def test_to_s_includes_name_and_email
user = User.new("John", "Smith", "jsmith@example.com")
assert_equal "#{user.last_name}, #{user.first_name} <#{user.email}>",
user.to_s
assert_equal "Smith, John <jsmith@example.com>", user.to_s
end

60
expect literals

61
Slow Tests

62
http://www.flickr.com/photos/cindy47452/441316645
63
Shallow Tests

64
http://www.flickr.com/photos/addictive_picasso/442553488
65
"No automated test suite can ever
replace exploratory testing."
Jay Fields

66
http://www.flickr.com/photos/jasmic/279741827
67
 “[Coverage tools] are only
helpful if they’re used to enhance
thought, not replace it.”
Brian Marick

68
Thanks to Jason Rudolph, for preparing these
slides based on Relevance’s experience with
100% test coverage over the past year, and to
the whole Relevance team.

Slides: jasonrudolph.com/downloads
Blog: jasonrudolph.com/blog
Relevance: thinkrelevance.com
This presentation is published under the Creative Commons Attribution Noncommercial Share Alike License Version 3.0.
(Please see http://creativecommons.org/licenses/by-nc-sa/3.0 for complete details.)

69