Mutation testing is a concept that I heard some time ago but lately on anti-if course (I highly recommend!) done by Andrzej Krzywda I had a chance to use it in practice.
I notice a big difference in the number of mutations generated by different frameworks (mutmut, cosmic-ray, and mutpy) when I was refactoring Gilded Rose Kata and playing around with mutation testing (you can find it here). Mutmut was easier to work with (almost working out of the box) but cosmic-ray generated a lot more mutations that should be covered with tests.
Cosmic-ray needs more configuration and is a little harder to use but delivers a complete framework with many features. Most interesting for me was the ability to add new mutations as a plug-in.
Example code
Based on Gilded Rose Kata I prepared a little code with tests that have 100% test coverage (with branches).
class Quality:
amount: int
def __init__(self, amount: int) -> None:
self.amount = amount
def increase(self) -> None:
if self.amount < 50:
self.amount += 1
def less_than_50(self) -> bool:
return self.amount < 50
def reset(self) -> None:
self.amount = 0
def update(sell_in: int, quality: int) -> int:
quality = Quality(quality)
quality.increase()
if quality.less_than_50():
if sell_in < 11:
quality.increase()
if sell_in < 6:
quality.increase()
sell_in -= 1
if sell_in < 0:
quality.reset()
return quality.amountfrom unittest import TestCase
from example import update
class TestUpdate(TestCase):
def test_update(self):
self.assertEqual(update(sell_in=0, quality=20), 0)
self.assertEqual(update(sell_in=1, quality=20), 23)
self.assertEqual(update(sell_in=4, quality=20), 23)
self.assertEqual(update(sell_in=5, quality=20), 23)
self.assertEqual(update(sell_in=6, quality=20), 22)
self.assertEqual(update(sell_in=8, quality=20), 22)
self.assertEqual(update(sell_in=10, quality=20), 22)
self.assertEqual(update(sell_in=10, quality=50), 50)
self.assertEqual(update(sell_in=11, quality=20), 21)
self.assertEqual(update(sell_in=11, quality=49), 50)
self.assertEqual(update(sell_in=11, quality=50), 50)
self.assertEqual(update(sell_in=13, quality=20), 21)
self.assertEqual(update(sell_in=13, quality=51), 51)
But when use mutation testing you will find out that there are situations that are not covered with tests. Here are this mutations.
def less_than_50(self) -> bool:
- return self.amount < 50
+ return self.amount != 50 def less_than_50(self) -> bool:
- return self.amount < 50
+ return self.amount <= 50 def less_than_50(self) -> bool:
- return self.amount < 50
+ return self.amount < 51 def less_than_50(self) -> bool:
- return self.amount < 50
+ return self.amount < 49Actually, there is no way to cover these mutations just because it’s dead code. Checking of the amount limit is duplicated and used twice in code. Ruby mutant framework is generating more mutations on the ruby version than cosmic-ray. There is a mutator that replaces the whole if condition with True and False. When you see that changing the whole condition is not making any difference then this condition is not needed. Let’s check if it’s hard to add a new mutation to cosmic-ray.
The new mutation for cosmic-ray
from abc import ABC
from typing import Dict, Iterator, Text, Tuple
import parso
from parso.python.tree import Node, IfStmt, Operator
from cosmic_ray.operators.operator import Operator as Mutator
Line = int
Column = int
Pos = Tuple[Line, Column]
Name = Text
class IfReplacer(Mutator, ABC):
STATEMENT = NotImplemented
def mutation_positions(self, node: Node) -> Tuple[Pos, Pos]:
if isinstance(node, IfStmt):
yield node.start_pos, node.end_pos
def mutate(self, node: Node, index):
children = [node.children[0], self.STATEMENT]
for i, c in enumerate(node.children):
if isinstance(c, Operator) and c.value == ':':
children.extend(node.children[i:])
break
node.children = children
return node
class IfTrueReplacer(IfReplacer):
STATEMENT = parso.parse(' True')
class IfFalseReplacer(IfReplacer):
STATEMENT = parso.parse(' False')
operators = {
'if-to-true-replacer': IfTrueReplacer,
'if-to-false-replacer': IfFalseReplacer,
}
class Provider:
_operators: Dict[Name, Mutator] = {
'if-to-true-replacer': IfTrueReplacer,
'if-to-false-replacer': IfFalseReplacer,
}
def __iter__(self) -> Iterator[Text]:
return iter(Provider._operators)
def __getitem__(self, name: Name) -> MutateOperator:
return Provider._operators[name]
from setuptools import find_packages, setup
setup(
name='mutators',
py_modules=['mutators'],
python_requires='>=3.7',
zip_safe=True,
entry_points={
'cosmic_ray.operator_providers': [
'custom = mutators:Provider',
]
}
)
New mutation at work
So as I was expecting new mutation shows more precise that one of condition is not needed.
def update(sell_in: int, quality: int) -> int:
quality = Quality(quality)
quality.increase()
- if quality.less_than_50():
+ if True:
if sell_in < 11:
quality.increase()
if sell_in < 6:Now we see that we can just remove this if statement and less_than_50 method.
class Quality:
amount: int
def __init__(self, amount: int) -> None:
self.amount = amount
def increase(self) -> None:
if self.amount < 50:
self.amount += 1
def reset(self) -> None:
self.amount = 0
def update(sell_in: int, quality: int) -> int:
quality = Quality(quality)
quality.increase()
if sell_in < 11:
quality.increase()
if sell_in < 6:
quality.increase()
sell_in -= 1
if sell_in < 0:
quality.reset()
return quality.amount