How to add new mutation for cosmic-ray framework

Mutation testing is a concept that I heard some time ago but lately on anti-if course (I highly recommend!) done by Andrzej Krzywda I had a chance to use it in practice.

I notice a big difference in the number of mutations generated by different frameworks (mutmut, cosmic-ray, and mutpy) when I was refactoring Gilded Rose Kata and playing around with mutation testing (you can find it here). Mutmut was easier to work with (almost working out of the box) but cosmic-ray generated a lot more mutations that should be covered with tests.

Cosmic-ray needs more configuration and is a little harder to use but delivers a complete framework with many features. Most interesting for me was the ability to add new mutations as a plug-in.

Example code

Based on Gilded Rose Kata I prepared a little code with tests that have 100% test coverage (with branches).

class Quality:
    amount: int

    def __init__(self, amount: int) -> None:
        self.amount = amount

    def increase(self) -> None:
        if self.amount < 50:
            self.amount += 1

    def less_than_50(self) -> bool:
        return self.amount < 50

    def reset(self) -> None:
        self.amount = 0


def update(sell_in: int, quality: int) -> int:
    quality = Quality(quality)
    quality.increase()
    if quality.less_than_50():
        if sell_in < 11:
            quality.increase()
        if sell_in < 6:
            quality.increase()
        sell_in -= 1
    if sell_in < 0:
        quality.reset()

    return quality.amount
from unittest import TestCase

from example import update


class TestUpdate(TestCase):
    def test_update(self):
        self.assertEqual(update(sell_in=0, quality=20), 0)
        self.assertEqual(update(sell_in=1, quality=20), 23)
        self.assertEqual(update(sell_in=4, quality=20), 23)
        self.assertEqual(update(sell_in=5, quality=20), 23)
        self.assertEqual(update(sell_in=6, quality=20), 22)
        self.assertEqual(update(sell_in=8, quality=20), 22)
        self.assertEqual(update(sell_in=10, quality=20), 22)
        self.assertEqual(update(sell_in=10, quality=50), 50)
        self.assertEqual(update(sell_in=11, quality=20), 21)
        self.assertEqual(update(sell_in=11, quality=49), 50)
        self.assertEqual(update(sell_in=11, quality=50), 50)
        self.assertEqual(update(sell_in=13, quality=20), 21)
        self.assertEqual(update(sell_in=13, quality=51), 51)

But when use mutation testing you will find out that there are situations that are not covered with tests. Here are this mutations.

     def less_than_50(self) -> bool:
-        return self.amount < 50
+        return self.amount != 50
     def less_than_50(self) -> bool:
-        return self.amount < 50
+        return self.amount <= 50
     def less_than_50(self) -> bool:
-        return self.amount < 50
+        return self.amount < 51
     def less_than_50(self) -> bool:
-        return self.amount < 50
+        return self.amount < 49

Actually, there is no way to cover these mutations just because it’s dead code. Checking of the amount limit is duplicated and used twice in code. Ruby mutant framework is generating more mutations on the ruby version than cosmic-ray. There is a mutator that replaces the whole if condition with True and False. When you see that changing the whole condition is not making any difference then this condition is not needed. Let’s check if it’s hard to add a new mutation to cosmic-ray.

The new mutation for cosmic-ray

from abc import ABC
from typing import Dict, Iterator, Text, Tuple

import parso
from parso.python.tree import Node, IfStmt, Operator
from cosmic_ray.operators.operator import Operator as Mutator

Line = int
Column = int
Pos = Tuple[Line, Column]
Name = Text


class IfReplacer(Mutator, ABC):
    STATEMENT = NotImplemented

    def mutation_positions(self, node: Node) -> Tuple[Pos, Pos]:
        if isinstance(node, IfStmt):
            yield node.start_pos, node.end_pos

    def mutate(self, node: Node, index):
        children = [node.children[0], self.STATEMENT]

        for i, c in enumerate(node.children):
            if isinstance(c, Operator) and c.value == ':':
                children.extend(node.children[i:])
                break

        node.children = children
        return node


class IfTrueReplacer(IfReplacer):
    STATEMENT = parso.parse(' True')


class IfFalseReplacer(IfReplacer):
    STATEMENT = parso.parse(' False')


operators = {
    'if-to-true-replacer': IfTrueReplacer,
    'if-to-false-replacer': IfFalseReplacer,
}


class Provider:
    _operators: Dict[Name, Mutator] = {
        'if-to-true-replacer': IfTrueReplacer,
        'if-to-false-replacer': IfFalseReplacer,
    }

    def __iter__(self) -> Iterator[Text]:
        return iter(Provider._operators)

    def __getitem__(self, name: Name) -> MutateOperator:
        return Provider._operators[name]
from setuptools import find_packages, setup


setup(
    name='mutators',
    py_modules=['mutators'],
    python_requires='>=3.7',
    zip_safe=True,
    entry_points={
        'cosmic_ray.operator_providers': [
            'custom = mutators:Provider',
        ]
    }
)

New mutation at work

So as I was expecting new mutation shows more precise that one of condition is not needed.

 def update(sell_in: int, quality: int) -> int:
     quality = Quality(quality)
     quality.increase()
-    if quality.less_than_50():
+    if True:
         if sell_in < 11:
             quality.increase()
         if sell_in < 6:

Now we see that we can just remove this if statement and less_than_50 method.

class Quality:
    amount: int

    def __init__(self, amount: int) -> None:
        self.amount = amount

    def increase(self) -> None:
        if self.amount < 50:
            self.amount += 1

    def reset(self) -> None:
        self.amount = 0


def update(sell_in: int, quality: int) -> int:
    quality = Quality(quality)
    quality.increase()
    if sell_in < 11:
        quality.increase()
    if sell_in < 6:
        quality.increase()
    sell_in -= 1
    if sell_in < 0:
        quality.reset()

    return quality.amount

Hi there 👋
It’s nice to meet you.

Sign up to join my mailing list.

I don’t spam!