[rhdunn]

I use promptfoo for evaluation. I'm experimenting with tests for my workflow/use cases.

I have a custom assert for loop/repeat detection that works well:

    def count_repeats(text: str, length: int) -> int:
        n = len(text)
        pattern = text[n - length : n]
        count = 1 # Include the end of the string as matching the substring.

        text = text[: -length]
        while text.endswith(pattern):
            text = text[: -length]
            count = count + 1

        return count


    def repeats(output: str, context: dict[str, any]) -> bool|float|dict[str, any]:
        threshold = context.get('config', {}).get('threshold', 3)
        count = 0
        length = 0

        for n in range(1, (len(output) // 2) + 1):
            n_count = count_repeats(output, n)
            if n_count > count:
                count = n_count
                length = n

        if count >= threshold:
            return { 'pass': True, 'score': 1.0, 'reason': f'Output repeats {count} times with length {length}.' }
        else:
            return { 'pass': False, 'score': 0.0, 'reason': f'Output doesn\'t repeat {threshold} or more times.' }


    def no_repeats(output: str, context) -> dict[str, any]:
        result = repeats(output, context)
        result['pass'] = not result['pass']
        result['score'] = 1.0 - result['score']
        return result

Just add it to your promptfooconfig.yaml:

    defaultTest:
      assert:
        - # ----- The output doesn't repeat/get stuck in a loop.
          type: python
          value: file://asserts/repeat.py:no_repeats