kiln_ai.adapters.prompt_builders

  1import json
  2from abc import ABCMeta, abstractmethod
  3from typing import Dict
  4
  5from kiln_ai.datamodel import Task, TaskRun
  6from kiln_ai.utils.formatting import snake_case
  7
  8
  9class BasePromptBuilder(metaclass=ABCMeta):
 10    """Base class for building prompts from tasks.
 11
 12    Provides the core interface and basic functionality for prompt builders.
 13    """
 14
 15    def __init__(self, task: Task):
 16        """Initialize the prompt builder with a task.
 17
 18        Args:
 19            task (Task): The task containing instructions and requirements.
 20        """
 21        self.task = task
 22
 23    def prompt_id(self) -> str | None:
 24        """Returns the ID of the prompt, scoped to this builder.
 25
 26        Returns:
 27            str | None: The ID of the prompt, or None if not set.
 28        """
 29        return None
 30
 31    def build_prompt(self, include_json_instructions: bool = False) -> str:
 32        """Build and return the complete prompt string.
 33
 34        Returns:
 35            str: The constructed prompt.
 36        """
 37        prompt = self.build_base_prompt()
 38
 39        if include_json_instructions and self.task.output_schema():
 40            prompt = (
 41                prompt
 42                + f"\n\n# Format Instructions\n\nReturn a JSON object conforming to the following schema:\n```\n{self.task.output_schema()}\n```"
 43            )
 44
 45        return prompt
 46
 47    @abstractmethod
 48    def build_base_prompt(self) -> str:
 49        """Build and return the complete prompt string.
 50
 51        Returns:
 52            str: The constructed prompt.
 53        """
 54        pass
 55
 56    @classmethod
 57    def prompt_builder_name(cls) -> str:
 58        """Returns the name of the prompt builder, to be used for persisting into the datastore.
 59
 60        Default implementation gets the name of the prompt builder in snake case. If you change the class name, you should override this so prior saved data is compatible.
 61
 62        Returns:
 63            str: The prompt builder name in snake_case format.
 64        """
 65        return snake_case(cls.__name__)
 66
 67    def build_user_message(self, input: Dict | str) -> str:
 68        """Build a user message from the input.
 69
 70        Args:
 71            input (Union[Dict, str]): The input to format into a message.
 72
 73        Returns:
 74            str: The formatted user message.
 75        """
 76        if isinstance(input, Dict):
 77            return f"The input is:\n{json.dumps(input, indent=2, ensure_ascii=False)}"
 78
 79        return f"The input is:\n{input}"
 80
 81    def chain_of_thought_prompt(self) -> str | None:
 82        """Build and return the chain of thought prompt string.
 83
 84        Returns:
 85            str: The constructed chain of thought prompt.
 86        """
 87        return None
 88
 89    def build_prompt_for_ui(self) -> str:
 90        """Build a prompt for the UI. It includes additional instructions (like chain of thought), even if they are passed to the model in stages.
 91
 92        Designed for end-user consumption, not for model consumption.
 93
 94        Returns:
 95            str: The constructed prompt string.
 96        """
 97        base_prompt = self.build_prompt()
 98        cot_prompt = self.chain_of_thought_prompt()
 99        if cot_prompt:
100            base_prompt += "\n# Thinking Instructions\n\n" + cot_prompt
101        return base_prompt
102
103
104class SimplePromptBuilder(BasePromptBuilder):
105    """A basic prompt builder that combines task instruction with requirements."""
106
107    def build_base_prompt(self) -> str:
108        """Build a simple prompt with instruction and requirements.
109
110        Returns:
111            str: The constructed prompt string.
112        """
113        base_prompt = self.task.instruction
114
115        # TODO: this is just a quick version. Formatting and best practices TBD
116        if len(self.task.requirements) > 0:
117            base_prompt += (
118                "\n\nYour response should respect the following requirements:\n"
119            )
120            # iterate requirements, formatting them in numbereed list like 1) task.instruction\n2)...
121            for i, requirement in enumerate(self.task.requirements):
122                base_prompt += f"{i + 1}) {requirement.instruction}\n"
123
124        return base_prompt
125
126
127class MultiShotPromptBuilder(BasePromptBuilder):
128    """A prompt builder that includes multiple examples in the prompt."""
129
130    @classmethod
131    def example_count(cls) -> int:
132        """Get the maximum number of examples to include in the prompt.
133
134        Returns:
135            int: The maximum number of examples (default 25).
136        """
137        return 25
138
139    def build_base_prompt(self) -> str:
140        """Build a prompt with instruction, requirements, and multiple examples.
141
142        Returns:
143            str: The constructed prompt string with examples.
144        """
145        base_prompt = f"# Instruction\n\n{self.task.instruction}\n\n"
146
147        if len(self.task.requirements) > 0:
148            base_prompt += "# Requirements\n\nYour response should respect the following requirements:\n"
149            for i, requirement in enumerate(self.task.requirements):
150                base_prompt += f"{i + 1}) {requirement.instruction}\n"
151            base_prompt += "\n"
152
153        valid_examples = self.collect_examples()
154
155        if len(valid_examples) == 0:
156            return base_prompt
157
158        base_prompt += "# Example Outputs\n\n"
159        for i, example in enumerate(valid_examples):
160            base_prompt += self.prompt_section_for_example(i, example)
161
162        return base_prompt
163
164    def prompt_section_for_example(self, index: int, example: TaskRun) -> str:
165        # Prefer repaired output if it exists, otherwise use the regular output
166        output = example.repaired_output or example.output
167        return f"## Example {index + 1}\n\nInput: {example.input}\nOutput: {output.output}\n\n"
168
169    def collect_examples(self) -> list[TaskRun]:
170        valid_examples: list[TaskRun] = []
171        runs = self.task.runs(readonly=True)
172
173        # first pass, we look for repaired outputs. These are the best examples.
174        for run in runs:
175            if len(valid_examples) >= self.__class__.example_count():
176                break
177            if run.repaired_output is not None:
178                valid_examples.append(run)
179
180        # second pass, we look for high quality outputs (rating based)
181        # Minimum is "high_quality" (4 star in star rating scale), then sort by rating
182        # exclude repaired outputs as they were used above
183        runs_with_rating = [
184            run
185            for run in runs
186            if run.output.rating is not None
187            and run.output.rating.value is not None
188            and run.output.rating.is_high_quality()
189            and run.repaired_output is None
190        ]
191        runs_with_rating.sort(
192            key=lambda x: (x.output.rating and x.output.rating.value) or 0, reverse=True
193        )
194        for run in runs_with_rating:
195            if len(valid_examples) >= self.__class__.example_count():
196                break
197            valid_examples.append(run)
198        return valid_examples
199
200
201class FewShotPromptBuilder(MultiShotPromptBuilder):
202    """A prompt builder that includes a small number of examples in the prompt."""
203
204    @classmethod
205    def example_count(cls) -> int:
206        """Get the maximum number of examples to include in the prompt.
207
208        Returns:
209            int: The maximum number of examples (4).
210        """
211        return 4
212
213
214class RepairsPromptBuilder(MultiShotPromptBuilder):
215    """A prompt builder that includes multiple examples in the prompt, including repaired instructions describing what was wrong, and how it was fixed."""
216
217    def prompt_section_for_example(self, index: int, example: TaskRun) -> str:
218        if (
219            not example.repaired_output
220            or not example.repair_instructions
221            or not example.repaired_output.output
222        ):
223            return super().prompt_section_for_example(index, example)
224
225        prompt_section = f"## Example {index + 1}\n\nInput: {example.input}\n\n"
226        prompt_section += (
227            f"Initial Output Which Was Insufficient: {example.output.output}\n\n"
228        )
229        prompt_section += f"Instructions On How to Improve the Initial Output: {example.repair_instructions}\n\n"
230        prompt_section += (
231            f"Repaired Output Which is Sufficient: {example.repaired_output.output}\n\n"
232        )
233        return prompt_section
234
235
236def chain_of_thought_prompt(task: Task) -> str | None:
237    """Standard implementation to build and return the chain of thought prompt string.
238
239    Returns:
240        str: The constructed chain of thought prompt.
241    """
242
243    cot_instruction = task.thinking_instruction
244    if not cot_instruction:
245        cot_instruction = "Think step by step, explaining your reasoning."
246
247    return cot_instruction
248
249
250class SimpleChainOfThoughtPromptBuilder(SimplePromptBuilder):
251    """A prompt builder that includes a chain of thought prompt on top of the simple prompt."""
252
253    def chain_of_thought_prompt(self) -> str | None:
254        return chain_of_thought_prompt(self.task)
255
256
257class FewShotChainOfThoughtPromptBuilder(FewShotPromptBuilder):
258    """A prompt builder that includes a chain of thought prompt on top of the few shot prompt."""
259
260    def chain_of_thought_prompt(self) -> str | None:
261        return chain_of_thought_prompt(self.task)
262
263
264class MultiShotChainOfThoughtPromptBuilder(MultiShotPromptBuilder):
265    """A prompt builder that includes a chain of thought prompt on top of the multi shot prompt."""
266
267    def chain_of_thought_prompt(self) -> str | None:
268        return chain_of_thought_prompt(self.task)
269
270
271class SavedPromptBuilder(BasePromptBuilder):
272    """A prompt builder that looks up a static prompt."""
273
274    def __init__(self, task: Task, prompt_id: str):
275        super().__init__(task)
276        prompt_model = next(
277            (
278                prompt
279                for prompt in task.prompts(readonly=True)
280                if prompt.id == prompt_id
281            ),
282            None,
283        )
284        if not prompt_model:
285            raise ValueError(f"Prompt ID not found: {prompt_id}")
286        self.prompt_model = prompt_model
287
288    def prompt_id(self) -> str | None:
289        return self.prompt_model.id
290
291    def build_base_prompt(self) -> str:
292        """Returns a saved prompt.
293
294        Returns:
295            str: The prompt string.
296        """
297        return self.prompt_model.prompt
298
299    def chain_of_thought_prompt(self) -> str | None:
300        return self.prompt_model.chain_of_thought_instructions
301
302
303# TODO P2: we end up with 2 IDs for these: the keys here (ui_name) and the prompt_builder_name from the class
304# We end up maintaining this in _prompt_generators as well.
305prompt_builder_registry = {
306    "simple_prompt_builder": SimplePromptBuilder,
307    "multi_shot_prompt_builder": MultiShotPromptBuilder,
308    "few_shot_prompt_builder": FewShotPromptBuilder,
309    "repairs_prompt_builder": RepairsPromptBuilder,
310    "simple_chain_of_thought_prompt_builder": SimpleChainOfThoughtPromptBuilder,
311    "few_shot_chain_of_thought_prompt_builder": FewShotChainOfThoughtPromptBuilder,
312    "multi_shot_chain_of_thought_prompt_builder": MultiShotChainOfThoughtPromptBuilder,
313}
314
315
316# Our UI has some names that are not the same as the class names, which also hint parameters.
317def prompt_builder_from_ui_name(ui_name: str, task: Task) -> BasePromptBuilder:
318    """Convert a name used in the UI to the corresponding prompt builder class.
319
320    Args:
321        ui_name (str): The UI name for the prompt builder type.
322
323    Returns:
324        type[BasePromptBuilder]: The corresponding prompt builder class.
325
326    Raises:
327        ValueError: If the UI name is not recognized.
328    """
329
330    # Saved prompts are prefixed with "id::"
331    if ui_name.startswith("id::"):
332        prompt_id = ui_name[4:]
333        return SavedPromptBuilder(task, prompt_id)
334
335    match ui_name:
336        case "basic":
337            return SimplePromptBuilder(task)
338        case "few_shot":
339            return FewShotPromptBuilder(task)
340        case "many_shot":
341            return MultiShotPromptBuilder(task)
342        case "repairs":
343            return RepairsPromptBuilder(task)
344        case "simple_chain_of_thought":
345            return SimpleChainOfThoughtPromptBuilder(task)
346        case "few_shot_chain_of_thought":
347            return FewShotChainOfThoughtPromptBuilder(task)
348        case "multi_shot_chain_of_thought":
349            return MultiShotChainOfThoughtPromptBuilder(task)
350        case _:
351            raise ValueError(f"Unknown prompt builder: {ui_name}")
class BasePromptBuilder:
 10class BasePromptBuilder(metaclass=ABCMeta):
 11    """Base class for building prompts from tasks.
 12
 13    Provides the core interface and basic functionality for prompt builders.
 14    """
 15
 16    def __init__(self, task: Task):
 17        """Initialize the prompt builder with a task.
 18
 19        Args:
 20            task (Task): The task containing instructions and requirements.
 21        """
 22        self.task = task
 23
 24    def prompt_id(self) -> str | None:
 25        """Returns the ID of the prompt, scoped to this builder.
 26
 27        Returns:
 28            str | None: The ID of the prompt, or None if not set.
 29        """
 30        return None
 31
 32    def build_prompt(self, include_json_instructions: bool = False) -> str:
 33        """Build and return the complete prompt string.
 34
 35        Returns:
 36            str: The constructed prompt.
 37        """
 38        prompt = self.build_base_prompt()
 39
 40        if include_json_instructions and self.task.output_schema():
 41            prompt = (
 42                prompt
 43                + f"\n\n# Format Instructions\n\nReturn a JSON object conforming to the following schema:\n```\n{self.task.output_schema()}\n```"
 44            )
 45
 46        return prompt
 47
 48    @abstractmethod
 49    def build_base_prompt(self) -> str:
 50        """Build and return the complete prompt string.
 51
 52        Returns:
 53            str: The constructed prompt.
 54        """
 55        pass
 56
 57    @classmethod
 58    def prompt_builder_name(cls) -> str:
 59        """Returns the name of the prompt builder, to be used for persisting into the datastore.
 60
 61        Default implementation gets the name of the prompt builder in snake case. If you change the class name, you should override this so prior saved data is compatible.
 62
 63        Returns:
 64            str: The prompt builder name in snake_case format.
 65        """
 66        return snake_case(cls.__name__)
 67
 68    def build_user_message(self, input: Dict | str) -> str:
 69        """Build a user message from the input.
 70
 71        Args:
 72            input (Union[Dict, str]): The input to format into a message.
 73
 74        Returns:
 75            str: The formatted user message.
 76        """
 77        if isinstance(input, Dict):
 78            return f"The input is:\n{json.dumps(input, indent=2, ensure_ascii=False)}"
 79
 80        return f"The input is:\n{input}"
 81
 82    def chain_of_thought_prompt(self) -> str | None:
 83        """Build and return the chain of thought prompt string.
 84
 85        Returns:
 86            str: The constructed chain of thought prompt.
 87        """
 88        return None
 89
 90    def build_prompt_for_ui(self) -> str:
 91        """Build a prompt for the UI. It includes additional instructions (like chain of thought), even if they are passed to the model in stages.
 92
 93        Designed for end-user consumption, not for model consumption.
 94
 95        Returns:
 96            str: The constructed prompt string.
 97        """
 98        base_prompt = self.build_prompt()
 99        cot_prompt = self.chain_of_thought_prompt()
100        if cot_prompt:
101            base_prompt += "\n# Thinking Instructions\n\n" + cot_prompt
102        return base_prompt

Base class for building prompts from tasks.

Provides the core interface and basic functionality for prompt builders.

BasePromptBuilder(task: kiln_ai.datamodel.Task)
16    def __init__(self, task: Task):
17        """Initialize the prompt builder with a task.
18
19        Args:
20            task (Task): The task containing instructions and requirements.
21        """
22        self.task = task

Initialize the prompt builder with a task.

Args: task (Task): The task containing instructions and requirements.

task
def prompt_id(self) -> str | None:
24    def prompt_id(self) -> str | None:
25        """Returns the ID of the prompt, scoped to this builder.
26
27        Returns:
28            str | None: The ID of the prompt, or None if not set.
29        """
30        return None

Returns the ID of the prompt, scoped to this builder.

Returns: str | None: The ID of the prompt, or None if not set.

def build_prompt(self, include_json_instructions: bool = False) -> str:
32    def build_prompt(self, include_json_instructions: bool = False) -> str:
33        """Build and return the complete prompt string.
34
35        Returns:
36            str: The constructed prompt.
37        """
38        prompt = self.build_base_prompt()
39
40        if include_json_instructions and self.task.output_schema():
41            prompt = (
42                prompt
43                + f"\n\n# Format Instructions\n\nReturn a JSON object conforming to the following schema:\n```\n{self.task.output_schema()}\n```"
44            )
45
46        return prompt

Build and return the complete prompt string.

Returns: str: The constructed prompt.

@abstractmethod
def build_base_prompt(self) -> str:
48    @abstractmethod
49    def build_base_prompt(self) -> str:
50        """Build and return the complete prompt string.
51
52        Returns:
53            str: The constructed prompt.
54        """
55        pass

Build and return the complete prompt string.

Returns: str: The constructed prompt.

@classmethod
def prompt_builder_name(cls) -> str:
57    @classmethod
58    def prompt_builder_name(cls) -> str:
59        """Returns the name of the prompt builder, to be used for persisting into the datastore.
60
61        Default implementation gets the name of the prompt builder in snake case. If you change the class name, you should override this so prior saved data is compatible.
62
63        Returns:
64            str: The prompt builder name in snake_case format.
65        """
66        return snake_case(cls.__name__)

Returns the name of the prompt builder, to be used for persisting into the datastore.

Default implementation gets the name of the prompt builder in snake case. If you change the class name, you should override this so prior saved data is compatible.

Returns: str: The prompt builder name in snake_case format.

def build_user_message(self, input: Union[Dict, str]) -> str:
68    def build_user_message(self, input: Dict | str) -> str:
69        """Build a user message from the input.
70
71        Args:
72            input (Union[Dict, str]): The input to format into a message.
73
74        Returns:
75            str: The formatted user message.
76        """
77        if isinstance(input, Dict):
78            return f"The input is:\n{json.dumps(input, indent=2, ensure_ascii=False)}"
79
80        return f"The input is:\n{input}"

Build a user message from the input.

Args: input (Union[Dict, str]): The input to format into a message.

Returns: str: The formatted user message.

def chain_of_thought_prompt(self) -> str | None:
82    def chain_of_thought_prompt(self) -> str | None:
83        """Build and return the chain of thought prompt string.
84
85        Returns:
86            str: The constructed chain of thought prompt.
87        """
88        return None

Build and return the chain of thought prompt string.

Returns: str: The constructed chain of thought prompt.

def build_prompt_for_ui(self) -> str:
 90    def build_prompt_for_ui(self) -> str:
 91        """Build a prompt for the UI. It includes additional instructions (like chain of thought), even if they are passed to the model in stages.
 92
 93        Designed for end-user consumption, not for model consumption.
 94
 95        Returns:
 96            str: The constructed prompt string.
 97        """
 98        base_prompt = self.build_prompt()
 99        cot_prompt = self.chain_of_thought_prompt()
100        if cot_prompt:
101            base_prompt += "\n# Thinking Instructions\n\n" + cot_prompt
102        return base_prompt

Build a prompt for the UI. It includes additional instructions (like chain of thought), even if they are passed to the model in stages.

Designed for end-user consumption, not for model consumption.

Returns: str: The constructed prompt string.

class SimplePromptBuilder(BasePromptBuilder):
105class SimplePromptBuilder(BasePromptBuilder):
106    """A basic prompt builder that combines task instruction with requirements."""
107
108    def build_base_prompt(self) -> str:
109        """Build a simple prompt with instruction and requirements.
110
111        Returns:
112            str: The constructed prompt string.
113        """
114        base_prompt = self.task.instruction
115
116        # TODO: this is just a quick version. Formatting and best practices TBD
117        if len(self.task.requirements) > 0:
118            base_prompt += (
119                "\n\nYour response should respect the following requirements:\n"
120            )
121            # iterate requirements, formatting them in numbereed list like 1) task.instruction\n2)...
122            for i, requirement in enumerate(self.task.requirements):
123                base_prompt += f"{i + 1}) {requirement.instruction}\n"
124
125        return base_prompt

A basic prompt builder that combines task instruction with requirements.

def build_base_prompt(self) -> str:
108    def build_base_prompt(self) -> str:
109        """Build a simple prompt with instruction and requirements.
110
111        Returns:
112            str: The constructed prompt string.
113        """
114        base_prompt = self.task.instruction
115
116        # TODO: this is just a quick version. Formatting and best practices TBD
117        if len(self.task.requirements) > 0:
118            base_prompt += (
119                "\n\nYour response should respect the following requirements:\n"
120            )
121            # iterate requirements, formatting them in numbereed list like 1) task.instruction\n2)...
122            for i, requirement in enumerate(self.task.requirements):
123                base_prompt += f"{i + 1}) {requirement.instruction}\n"
124
125        return base_prompt

Build a simple prompt with instruction and requirements.

Returns: str: The constructed prompt string.

class MultiShotPromptBuilder(BasePromptBuilder):
128class MultiShotPromptBuilder(BasePromptBuilder):
129    """A prompt builder that includes multiple examples in the prompt."""
130
131    @classmethod
132    def example_count(cls) -> int:
133        """Get the maximum number of examples to include in the prompt.
134
135        Returns:
136            int: The maximum number of examples (default 25).
137        """
138        return 25
139
140    def build_base_prompt(self) -> str:
141        """Build a prompt with instruction, requirements, and multiple examples.
142
143        Returns:
144            str: The constructed prompt string with examples.
145        """
146        base_prompt = f"# Instruction\n\n{self.task.instruction}\n\n"
147
148        if len(self.task.requirements) > 0:
149            base_prompt += "# Requirements\n\nYour response should respect the following requirements:\n"
150            for i, requirement in enumerate(self.task.requirements):
151                base_prompt += f"{i + 1}) {requirement.instruction}\n"
152            base_prompt += "\n"
153
154        valid_examples = self.collect_examples()
155
156        if len(valid_examples) == 0:
157            return base_prompt
158
159        base_prompt += "# Example Outputs\n\n"
160        for i, example in enumerate(valid_examples):
161            base_prompt += self.prompt_section_for_example(i, example)
162
163        return base_prompt
164
165    def prompt_section_for_example(self, index: int, example: TaskRun) -> str:
166        # Prefer repaired output if it exists, otherwise use the regular output
167        output = example.repaired_output or example.output
168        return f"## Example {index + 1}\n\nInput: {example.input}\nOutput: {output.output}\n\n"
169
170    def collect_examples(self) -> list[TaskRun]:
171        valid_examples: list[TaskRun] = []
172        runs = self.task.runs(readonly=True)
173
174        # first pass, we look for repaired outputs. These are the best examples.
175        for run in runs:
176            if len(valid_examples) >= self.__class__.example_count():
177                break
178            if run.repaired_output is not None:
179                valid_examples.append(run)
180
181        # second pass, we look for high quality outputs (rating based)
182        # Minimum is "high_quality" (4 star in star rating scale), then sort by rating
183        # exclude repaired outputs as they were used above
184        runs_with_rating = [
185            run
186            for run in runs
187            if run.output.rating is not None
188            and run.output.rating.value is not None
189            and run.output.rating.is_high_quality()
190            and run.repaired_output is None
191        ]
192        runs_with_rating.sort(
193            key=lambda x: (x.output.rating and x.output.rating.value) or 0, reverse=True
194        )
195        for run in runs_with_rating:
196            if len(valid_examples) >= self.__class__.example_count():
197                break
198            valid_examples.append(run)
199        return valid_examples

A prompt builder that includes multiple examples in the prompt.

@classmethod
def example_count(cls) -> int:
131    @classmethod
132    def example_count(cls) -> int:
133        """Get the maximum number of examples to include in the prompt.
134
135        Returns:
136            int: The maximum number of examples (default 25).
137        """
138        return 25

Get the maximum number of examples to include in the prompt.

Returns: int: The maximum number of examples (default 25).

def build_base_prompt(self) -> str:
140    def build_base_prompt(self) -> str:
141        """Build a prompt with instruction, requirements, and multiple examples.
142
143        Returns:
144            str: The constructed prompt string with examples.
145        """
146        base_prompt = f"# Instruction\n\n{self.task.instruction}\n\n"
147
148        if len(self.task.requirements) > 0:
149            base_prompt += "# Requirements\n\nYour response should respect the following requirements:\n"
150            for i, requirement in enumerate(self.task.requirements):
151                base_prompt += f"{i + 1}) {requirement.instruction}\n"
152            base_prompt += "\n"
153
154        valid_examples = self.collect_examples()
155
156        if len(valid_examples) == 0:
157            return base_prompt
158
159        base_prompt += "# Example Outputs\n\n"
160        for i, example in enumerate(valid_examples):
161            base_prompt += self.prompt_section_for_example(i, example)
162
163        return base_prompt

Build a prompt with instruction, requirements, and multiple examples.

Returns: str: The constructed prompt string with examples.

def prompt_section_for_example(self, index: int, example: kiln_ai.datamodel.TaskRun) -> str:
165    def prompt_section_for_example(self, index: int, example: TaskRun) -> str:
166        # Prefer repaired output if it exists, otherwise use the regular output
167        output = example.repaired_output or example.output
168        return f"## Example {index + 1}\n\nInput: {example.input}\nOutput: {output.output}\n\n"
def collect_examples(self) -> list[kiln_ai.datamodel.TaskRun]:
170    def collect_examples(self) -> list[TaskRun]:
171        valid_examples: list[TaskRun] = []
172        runs = self.task.runs(readonly=True)
173
174        # first pass, we look for repaired outputs. These are the best examples.
175        for run in runs:
176            if len(valid_examples) >= self.__class__.example_count():
177                break
178            if run.repaired_output is not None:
179                valid_examples.append(run)
180
181        # second pass, we look for high quality outputs (rating based)
182        # Minimum is "high_quality" (4 star in star rating scale), then sort by rating
183        # exclude repaired outputs as they were used above
184        runs_with_rating = [
185            run
186            for run in runs
187            if run.output.rating is not None
188            and run.output.rating.value is not None
189            and run.output.rating.is_high_quality()
190            and run.repaired_output is None
191        ]
192        runs_with_rating.sort(
193            key=lambda x: (x.output.rating and x.output.rating.value) or 0, reverse=True
194        )
195        for run in runs_with_rating:
196            if len(valid_examples) >= self.__class__.example_count():
197                break
198            valid_examples.append(run)
199        return valid_examples
class FewShotPromptBuilder(MultiShotPromptBuilder):
202class FewShotPromptBuilder(MultiShotPromptBuilder):
203    """A prompt builder that includes a small number of examples in the prompt."""
204
205    @classmethod
206    def example_count(cls) -> int:
207        """Get the maximum number of examples to include in the prompt.
208
209        Returns:
210            int: The maximum number of examples (4).
211        """
212        return 4

A prompt builder that includes a small number of examples in the prompt.

@classmethod
def example_count(cls) -> int:
205    @classmethod
206    def example_count(cls) -> int:
207        """Get the maximum number of examples to include in the prompt.
208
209        Returns:
210            int: The maximum number of examples (4).
211        """
212        return 4

Get the maximum number of examples to include in the prompt.

Returns: int: The maximum number of examples (4).

class RepairsPromptBuilder(MultiShotPromptBuilder):
215class RepairsPromptBuilder(MultiShotPromptBuilder):
216    """A prompt builder that includes multiple examples in the prompt, including repaired instructions describing what was wrong, and how it was fixed."""
217
218    def prompt_section_for_example(self, index: int, example: TaskRun) -> str:
219        if (
220            not example.repaired_output
221            or not example.repair_instructions
222            or not example.repaired_output.output
223        ):
224            return super().prompt_section_for_example(index, example)
225
226        prompt_section = f"## Example {index + 1}\n\nInput: {example.input}\n\n"
227        prompt_section += (
228            f"Initial Output Which Was Insufficient: {example.output.output}\n\n"
229        )
230        prompt_section += f"Instructions On How to Improve the Initial Output: {example.repair_instructions}\n\n"
231        prompt_section += (
232            f"Repaired Output Which is Sufficient: {example.repaired_output.output}\n\n"
233        )
234        return prompt_section

A prompt builder that includes multiple examples in the prompt, including repaired instructions describing what was wrong, and how it was fixed.

def prompt_section_for_example(self, index: int, example: kiln_ai.datamodel.TaskRun) -> str:
218    def prompt_section_for_example(self, index: int, example: TaskRun) -> str:
219        if (
220            not example.repaired_output
221            or not example.repair_instructions
222            or not example.repaired_output.output
223        ):
224            return super().prompt_section_for_example(index, example)
225
226        prompt_section = f"## Example {index + 1}\n\nInput: {example.input}\n\n"
227        prompt_section += (
228            f"Initial Output Which Was Insufficient: {example.output.output}\n\n"
229        )
230        prompt_section += f"Instructions On How to Improve the Initial Output: {example.repair_instructions}\n\n"
231        prompt_section += (
232            f"Repaired Output Which is Sufficient: {example.repaired_output.output}\n\n"
233        )
234        return prompt_section
def chain_of_thought_prompt(task: kiln_ai.datamodel.Task) -> str | None:
237def chain_of_thought_prompt(task: Task) -> str | None:
238    """Standard implementation to build and return the chain of thought prompt string.
239
240    Returns:
241        str: The constructed chain of thought prompt.
242    """
243
244    cot_instruction = task.thinking_instruction
245    if not cot_instruction:
246        cot_instruction = "Think step by step, explaining your reasoning."
247
248    return cot_instruction

Standard implementation to build and return the chain of thought prompt string.

Returns: str: The constructed chain of thought prompt.

class SimpleChainOfThoughtPromptBuilder(SimplePromptBuilder):
251class SimpleChainOfThoughtPromptBuilder(SimplePromptBuilder):
252    """A prompt builder that includes a chain of thought prompt on top of the simple prompt."""
253
254    def chain_of_thought_prompt(self) -> str | None:
255        return chain_of_thought_prompt(self.task)

A prompt builder that includes a chain of thought prompt on top of the simple prompt.

def chain_of_thought_prompt(self) -> str | None:
254    def chain_of_thought_prompt(self) -> str | None:
255        return chain_of_thought_prompt(self.task)

Build and return the chain of thought prompt string.

Returns: str: The constructed chain of thought prompt.

class FewShotChainOfThoughtPromptBuilder(FewShotPromptBuilder):
258class FewShotChainOfThoughtPromptBuilder(FewShotPromptBuilder):
259    """A prompt builder that includes a chain of thought prompt on top of the few shot prompt."""
260
261    def chain_of_thought_prompt(self) -> str | None:
262        return chain_of_thought_prompt(self.task)

A prompt builder that includes a chain of thought prompt on top of the few shot prompt.

def chain_of_thought_prompt(self) -> str | None:
261    def chain_of_thought_prompt(self) -> str | None:
262        return chain_of_thought_prompt(self.task)

Build and return the chain of thought prompt string.

Returns: str: The constructed chain of thought prompt.

class MultiShotChainOfThoughtPromptBuilder(MultiShotPromptBuilder):
265class MultiShotChainOfThoughtPromptBuilder(MultiShotPromptBuilder):
266    """A prompt builder that includes a chain of thought prompt on top of the multi shot prompt."""
267
268    def chain_of_thought_prompt(self) -> str | None:
269        return chain_of_thought_prompt(self.task)

A prompt builder that includes a chain of thought prompt on top of the multi shot prompt.

def chain_of_thought_prompt(self) -> str | None:
268    def chain_of_thought_prompt(self) -> str | None:
269        return chain_of_thought_prompt(self.task)

Build and return the chain of thought prompt string.

Returns: str: The constructed chain of thought prompt.

class SavedPromptBuilder(BasePromptBuilder):
272class SavedPromptBuilder(BasePromptBuilder):
273    """A prompt builder that looks up a static prompt."""
274
275    def __init__(self, task: Task, prompt_id: str):
276        super().__init__(task)
277        prompt_model = next(
278            (
279                prompt
280                for prompt in task.prompts(readonly=True)
281                if prompt.id == prompt_id
282            ),
283            None,
284        )
285        if not prompt_model:
286            raise ValueError(f"Prompt ID not found: {prompt_id}")
287        self.prompt_model = prompt_model
288
289    def prompt_id(self) -> str | None:
290        return self.prompt_model.id
291
292    def build_base_prompt(self) -> str:
293        """Returns a saved prompt.
294
295        Returns:
296            str: The prompt string.
297        """
298        return self.prompt_model.prompt
299
300    def chain_of_thought_prompt(self) -> str | None:
301        return self.prompt_model.chain_of_thought_instructions

A prompt builder that looks up a static prompt.

SavedPromptBuilder(task: kiln_ai.datamodel.Task, prompt_id: str)
275    def __init__(self, task: Task, prompt_id: str):
276        super().__init__(task)
277        prompt_model = next(
278            (
279                prompt
280                for prompt in task.prompts(readonly=True)
281                if prompt.id == prompt_id
282            ),
283            None,
284        )
285        if not prompt_model:
286            raise ValueError(f"Prompt ID not found: {prompt_id}")
287        self.prompt_model = prompt_model

Initialize the prompt builder with a task.

Args: task (Task): The task containing instructions and requirements.

prompt_model
def prompt_id(self) -> str | None:
289    def prompt_id(self) -> str | None:
290        return self.prompt_model.id

Returns the ID of the prompt, scoped to this builder.

Returns: str | None: The ID of the prompt, or None if not set.

def build_base_prompt(self) -> str:
292    def build_base_prompt(self) -> str:
293        """Returns a saved prompt.
294
295        Returns:
296            str: The prompt string.
297        """
298        return self.prompt_model.prompt

Returns a saved prompt.

Returns: str: The prompt string.

def chain_of_thought_prompt(self) -> str | None:
300    def chain_of_thought_prompt(self) -> str | None:
301        return self.prompt_model.chain_of_thought_instructions

Build and return the chain of thought prompt string.

Returns: str: The constructed chain of thought prompt.

prompt_builder_registry = {'simple_prompt_builder': <class 'SimplePromptBuilder'>, 'multi_shot_prompt_builder': <class 'MultiShotPromptBuilder'>, 'few_shot_prompt_builder': <class 'FewShotPromptBuilder'>, 'repairs_prompt_builder': <class 'RepairsPromptBuilder'>, 'simple_chain_of_thought_prompt_builder': <class 'SimpleChainOfThoughtPromptBuilder'>, 'few_shot_chain_of_thought_prompt_builder': <class 'FewShotChainOfThoughtPromptBuilder'>, 'multi_shot_chain_of_thought_prompt_builder': <class 'MultiShotChainOfThoughtPromptBuilder'>}
def prompt_builder_from_ui_name( ui_name: str, task: kiln_ai.datamodel.Task) -> BasePromptBuilder:
318def prompt_builder_from_ui_name(ui_name: str, task: Task) -> BasePromptBuilder:
319    """Convert a name used in the UI to the corresponding prompt builder class.
320
321    Args:
322        ui_name (str): The UI name for the prompt builder type.
323
324    Returns:
325        type[BasePromptBuilder]: The corresponding prompt builder class.
326
327    Raises:
328        ValueError: If the UI name is not recognized.
329    """
330
331    # Saved prompts are prefixed with "id::"
332    if ui_name.startswith("id::"):
333        prompt_id = ui_name[4:]
334        return SavedPromptBuilder(task, prompt_id)
335
336    match ui_name:
337        case "basic":
338            return SimplePromptBuilder(task)
339        case "few_shot":
340            return FewShotPromptBuilder(task)
341        case "many_shot":
342            return MultiShotPromptBuilder(task)
343        case "repairs":
344            return RepairsPromptBuilder(task)
345        case "simple_chain_of_thought":
346            return SimpleChainOfThoughtPromptBuilder(task)
347        case "few_shot_chain_of_thought":
348            return FewShotChainOfThoughtPromptBuilder(task)
349        case "multi_shot_chain_of_thought":
350            return MultiShotChainOfThoughtPromptBuilder(task)
351        case _:
352            raise ValueError(f"Unknown prompt builder: {ui_name}")

Convert a name used in the UI to the corresponding prompt builder class.

Args: ui_name (str): The UI name for the prompt builder type.

Returns: type[BasePromptBuilder]: The corresponding prompt builder class.

Raises: ValueError: If the UI name is not recognized.