kiln_ai.adapters.prompt_builders
1import json 2from abc import ABCMeta, abstractmethod 3from typing import Dict 4 5from kiln_ai.datamodel import Task, TaskRun 6from kiln_ai.utils.formatting import snake_case 7 8 9class BasePromptBuilder(metaclass=ABCMeta): 10 """Base class for building prompts from tasks. 11 12 Provides the core interface and basic functionality for prompt builders. 13 """ 14 15 def __init__(self, task: Task): 16 """Initialize the prompt builder with a task. 17 18 Args: 19 task (Task): The task containing instructions and requirements. 20 """ 21 self.task = task 22 23 def prompt_id(self) -> str | None: 24 """Returns the ID of the prompt, scoped to this builder. 25 26 Returns: 27 str | None: The ID of the prompt, or None if not set. 28 """ 29 return None 30 31 def build_prompt(self, include_json_instructions: bool = False) -> str: 32 """Build and return the complete prompt string. 33 34 Returns: 35 str: The constructed prompt. 36 """ 37 prompt = self.build_base_prompt() 38 39 if include_json_instructions and self.task.output_schema(): 40 prompt = ( 41 prompt 42 + f"\n\n# Format Instructions\n\nReturn a JSON object conforming to the following schema:\n```\n{self.task.output_schema()}\n```" 43 ) 44 45 return prompt 46 47 @abstractmethod 48 def build_base_prompt(self) -> str: 49 """Build and return the complete prompt string. 50 51 Returns: 52 str: The constructed prompt. 53 """ 54 pass 55 56 @classmethod 57 def prompt_builder_name(cls) -> str: 58 """Returns the name of the prompt builder, to be used for persisting into the datastore. 59 60 Default implementation gets the name of the prompt builder in snake case. If you change the class name, you should override this so prior saved data is compatible. 61 62 Returns: 63 str: The prompt builder name in snake_case format. 64 """ 65 return snake_case(cls.__name__) 66 67 def build_user_message(self, input: Dict | str) -> str: 68 """Build a user message from the input. 69 70 Args: 71 input (Union[Dict, str]): The input to format into a message. 72 73 Returns: 74 str: The formatted user message. 75 """ 76 if isinstance(input, Dict): 77 return f"The input is:\n{json.dumps(input, indent=2, ensure_ascii=False)}" 78 79 return f"The input is:\n{input}" 80 81 def chain_of_thought_prompt(self) -> str | None: 82 """Build and return the chain of thought prompt string. 83 84 Returns: 85 str: The constructed chain of thought prompt. 86 """ 87 return None 88 89 def build_prompt_for_ui(self) -> str: 90 """Build a prompt for the UI. It includes additional instructions (like chain of thought), even if they are passed to the model in stages. 91 92 Designed for end-user consumption, not for model consumption. 93 94 Returns: 95 str: The constructed prompt string. 96 """ 97 base_prompt = self.build_prompt() 98 cot_prompt = self.chain_of_thought_prompt() 99 if cot_prompt: 100 base_prompt += "\n# Thinking Instructions\n\n" + cot_prompt 101 return base_prompt 102 103 104class SimplePromptBuilder(BasePromptBuilder): 105 """A basic prompt builder that combines task instruction with requirements.""" 106 107 def build_base_prompt(self) -> str: 108 """Build a simple prompt with instruction and requirements. 109 110 Returns: 111 str: The constructed prompt string. 112 """ 113 base_prompt = self.task.instruction 114 115 # TODO: this is just a quick version. Formatting and best practices TBD 116 if len(self.task.requirements) > 0: 117 base_prompt += ( 118 "\n\nYour response should respect the following requirements:\n" 119 ) 120 # iterate requirements, formatting them in numbereed list like 1) task.instruction\n2)... 121 for i, requirement in enumerate(self.task.requirements): 122 base_prompt += f"{i + 1}) {requirement.instruction}\n" 123 124 return base_prompt 125 126 127class MultiShotPromptBuilder(BasePromptBuilder): 128 """A prompt builder that includes multiple examples in the prompt.""" 129 130 @classmethod 131 def example_count(cls) -> int: 132 """Get the maximum number of examples to include in the prompt. 133 134 Returns: 135 int: The maximum number of examples (default 25). 136 """ 137 return 25 138 139 def build_base_prompt(self) -> str: 140 """Build a prompt with instruction, requirements, and multiple examples. 141 142 Returns: 143 str: The constructed prompt string with examples. 144 """ 145 base_prompt = f"# Instruction\n\n{self.task.instruction}\n\n" 146 147 if len(self.task.requirements) > 0: 148 base_prompt += "# Requirements\n\nYour response should respect the following requirements:\n" 149 for i, requirement in enumerate(self.task.requirements): 150 base_prompt += f"{i + 1}) {requirement.instruction}\n" 151 base_prompt += "\n" 152 153 valid_examples = self.collect_examples() 154 155 if len(valid_examples) == 0: 156 return base_prompt 157 158 base_prompt += "# Example Outputs\n\n" 159 for i, example in enumerate(valid_examples): 160 base_prompt += self.prompt_section_for_example(i, example) 161 162 return base_prompt 163 164 def prompt_section_for_example(self, index: int, example: TaskRun) -> str: 165 # Prefer repaired output if it exists, otherwise use the regular output 166 output = example.repaired_output or example.output 167 return f"## Example {index + 1}\n\nInput: {example.input}\nOutput: {output.output}\n\n" 168 169 def collect_examples(self) -> list[TaskRun]: 170 valid_examples: list[TaskRun] = [] 171 runs = self.task.runs(readonly=True) 172 173 # first pass, we look for repaired outputs. These are the best examples. 174 for run in runs: 175 if len(valid_examples) >= self.__class__.example_count(): 176 break 177 if run.repaired_output is not None: 178 valid_examples.append(run) 179 180 # second pass, we look for high quality outputs (rating based) 181 # Minimum is "high_quality" (4 star in star rating scale), then sort by rating 182 # exclude repaired outputs as they were used above 183 runs_with_rating = [ 184 run 185 for run in runs 186 if run.output.rating is not None 187 and run.output.rating.value is not None 188 and run.output.rating.is_high_quality() 189 and run.repaired_output is None 190 ] 191 runs_with_rating.sort( 192 key=lambda x: (x.output.rating and x.output.rating.value) or 0, reverse=True 193 ) 194 for run in runs_with_rating: 195 if len(valid_examples) >= self.__class__.example_count(): 196 break 197 valid_examples.append(run) 198 return valid_examples 199 200 201class FewShotPromptBuilder(MultiShotPromptBuilder): 202 """A prompt builder that includes a small number of examples in the prompt.""" 203 204 @classmethod 205 def example_count(cls) -> int: 206 """Get the maximum number of examples to include in the prompt. 207 208 Returns: 209 int: The maximum number of examples (4). 210 """ 211 return 4 212 213 214class RepairsPromptBuilder(MultiShotPromptBuilder): 215 """A prompt builder that includes multiple examples in the prompt, including repaired instructions describing what was wrong, and how it was fixed.""" 216 217 def prompt_section_for_example(self, index: int, example: TaskRun) -> str: 218 if ( 219 not example.repaired_output 220 or not example.repair_instructions 221 or not example.repaired_output.output 222 ): 223 return super().prompt_section_for_example(index, example) 224 225 prompt_section = f"## Example {index + 1}\n\nInput: {example.input}\n\n" 226 prompt_section += ( 227 f"Initial Output Which Was Insufficient: {example.output.output}\n\n" 228 ) 229 prompt_section += f"Instructions On How to Improve the Initial Output: {example.repair_instructions}\n\n" 230 prompt_section += ( 231 f"Repaired Output Which is Sufficient: {example.repaired_output.output}\n\n" 232 ) 233 return prompt_section 234 235 236def chain_of_thought_prompt(task: Task) -> str | None: 237 """Standard implementation to build and return the chain of thought prompt string. 238 239 Returns: 240 str: The constructed chain of thought prompt. 241 """ 242 243 cot_instruction = task.thinking_instruction 244 if not cot_instruction: 245 cot_instruction = "Think step by step, explaining your reasoning." 246 247 return cot_instruction 248 249 250class SimpleChainOfThoughtPromptBuilder(SimplePromptBuilder): 251 """A prompt builder that includes a chain of thought prompt on top of the simple prompt.""" 252 253 def chain_of_thought_prompt(self) -> str | None: 254 return chain_of_thought_prompt(self.task) 255 256 257class FewShotChainOfThoughtPromptBuilder(FewShotPromptBuilder): 258 """A prompt builder that includes a chain of thought prompt on top of the few shot prompt.""" 259 260 def chain_of_thought_prompt(self) -> str | None: 261 return chain_of_thought_prompt(self.task) 262 263 264class MultiShotChainOfThoughtPromptBuilder(MultiShotPromptBuilder): 265 """A prompt builder that includes a chain of thought prompt on top of the multi shot prompt.""" 266 267 def chain_of_thought_prompt(self) -> str | None: 268 return chain_of_thought_prompt(self.task) 269 270 271class SavedPromptBuilder(BasePromptBuilder): 272 """A prompt builder that looks up a static prompt.""" 273 274 def __init__(self, task: Task, prompt_id: str): 275 super().__init__(task) 276 prompt_model = next( 277 ( 278 prompt 279 for prompt in task.prompts(readonly=True) 280 if prompt.id == prompt_id 281 ), 282 None, 283 ) 284 if not prompt_model: 285 raise ValueError(f"Prompt ID not found: {prompt_id}") 286 self.prompt_model = prompt_model 287 288 def prompt_id(self) -> str | None: 289 return self.prompt_model.id 290 291 def build_base_prompt(self) -> str: 292 """Returns a saved prompt. 293 294 Returns: 295 str: The prompt string. 296 """ 297 return self.prompt_model.prompt 298 299 def chain_of_thought_prompt(self) -> str | None: 300 return self.prompt_model.chain_of_thought_instructions 301 302 303# TODO P2: we end up with 2 IDs for these: the keys here (ui_name) and the prompt_builder_name from the class 304# We end up maintaining this in _prompt_generators as well. 305prompt_builder_registry = { 306 "simple_prompt_builder": SimplePromptBuilder, 307 "multi_shot_prompt_builder": MultiShotPromptBuilder, 308 "few_shot_prompt_builder": FewShotPromptBuilder, 309 "repairs_prompt_builder": RepairsPromptBuilder, 310 "simple_chain_of_thought_prompt_builder": SimpleChainOfThoughtPromptBuilder, 311 "few_shot_chain_of_thought_prompt_builder": FewShotChainOfThoughtPromptBuilder, 312 "multi_shot_chain_of_thought_prompt_builder": MultiShotChainOfThoughtPromptBuilder, 313} 314 315 316# Our UI has some names that are not the same as the class names, which also hint parameters. 317def prompt_builder_from_ui_name(ui_name: str, task: Task) -> BasePromptBuilder: 318 """Convert a name used in the UI to the corresponding prompt builder class. 319 320 Args: 321 ui_name (str): The UI name for the prompt builder type. 322 323 Returns: 324 type[BasePromptBuilder]: The corresponding prompt builder class. 325 326 Raises: 327 ValueError: If the UI name is not recognized. 328 """ 329 330 # Saved prompts are prefixed with "id::" 331 if ui_name.startswith("id::"): 332 prompt_id = ui_name[4:] 333 return SavedPromptBuilder(task, prompt_id) 334 335 match ui_name: 336 case "basic": 337 return SimplePromptBuilder(task) 338 case "few_shot": 339 return FewShotPromptBuilder(task) 340 case "many_shot": 341 return MultiShotPromptBuilder(task) 342 case "repairs": 343 return RepairsPromptBuilder(task) 344 case "simple_chain_of_thought": 345 return SimpleChainOfThoughtPromptBuilder(task) 346 case "few_shot_chain_of_thought": 347 return FewShotChainOfThoughtPromptBuilder(task) 348 case "multi_shot_chain_of_thought": 349 return MultiShotChainOfThoughtPromptBuilder(task) 350 case _: 351 raise ValueError(f"Unknown prompt builder: {ui_name}")
10class BasePromptBuilder(metaclass=ABCMeta): 11 """Base class for building prompts from tasks. 12 13 Provides the core interface and basic functionality for prompt builders. 14 """ 15 16 def __init__(self, task: Task): 17 """Initialize the prompt builder with a task. 18 19 Args: 20 task (Task): The task containing instructions and requirements. 21 """ 22 self.task = task 23 24 def prompt_id(self) -> str | None: 25 """Returns the ID of the prompt, scoped to this builder. 26 27 Returns: 28 str | None: The ID of the prompt, or None if not set. 29 """ 30 return None 31 32 def build_prompt(self, include_json_instructions: bool = False) -> str: 33 """Build and return the complete prompt string. 34 35 Returns: 36 str: The constructed prompt. 37 """ 38 prompt = self.build_base_prompt() 39 40 if include_json_instructions and self.task.output_schema(): 41 prompt = ( 42 prompt 43 + f"\n\n# Format Instructions\n\nReturn a JSON object conforming to the following schema:\n```\n{self.task.output_schema()}\n```" 44 ) 45 46 return prompt 47 48 @abstractmethod 49 def build_base_prompt(self) -> str: 50 """Build and return the complete prompt string. 51 52 Returns: 53 str: The constructed prompt. 54 """ 55 pass 56 57 @classmethod 58 def prompt_builder_name(cls) -> str: 59 """Returns the name of the prompt builder, to be used for persisting into the datastore. 60 61 Default implementation gets the name of the prompt builder in snake case. If you change the class name, you should override this so prior saved data is compatible. 62 63 Returns: 64 str: The prompt builder name in snake_case format. 65 """ 66 return snake_case(cls.__name__) 67 68 def build_user_message(self, input: Dict | str) -> str: 69 """Build a user message from the input. 70 71 Args: 72 input (Union[Dict, str]): The input to format into a message. 73 74 Returns: 75 str: The formatted user message. 76 """ 77 if isinstance(input, Dict): 78 return f"The input is:\n{json.dumps(input, indent=2, ensure_ascii=False)}" 79 80 return f"The input is:\n{input}" 81 82 def chain_of_thought_prompt(self) -> str | None: 83 """Build and return the chain of thought prompt string. 84 85 Returns: 86 str: The constructed chain of thought prompt. 87 """ 88 return None 89 90 def build_prompt_for_ui(self) -> str: 91 """Build a prompt for the UI. It includes additional instructions (like chain of thought), even if they are passed to the model in stages. 92 93 Designed for end-user consumption, not for model consumption. 94 95 Returns: 96 str: The constructed prompt string. 97 """ 98 base_prompt = self.build_prompt() 99 cot_prompt = self.chain_of_thought_prompt() 100 if cot_prompt: 101 base_prompt += "\n# Thinking Instructions\n\n" + cot_prompt 102 return base_prompt
Base class for building prompts from tasks.
Provides the core interface and basic functionality for prompt builders.
16 def __init__(self, task: Task): 17 """Initialize the prompt builder with a task. 18 19 Args: 20 task (Task): The task containing instructions and requirements. 21 """ 22 self.task = task
Initialize the prompt builder with a task.
Args: task (Task): The task containing instructions and requirements.
24 def prompt_id(self) -> str | None: 25 """Returns the ID of the prompt, scoped to this builder. 26 27 Returns: 28 str | None: The ID of the prompt, or None if not set. 29 """ 30 return None
Returns the ID of the prompt, scoped to this builder.
Returns: str | None: The ID of the prompt, or None if not set.
32 def build_prompt(self, include_json_instructions: bool = False) -> str: 33 """Build and return the complete prompt string. 34 35 Returns: 36 str: The constructed prompt. 37 """ 38 prompt = self.build_base_prompt() 39 40 if include_json_instructions and self.task.output_schema(): 41 prompt = ( 42 prompt 43 + f"\n\n# Format Instructions\n\nReturn a JSON object conforming to the following schema:\n```\n{self.task.output_schema()}\n```" 44 ) 45 46 return prompt
Build and return the complete prompt string.
Returns: str: The constructed prompt.
48 @abstractmethod 49 def build_base_prompt(self) -> str: 50 """Build and return the complete prompt string. 51 52 Returns: 53 str: The constructed prompt. 54 """ 55 pass
Build and return the complete prompt string.
Returns: str: The constructed prompt.
57 @classmethod 58 def prompt_builder_name(cls) -> str: 59 """Returns the name of the prompt builder, to be used for persisting into the datastore. 60 61 Default implementation gets the name of the prompt builder in snake case. If you change the class name, you should override this so prior saved data is compatible. 62 63 Returns: 64 str: The prompt builder name in snake_case format. 65 """ 66 return snake_case(cls.__name__)
Returns the name of the prompt builder, to be used for persisting into the datastore.
Default implementation gets the name of the prompt builder in snake case. If you change the class name, you should override this so prior saved data is compatible.
Returns: str: The prompt builder name in snake_case format.
68 def build_user_message(self, input: Dict | str) -> str: 69 """Build a user message from the input. 70 71 Args: 72 input (Union[Dict, str]): The input to format into a message. 73 74 Returns: 75 str: The formatted user message. 76 """ 77 if isinstance(input, Dict): 78 return f"The input is:\n{json.dumps(input, indent=2, ensure_ascii=False)}" 79 80 return f"The input is:\n{input}"
Build a user message from the input.
Args: input (Union[Dict, str]): The input to format into a message.
Returns: str: The formatted user message.
82 def chain_of_thought_prompt(self) -> str | None: 83 """Build and return the chain of thought prompt string. 84 85 Returns: 86 str: The constructed chain of thought prompt. 87 """ 88 return None
Build and return the chain of thought prompt string.
Returns: str: The constructed chain of thought prompt.
90 def build_prompt_for_ui(self) -> str: 91 """Build a prompt for the UI. It includes additional instructions (like chain of thought), even if they are passed to the model in stages. 92 93 Designed for end-user consumption, not for model consumption. 94 95 Returns: 96 str: The constructed prompt string. 97 """ 98 base_prompt = self.build_prompt() 99 cot_prompt = self.chain_of_thought_prompt() 100 if cot_prompt: 101 base_prompt += "\n# Thinking Instructions\n\n" + cot_prompt 102 return base_prompt
Build a prompt for the UI. It includes additional instructions (like chain of thought), even if they are passed to the model in stages.
Designed for end-user consumption, not for model consumption.
Returns: str: The constructed prompt string.
105class SimplePromptBuilder(BasePromptBuilder): 106 """A basic prompt builder that combines task instruction with requirements.""" 107 108 def build_base_prompt(self) -> str: 109 """Build a simple prompt with instruction and requirements. 110 111 Returns: 112 str: The constructed prompt string. 113 """ 114 base_prompt = self.task.instruction 115 116 # TODO: this is just a quick version. Formatting and best practices TBD 117 if len(self.task.requirements) > 0: 118 base_prompt += ( 119 "\n\nYour response should respect the following requirements:\n" 120 ) 121 # iterate requirements, formatting them in numbereed list like 1) task.instruction\n2)... 122 for i, requirement in enumerate(self.task.requirements): 123 base_prompt += f"{i + 1}) {requirement.instruction}\n" 124 125 return base_prompt
A basic prompt builder that combines task instruction with requirements.
108 def build_base_prompt(self) -> str: 109 """Build a simple prompt with instruction and requirements. 110 111 Returns: 112 str: The constructed prompt string. 113 """ 114 base_prompt = self.task.instruction 115 116 # TODO: this is just a quick version. Formatting and best practices TBD 117 if len(self.task.requirements) > 0: 118 base_prompt += ( 119 "\n\nYour response should respect the following requirements:\n" 120 ) 121 # iterate requirements, formatting them in numbereed list like 1) task.instruction\n2)... 122 for i, requirement in enumerate(self.task.requirements): 123 base_prompt += f"{i + 1}) {requirement.instruction}\n" 124 125 return base_prompt
Build a simple prompt with instruction and requirements.
Returns: str: The constructed prompt string.
128class MultiShotPromptBuilder(BasePromptBuilder): 129 """A prompt builder that includes multiple examples in the prompt.""" 130 131 @classmethod 132 def example_count(cls) -> int: 133 """Get the maximum number of examples to include in the prompt. 134 135 Returns: 136 int: The maximum number of examples (default 25). 137 """ 138 return 25 139 140 def build_base_prompt(self) -> str: 141 """Build a prompt with instruction, requirements, and multiple examples. 142 143 Returns: 144 str: The constructed prompt string with examples. 145 """ 146 base_prompt = f"# Instruction\n\n{self.task.instruction}\n\n" 147 148 if len(self.task.requirements) > 0: 149 base_prompt += "# Requirements\n\nYour response should respect the following requirements:\n" 150 for i, requirement in enumerate(self.task.requirements): 151 base_prompt += f"{i + 1}) {requirement.instruction}\n" 152 base_prompt += "\n" 153 154 valid_examples = self.collect_examples() 155 156 if len(valid_examples) == 0: 157 return base_prompt 158 159 base_prompt += "# Example Outputs\n\n" 160 for i, example in enumerate(valid_examples): 161 base_prompt += self.prompt_section_for_example(i, example) 162 163 return base_prompt 164 165 def prompt_section_for_example(self, index: int, example: TaskRun) -> str: 166 # Prefer repaired output if it exists, otherwise use the regular output 167 output = example.repaired_output or example.output 168 return f"## Example {index + 1}\n\nInput: {example.input}\nOutput: {output.output}\n\n" 169 170 def collect_examples(self) -> list[TaskRun]: 171 valid_examples: list[TaskRun] = [] 172 runs = self.task.runs(readonly=True) 173 174 # first pass, we look for repaired outputs. These are the best examples. 175 for run in runs: 176 if len(valid_examples) >= self.__class__.example_count(): 177 break 178 if run.repaired_output is not None: 179 valid_examples.append(run) 180 181 # second pass, we look for high quality outputs (rating based) 182 # Minimum is "high_quality" (4 star in star rating scale), then sort by rating 183 # exclude repaired outputs as they were used above 184 runs_with_rating = [ 185 run 186 for run in runs 187 if run.output.rating is not None 188 and run.output.rating.value is not None 189 and run.output.rating.is_high_quality() 190 and run.repaired_output is None 191 ] 192 runs_with_rating.sort( 193 key=lambda x: (x.output.rating and x.output.rating.value) or 0, reverse=True 194 ) 195 for run in runs_with_rating: 196 if len(valid_examples) >= self.__class__.example_count(): 197 break 198 valid_examples.append(run) 199 return valid_examples
A prompt builder that includes multiple examples in the prompt.
131 @classmethod 132 def example_count(cls) -> int: 133 """Get the maximum number of examples to include in the prompt. 134 135 Returns: 136 int: The maximum number of examples (default 25). 137 """ 138 return 25
Get the maximum number of examples to include in the prompt.
Returns: int: The maximum number of examples (default 25).
140 def build_base_prompt(self) -> str: 141 """Build a prompt with instruction, requirements, and multiple examples. 142 143 Returns: 144 str: The constructed prompt string with examples. 145 """ 146 base_prompt = f"# Instruction\n\n{self.task.instruction}\n\n" 147 148 if len(self.task.requirements) > 0: 149 base_prompt += "# Requirements\n\nYour response should respect the following requirements:\n" 150 for i, requirement in enumerate(self.task.requirements): 151 base_prompt += f"{i + 1}) {requirement.instruction}\n" 152 base_prompt += "\n" 153 154 valid_examples = self.collect_examples() 155 156 if len(valid_examples) == 0: 157 return base_prompt 158 159 base_prompt += "# Example Outputs\n\n" 160 for i, example in enumerate(valid_examples): 161 base_prompt += self.prompt_section_for_example(i, example) 162 163 return base_prompt
Build a prompt with instruction, requirements, and multiple examples.
Returns: str: The constructed prompt string with examples.
165 def prompt_section_for_example(self, index: int, example: TaskRun) -> str: 166 # Prefer repaired output if it exists, otherwise use the regular output 167 output = example.repaired_output or example.output 168 return f"## Example {index + 1}\n\nInput: {example.input}\nOutput: {output.output}\n\n"
170 def collect_examples(self) -> list[TaskRun]: 171 valid_examples: list[TaskRun] = [] 172 runs = self.task.runs(readonly=True) 173 174 # first pass, we look for repaired outputs. These are the best examples. 175 for run in runs: 176 if len(valid_examples) >= self.__class__.example_count(): 177 break 178 if run.repaired_output is not None: 179 valid_examples.append(run) 180 181 # second pass, we look for high quality outputs (rating based) 182 # Minimum is "high_quality" (4 star in star rating scale), then sort by rating 183 # exclude repaired outputs as they were used above 184 runs_with_rating = [ 185 run 186 for run in runs 187 if run.output.rating is not None 188 and run.output.rating.value is not None 189 and run.output.rating.is_high_quality() 190 and run.repaired_output is None 191 ] 192 runs_with_rating.sort( 193 key=lambda x: (x.output.rating and x.output.rating.value) or 0, reverse=True 194 ) 195 for run in runs_with_rating: 196 if len(valid_examples) >= self.__class__.example_count(): 197 break 198 valid_examples.append(run) 199 return valid_examples
202class FewShotPromptBuilder(MultiShotPromptBuilder): 203 """A prompt builder that includes a small number of examples in the prompt.""" 204 205 @classmethod 206 def example_count(cls) -> int: 207 """Get the maximum number of examples to include in the prompt. 208 209 Returns: 210 int: The maximum number of examples (4). 211 """ 212 return 4
A prompt builder that includes a small number of examples in the prompt.
205 @classmethod 206 def example_count(cls) -> int: 207 """Get the maximum number of examples to include in the prompt. 208 209 Returns: 210 int: The maximum number of examples (4). 211 """ 212 return 4
Get the maximum number of examples to include in the prompt.
Returns: int: The maximum number of examples (4).
215class RepairsPromptBuilder(MultiShotPromptBuilder): 216 """A prompt builder that includes multiple examples in the prompt, including repaired instructions describing what was wrong, and how it was fixed.""" 217 218 def prompt_section_for_example(self, index: int, example: TaskRun) -> str: 219 if ( 220 not example.repaired_output 221 or not example.repair_instructions 222 or not example.repaired_output.output 223 ): 224 return super().prompt_section_for_example(index, example) 225 226 prompt_section = f"## Example {index + 1}\n\nInput: {example.input}\n\n" 227 prompt_section += ( 228 f"Initial Output Which Was Insufficient: {example.output.output}\n\n" 229 ) 230 prompt_section += f"Instructions On How to Improve the Initial Output: {example.repair_instructions}\n\n" 231 prompt_section += ( 232 f"Repaired Output Which is Sufficient: {example.repaired_output.output}\n\n" 233 ) 234 return prompt_section
A prompt builder that includes multiple examples in the prompt, including repaired instructions describing what was wrong, and how it was fixed.
218 def prompt_section_for_example(self, index: int, example: TaskRun) -> str: 219 if ( 220 not example.repaired_output 221 or not example.repair_instructions 222 or not example.repaired_output.output 223 ): 224 return super().prompt_section_for_example(index, example) 225 226 prompt_section = f"## Example {index + 1}\n\nInput: {example.input}\n\n" 227 prompt_section += ( 228 f"Initial Output Which Was Insufficient: {example.output.output}\n\n" 229 ) 230 prompt_section += f"Instructions On How to Improve the Initial Output: {example.repair_instructions}\n\n" 231 prompt_section += ( 232 f"Repaired Output Which is Sufficient: {example.repaired_output.output}\n\n" 233 ) 234 return prompt_section
237def chain_of_thought_prompt(task: Task) -> str | None: 238 """Standard implementation to build and return the chain of thought prompt string. 239 240 Returns: 241 str: The constructed chain of thought prompt. 242 """ 243 244 cot_instruction = task.thinking_instruction 245 if not cot_instruction: 246 cot_instruction = "Think step by step, explaining your reasoning." 247 248 return cot_instruction
Standard implementation to build and return the chain of thought prompt string.
Returns: str: The constructed chain of thought prompt.
251class SimpleChainOfThoughtPromptBuilder(SimplePromptBuilder): 252 """A prompt builder that includes a chain of thought prompt on top of the simple prompt.""" 253 254 def chain_of_thought_prompt(self) -> str | None: 255 return chain_of_thought_prompt(self.task)
A prompt builder that includes a chain of thought prompt on top of the simple prompt.
258class FewShotChainOfThoughtPromptBuilder(FewShotPromptBuilder): 259 """A prompt builder that includes a chain of thought prompt on top of the few shot prompt.""" 260 261 def chain_of_thought_prompt(self) -> str | None: 262 return chain_of_thought_prompt(self.task)
A prompt builder that includes a chain of thought prompt on top of the few shot prompt.
265class MultiShotChainOfThoughtPromptBuilder(MultiShotPromptBuilder): 266 """A prompt builder that includes a chain of thought prompt on top of the multi shot prompt.""" 267 268 def chain_of_thought_prompt(self) -> str | None: 269 return chain_of_thought_prompt(self.task)
A prompt builder that includes a chain of thought prompt on top of the multi shot prompt.
272class SavedPromptBuilder(BasePromptBuilder): 273 """A prompt builder that looks up a static prompt.""" 274 275 def __init__(self, task: Task, prompt_id: str): 276 super().__init__(task) 277 prompt_model = next( 278 ( 279 prompt 280 for prompt in task.prompts(readonly=True) 281 if prompt.id == prompt_id 282 ), 283 None, 284 ) 285 if not prompt_model: 286 raise ValueError(f"Prompt ID not found: {prompt_id}") 287 self.prompt_model = prompt_model 288 289 def prompt_id(self) -> str | None: 290 return self.prompt_model.id 291 292 def build_base_prompt(self) -> str: 293 """Returns a saved prompt. 294 295 Returns: 296 str: The prompt string. 297 """ 298 return self.prompt_model.prompt 299 300 def chain_of_thought_prompt(self) -> str | None: 301 return self.prompt_model.chain_of_thought_instructions
A prompt builder that looks up a static prompt.
275 def __init__(self, task: Task, prompt_id: str): 276 super().__init__(task) 277 prompt_model = next( 278 ( 279 prompt 280 for prompt in task.prompts(readonly=True) 281 if prompt.id == prompt_id 282 ), 283 None, 284 ) 285 if not prompt_model: 286 raise ValueError(f"Prompt ID not found: {prompt_id}") 287 self.prompt_model = prompt_model
Initialize the prompt builder with a task.
Args: task (Task): The task containing instructions and requirements.
Returns the ID of the prompt, scoped to this builder.
Returns: str | None: The ID of the prompt, or None if not set.
292 def build_base_prompt(self) -> str: 293 """Returns a saved prompt. 294 295 Returns: 296 str: The prompt string. 297 """ 298 return self.prompt_model.prompt
Returns a saved prompt.
Returns: str: The prompt string.
300 def chain_of_thought_prompt(self) -> str | None: 301 return self.prompt_model.chain_of_thought_instructions
Build and return the chain of thought prompt string.
Returns: str: The constructed chain of thought prompt.
Inherited Members
318def prompt_builder_from_ui_name(ui_name: str, task: Task) -> BasePromptBuilder: 319 """Convert a name used in the UI to the corresponding prompt builder class. 320 321 Args: 322 ui_name (str): The UI name for the prompt builder type. 323 324 Returns: 325 type[BasePromptBuilder]: The corresponding prompt builder class. 326 327 Raises: 328 ValueError: If the UI name is not recognized. 329 """ 330 331 # Saved prompts are prefixed with "id::" 332 if ui_name.startswith("id::"): 333 prompt_id = ui_name[4:] 334 return SavedPromptBuilder(task, prompt_id) 335 336 match ui_name: 337 case "basic": 338 return SimplePromptBuilder(task) 339 case "few_shot": 340 return FewShotPromptBuilder(task) 341 case "many_shot": 342 return MultiShotPromptBuilder(task) 343 case "repairs": 344 return RepairsPromptBuilder(task) 345 case "simple_chain_of_thought": 346 return SimpleChainOfThoughtPromptBuilder(task) 347 case "few_shot_chain_of_thought": 348 return FewShotChainOfThoughtPromptBuilder(task) 349 case "multi_shot_chain_of_thought": 350 return MultiShotChainOfThoughtPromptBuilder(task) 351 case _: 352 raise ValueError(f"Unknown prompt builder: {ui_name}")
Convert a name used in the UI to the corresponding prompt builder class.
Args: ui_name (str): The UI name for the prompt builder type.
Returns: type[BasePromptBuilder]: The corresponding prompt builder class.
Raises: ValueError: If the UI name is not recognized.