kiln_ai.datamodel.basemodel
1import json 2import re 3import shutil 4import uuid 5from abc import ABCMeta 6from builtins import classmethod 7from datetime import datetime 8from pathlib import Path 9from typing import ( 10 TYPE_CHECKING, 11 Any, 12 Dict, 13 List, 14 Optional, 15 Type, 16 TypeVar, 17) 18 19from pydantic import ( 20 BaseModel, 21 ConfigDict, 22 Field, 23 ValidationError, 24 computed_field, 25 model_validator, 26) 27from pydantic_core import ErrorDetails 28from typing_extensions import Self 29 30from kiln_ai.utils.config import Config 31from kiln_ai.utils.formatting import snake_case 32 33# ID is a 12 digit random integer string. 34# Should be unique per item, at least inside the context of a parent/child relationship. 35# Use integers to make it easier to type for a search function. 36# Allow none, even though we generate it, because we clear it in the REST API if the object is ephemeral (not persisted to disk) 37ID_FIELD = Field(default_factory=lambda: str(uuid.uuid4().int)[:12]) 38ID_TYPE = Optional[str] 39T = TypeVar("T", bound="KilnBaseModel") 40PT = TypeVar("PT", bound="KilnParentedModel") 41 42# Naming conventions: 43# 1) Names are filename safe as they may be used as file names. They are informational and not to be used in prompts/training/validation. 44# 2) Descrptions are for Kiln users to describe/understanding the purpose of this object. They must never be used in prompts/training/validation. Use "instruction/requirements" instead. 45 46# Filename compatible names 47NAME_REGEX = r"^[A-Za-z0-9 _-]+$" 48NAME_FIELD = Field( 49 min_length=1, 50 max_length=120, 51 pattern=NAME_REGEX, 52 description="A name for this entity.", 53) 54SHORT_NAME_FIELD = Field( 55 min_length=1, 56 max_length=32, 57 pattern=NAME_REGEX, 58 description="A name for this entity", 59) 60 61 62def string_to_valid_name(name: str) -> str: 63 # Replace any character not allowed by NAME_REGEX with an underscore 64 valid_name = re.sub(r"[^A-Za-z0-9 _-]", "_", name) 65 # Replace consecutive underscores with a single underscore 66 valid_name = re.sub(r"_+", "_", valid_name) 67 # Remove leading and trailing underscores or whitespace 68 return valid_name.strip("_").strip() 69 70 71class KilnBaseModel(BaseModel): 72 """Base model for all Kiln data models with common functionality for persistence and versioning. 73 74 Attributes: 75 v (int): Schema version number for migration support 76 id (str): Unique identifier for the model instance 77 path (Path): File system path where the model is stored 78 created_at (datetime): Timestamp when the model was created 79 created_by (str): User ID of the creator 80 """ 81 82 model_config = ConfigDict(validate_assignment=True) 83 84 v: int = Field(default=1) # schema_version 85 id: ID_TYPE = ID_FIELD 86 path: Optional[Path] = Field(default=None) 87 created_at: datetime = Field(default_factory=datetime.now) 88 created_by: str = Field(default_factory=lambda: Config.shared().user_id) 89 90 @computed_field() 91 def model_type(self) -> str: 92 return self.type_name() 93 94 # if changing the model name, should keep the original name here for parsing old files 95 @classmethod 96 def type_name(cls) -> str: 97 return snake_case(cls.__name__) 98 99 # used as /obj_folder/base_filename.kiln 100 @classmethod 101 def base_filename(cls) -> str: 102 return cls.type_name() + ".kiln" 103 104 @classmethod 105 def load_from_folder(cls: Type[T], folderPath: Path) -> T: 106 """Load a model instance from a folder using the default filename. 107 108 Args: 109 folderPath (Path): Directory path containing the model file 110 111 Returns: 112 T: Instance of the model 113 """ 114 path = folderPath / cls.base_filename() 115 return cls.load_from_file(path) 116 117 @classmethod 118 def load_from_file(cls: Type[T], path: Path) -> T: 119 """Load a model instance from a specific file path. 120 121 Args: 122 path (Path): Path to the model file 123 124 Returns: 125 T: Instance of the model 126 127 Raises: 128 ValueError: If the loaded model is not of the expected type or version 129 FileNotFoundError: If the file does not exist 130 """ 131 with open(path, "r") as file: 132 file_data = file.read() 133 # TODO P2 perf: parsing the JSON twice here. 134 # Once for model_type, once for model. Can't call model_validate with parsed json because enum types break; they get strings instead of enums. 135 parsed_json = json.loads(file_data) 136 m = cls.model_validate_json(file_data, strict=True) 137 if not isinstance(m, cls): 138 raise ValueError(f"Loaded model is not of type {cls.__name__}") 139 file_data = None 140 m.path = path 141 if m.v > m.max_schema_version(): 142 raise ValueError( 143 f"Cannot load from file because the schema version is higher than the current version. Upgrade kiln to the latest version. " 144 f"Class: {m.__class__.__name__}, id: {getattr(m, 'id', None)}, path: {path}, " 145 f"version: {m.v}, max version: {m.max_schema_version()}" 146 ) 147 if parsed_json["model_type"] != cls.type_name(): 148 raise ValueError( 149 f"Cannot load from file because the model type is incorrect. Expected {cls.type_name()}, got {parsed_json['model_type']}. " 150 f"Class: {m.__class__.__name__}, id: {getattr(m, 'id', None)}, path: {path}, " 151 f"version: {m.v}, max version: {m.max_schema_version()}" 152 ) 153 return m 154 155 def save_to_file(self) -> None: 156 """Save the model instance to a file. 157 158 Raises: 159 ValueError: If the path is not set 160 """ 161 path = self.build_path() 162 if path is None: 163 raise ValueError( 164 f"Cannot save to file because 'path' is not set. Class: {self.__class__.__name__}, " 165 f"id: {getattr(self, 'id', None)}, path: {path}" 166 ) 167 path.parent.mkdir(parents=True, exist_ok=True) 168 json_data = self.model_dump_json(indent=2, exclude={"path"}) 169 with open(path, "w") as file: 170 file.write(json_data) 171 # save the path so even if something like name changes, the file doesn't move 172 self.path = path 173 174 def delete(self) -> None: 175 if self.path is None: 176 raise ValueError("Cannot delete model because path is not set") 177 dir_path = self.path.parent if self.path.is_file() else self.path 178 if dir_path is None: 179 raise ValueError("Cannot delete model because path is not set") 180 shutil.rmtree(dir_path) 181 self.path = None 182 183 def build_path(self) -> Path | None: 184 if self.path is not None: 185 return self.path 186 return None 187 188 # increment for breaking changes 189 def max_schema_version(self) -> int: 190 return 1 191 192 193class KilnParentedModel(KilnBaseModel, metaclass=ABCMeta): 194 """Base model for Kiln models that have a parent-child relationship. This base class is for child models. 195 196 This class provides functionality for managing hierarchical relationships between models, 197 including parent reference handling and file system organization. 198 199 Attributes: 200 _parent (KilnBaseModel): Reference to the parent model instance 201 """ 202 203 _parent: KilnBaseModel | None = None 204 205 # workaround to tell typechecker that we support the parent property, even though it's not a stock property 206 if TYPE_CHECKING: 207 parent: KilnBaseModel # type: ignore 208 209 def __init__(self, **data): 210 super().__init__(**data) 211 if "parent" in data: 212 self.parent = data["parent"] 213 214 @property 215 def parent(self) -> Optional[KilnBaseModel]: 216 """Get the parent model instance, loading it from disk if necessary. 217 218 Returns: 219 Optional[KilnBaseModel]: The parent model instance or None if not set 220 """ 221 if self._parent is not None: 222 return self._parent 223 # lazy load parent from path 224 if self.path is None: 225 return None 226 # TODO: this only works with base_filename. If we every support custom names, we need to change this. 227 parent_path = ( 228 self.path.parent.parent.parent 229 / self.__class__.parent_type().base_filename() 230 ) 231 if parent_path is None: 232 return None 233 self._parent = self.__class__.parent_type().load_from_file(parent_path) 234 return self._parent 235 236 @parent.setter 237 def parent(self, value: Optional[KilnBaseModel]): 238 if value is not None: 239 expected_parent_type = self.__class__.parent_type() 240 if not isinstance(value, expected_parent_type): 241 raise ValueError( 242 f"Parent must be of type {expected_parent_type}, but was {type(value)}" 243 ) 244 self._parent = value 245 246 # Dynamically implemented by KilnParentModel method injection 247 @classmethod 248 def relationship_name(cls) -> str: 249 raise NotImplementedError("Relationship name must be implemented") 250 251 # Dynamically implemented by KilnParentModel method injection 252 @classmethod 253 def parent_type(cls) -> Type[KilnBaseModel]: 254 raise NotImplementedError("Parent type must be implemented") 255 256 @model_validator(mode="after") 257 def check_parent_type(self) -> Self: 258 if self._parent is not None: 259 expected_parent_type = self.__class__.parent_type() 260 if not isinstance(self._parent, expected_parent_type): 261 raise ValueError( 262 f"Parent must be of type {expected_parent_type}, but was {type(self._parent)}" 263 ) 264 return self 265 266 def build_child_dirname(self) -> Path: 267 # Default implementation for readable folder names. 268 # {id} - {name}/{type}.kiln 269 if self.id is None: 270 # consider generating an ID here. But if it's been cleared, we've already used this without one so raise for now. 271 raise ValueError("ID is not set - can not save or build path") 272 path = self.id 273 name = getattr(self, "name", None) 274 if name is not None: 275 path = f"{path} - {name[:32]}" 276 return Path(path) 277 278 def build_path(self) -> Path | None: 279 # if specifically loaded from an existing path, keep that no matter what 280 # this ensures the file structure is easy to use with git/version control 281 # and that changes to things like name (which impacts default path) don't leave dangling files 282 if self.path is not None: 283 return self.path 284 # Build a path under parent_folder/relationship/file.kiln 285 if self.parent is None: 286 return None 287 parent_path = self.parent.build_path() 288 if parent_path is None: 289 return None 290 parent_folder = parent_path.parent 291 if parent_folder is None: 292 return None 293 return ( 294 parent_folder 295 / self.__class__.relationship_name() 296 / self.build_child_dirname() 297 / self.__class__.base_filename() 298 ) 299 300 @classmethod 301 def all_children_of_parent_path( 302 cls: Type[PT], parent_path: Path | None 303 ) -> list[PT]: 304 if parent_path is None: 305 # children are disk based. If not saved, they don't exist 306 return [] 307 308 # Determine the parent folder 309 if parent_path.is_file(): 310 parent_folder = parent_path.parent 311 else: 312 parent_folder = parent_path 313 314 parent = cls.parent_type().load_from_file(parent_path) 315 if parent is None: 316 raise ValueError("Parent must be set to load children") 317 318 # Ignore type error: this is abstract base class, but children must implement relationship_name 319 relationship_folder = parent_folder / Path(cls.relationship_name()) # type: ignore 320 321 if not relationship_folder.exists() or not relationship_folder.is_dir(): 322 return [] 323 324 # Collect all /relationship/{id}/{base_filename.kiln} files in the relationship folder 325 children = [] 326 for child_file in relationship_folder.glob(f"**/{cls.base_filename()}"): 327 child = cls.load_from_file(child_file) 328 children.append(child) 329 330 return children 331 332 333# Parent create methods for all child relationships 334# You must pass in parent_of in the subclass definition, defining the child relationships 335class KilnParentModel(KilnBaseModel, metaclass=ABCMeta): 336 """Base model for Kiln models that can have child models. 337 338 This class provides functionality for managing collections of child models and their persistence. 339 Child relationships must be defined using the parent_of parameter in the class definition. 340 341 Args: 342 parent_of (Dict[str, Type[KilnParentedModel]]): Mapping of relationship names to child model types 343 """ 344 345 @classmethod 346 def _create_child_method( 347 cls, relationship_name: str, child_class: Type[KilnParentedModel] 348 ): 349 def child_method(self) -> list[child_class]: 350 return child_class.all_children_of_parent_path(self.path) 351 352 child_method.__name__ = relationship_name 353 child_method.__annotations__ = {"return": List[child_class]} 354 setattr(cls, relationship_name, child_method) 355 356 @classmethod 357 def _create_parent_methods( 358 cls, targetCls: Type[KilnParentedModel], relationship_name: str 359 ): 360 def parent_class_method() -> Type[KilnParentModel]: 361 return cls 362 363 parent_class_method.__name__ = "parent_type" 364 parent_class_method.__annotations__ = {"return": Type[KilnParentModel]} 365 setattr(targetCls, "parent_type", parent_class_method) 366 367 def relationship_name_method() -> str: 368 return relationship_name 369 370 relationship_name_method.__name__ = "relationship_name" 371 relationship_name_method.__annotations__ = {"return": str} 372 setattr(targetCls, "relationship_name", relationship_name_method) 373 374 @classmethod 375 def __init_subclass__(cls, parent_of: Dict[str, Type[KilnParentedModel]], **kwargs): 376 super().__init_subclass__(**kwargs) 377 cls._parent_of = parent_of 378 for relationship_name, child_class in parent_of.items(): 379 cls._create_child_method(relationship_name, child_class) 380 cls._create_parent_methods(child_class, relationship_name) 381 382 @classmethod 383 def validate_and_save_with_subrelations( 384 cls, 385 data: Dict[str, Any], 386 path: Path | None = None, 387 parent: KilnBaseModel | None = None, 388 ): 389 """Validate and save a model instance along with all its nested child relationships. 390 391 Args: 392 data (Dict[str, Any]): Model data including child relationships 393 path (Path, optional): Path where the model should be saved 394 parent (KilnBaseModel, optional): Parent model instance for parented models 395 396 Returns: 397 KilnParentModel: The validated and saved model instance 398 399 Raises: 400 ValidationError: If validation fails for the model or any of its children 401 """ 402 # Validate first, then save. Don't want error half way through, and partly persisted 403 # TODO P2: save to tmp dir, then move atomically. But need to merge directories so later. 404 cls._validate_nested(data, save=False, path=path, parent=parent) 405 instance = cls._validate_nested(data, save=True, path=path, parent=parent) 406 return instance 407 408 @classmethod 409 def _validate_nested( 410 cls, 411 data: Dict[str, Any], 412 save: bool = False, 413 parent: KilnBaseModel | None = None, 414 path: Path | None = None, 415 ): 416 # Collect all validation errors so we can report them all at once 417 validation_errors = [] 418 419 try: 420 instance = cls.model_validate(data, strict=True) 421 if path is not None: 422 instance.path = path 423 if parent is not None and isinstance(instance, KilnParentedModel): 424 instance.parent = parent 425 if save: 426 instance.save_to_file() 427 except ValidationError as e: 428 instance = None 429 for suberror in e.errors(): 430 validation_errors.append(suberror) 431 432 for key, value_list in data.items(): 433 if key in cls._parent_of: 434 parent_type = cls._parent_of[key] 435 if not isinstance(value_list, list): 436 raise ValueError( 437 f"Expected a list for {key}, but got {type(value_list)}" 438 ) 439 for value_index, value in enumerate(value_list): 440 try: 441 if issubclass(parent_type, KilnParentModel): 442 kwargs = {"data": value, "save": save} 443 if instance is not None: 444 kwargs["parent"] = instance 445 parent_type._validate_nested(**kwargs) 446 elif issubclass(parent_type, KilnParentedModel): 447 # Root node 448 subinstance = parent_type.model_validate(value, strict=True) 449 if instance is not None: 450 subinstance.parent = instance 451 if save: 452 subinstance.save_to_file() 453 else: 454 raise ValueError( 455 f"Invalid type {parent_type}. Should be KilnBaseModel based." 456 ) 457 except ValidationError as e: 458 for suberror in e.errors(): 459 cls._append_loc(suberror, key, value_index) 460 validation_errors.append(suberror) 461 462 if len(validation_errors) > 0: 463 raise ValidationError.from_exception_data( 464 title=f"Validation failed for {cls.__name__}", 465 line_errors=validation_errors, 466 input_type="json", 467 ) 468 469 return instance 470 471 @classmethod 472 def _append_loc( 473 cls, error: ErrorDetails, current_loc: str, value_index: int | None = None 474 ): 475 orig_loc = error["loc"] if "loc" in error else None 476 new_loc: list[str | int] = [current_loc] 477 if value_index is not None: 478 new_loc.append(value_index) 479 if isinstance(orig_loc, tuple): 480 new_loc.extend(list(orig_loc)) 481 elif isinstance(orig_loc, list): 482 new_loc.extend(orig_loc) 483 error["loc"] = tuple(new_loc)
63def string_to_valid_name(name: str) -> str: 64 # Replace any character not allowed by NAME_REGEX with an underscore 65 valid_name = re.sub(r"[^A-Za-z0-9 _-]", "_", name) 66 # Replace consecutive underscores with a single underscore 67 valid_name = re.sub(r"_+", "_", valid_name) 68 # Remove leading and trailing underscores or whitespace 69 return valid_name.strip("_").strip()
72class KilnBaseModel(BaseModel): 73 """Base model for all Kiln data models with common functionality for persistence and versioning. 74 75 Attributes: 76 v (int): Schema version number for migration support 77 id (str): Unique identifier for the model instance 78 path (Path): File system path where the model is stored 79 created_at (datetime): Timestamp when the model was created 80 created_by (str): User ID of the creator 81 """ 82 83 model_config = ConfigDict(validate_assignment=True) 84 85 v: int = Field(default=1) # schema_version 86 id: ID_TYPE = ID_FIELD 87 path: Optional[Path] = Field(default=None) 88 created_at: datetime = Field(default_factory=datetime.now) 89 created_by: str = Field(default_factory=lambda: Config.shared().user_id) 90 91 @computed_field() 92 def model_type(self) -> str: 93 return self.type_name() 94 95 # if changing the model name, should keep the original name here for parsing old files 96 @classmethod 97 def type_name(cls) -> str: 98 return snake_case(cls.__name__) 99 100 # used as /obj_folder/base_filename.kiln 101 @classmethod 102 def base_filename(cls) -> str: 103 return cls.type_name() + ".kiln" 104 105 @classmethod 106 def load_from_folder(cls: Type[T], folderPath: Path) -> T: 107 """Load a model instance from a folder using the default filename. 108 109 Args: 110 folderPath (Path): Directory path containing the model file 111 112 Returns: 113 T: Instance of the model 114 """ 115 path = folderPath / cls.base_filename() 116 return cls.load_from_file(path) 117 118 @classmethod 119 def load_from_file(cls: Type[T], path: Path) -> T: 120 """Load a model instance from a specific file path. 121 122 Args: 123 path (Path): Path to the model file 124 125 Returns: 126 T: Instance of the model 127 128 Raises: 129 ValueError: If the loaded model is not of the expected type or version 130 FileNotFoundError: If the file does not exist 131 """ 132 with open(path, "r") as file: 133 file_data = file.read() 134 # TODO P2 perf: parsing the JSON twice here. 135 # Once for model_type, once for model. Can't call model_validate with parsed json because enum types break; they get strings instead of enums. 136 parsed_json = json.loads(file_data) 137 m = cls.model_validate_json(file_data, strict=True) 138 if not isinstance(m, cls): 139 raise ValueError(f"Loaded model is not of type {cls.__name__}") 140 file_data = None 141 m.path = path 142 if m.v > m.max_schema_version(): 143 raise ValueError( 144 f"Cannot load from file because the schema version is higher than the current version. Upgrade kiln to the latest version. " 145 f"Class: {m.__class__.__name__}, id: {getattr(m, 'id', None)}, path: {path}, " 146 f"version: {m.v}, max version: {m.max_schema_version()}" 147 ) 148 if parsed_json["model_type"] != cls.type_name(): 149 raise ValueError( 150 f"Cannot load from file because the model type is incorrect. Expected {cls.type_name()}, got {parsed_json['model_type']}. " 151 f"Class: {m.__class__.__name__}, id: {getattr(m, 'id', None)}, path: {path}, " 152 f"version: {m.v}, max version: {m.max_schema_version()}" 153 ) 154 return m 155 156 def save_to_file(self) -> None: 157 """Save the model instance to a file. 158 159 Raises: 160 ValueError: If the path is not set 161 """ 162 path = self.build_path() 163 if path is None: 164 raise ValueError( 165 f"Cannot save to file because 'path' is not set. Class: {self.__class__.__name__}, " 166 f"id: {getattr(self, 'id', None)}, path: {path}" 167 ) 168 path.parent.mkdir(parents=True, exist_ok=True) 169 json_data = self.model_dump_json(indent=2, exclude={"path"}) 170 with open(path, "w") as file: 171 file.write(json_data) 172 # save the path so even if something like name changes, the file doesn't move 173 self.path = path 174 175 def delete(self) -> None: 176 if self.path is None: 177 raise ValueError("Cannot delete model because path is not set") 178 dir_path = self.path.parent if self.path.is_file() else self.path 179 if dir_path is None: 180 raise ValueError("Cannot delete model because path is not set") 181 shutil.rmtree(dir_path) 182 self.path = None 183 184 def build_path(self) -> Path | None: 185 if self.path is not None: 186 return self.path 187 return None 188 189 # increment for breaking changes 190 def max_schema_version(self) -> int: 191 return 1
Base model for all Kiln data models with common functionality for persistence and versioning.
Attributes: v (int): Schema version number for migration support id (str): Unique identifier for the model instance path (Path): File system path where the model is stored created_at (datetime): Timestamp when the model was created created_by (str): User ID of the creator
Configuration for the model, should be a dictionary conforming to [ConfigDict
][pydantic.config.ConfigDict].
105 @classmethod 106 def load_from_folder(cls: Type[T], folderPath: Path) -> T: 107 """Load a model instance from a folder using the default filename. 108 109 Args: 110 folderPath (Path): Directory path containing the model file 111 112 Returns: 113 T: Instance of the model 114 """ 115 path = folderPath / cls.base_filename() 116 return cls.load_from_file(path)
Load a model instance from a folder using the default filename.
Args: folderPath (Path): Directory path containing the model file
Returns: T: Instance of the model
118 @classmethod 119 def load_from_file(cls: Type[T], path: Path) -> T: 120 """Load a model instance from a specific file path. 121 122 Args: 123 path (Path): Path to the model file 124 125 Returns: 126 T: Instance of the model 127 128 Raises: 129 ValueError: If the loaded model is not of the expected type or version 130 FileNotFoundError: If the file does not exist 131 """ 132 with open(path, "r") as file: 133 file_data = file.read() 134 # TODO P2 perf: parsing the JSON twice here. 135 # Once for model_type, once for model. Can't call model_validate with parsed json because enum types break; they get strings instead of enums. 136 parsed_json = json.loads(file_data) 137 m = cls.model_validate_json(file_data, strict=True) 138 if not isinstance(m, cls): 139 raise ValueError(f"Loaded model is not of type {cls.__name__}") 140 file_data = None 141 m.path = path 142 if m.v > m.max_schema_version(): 143 raise ValueError( 144 f"Cannot load from file because the schema version is higher than the current version. Upgrade kiln to the latest version. " 145 f"Class: {m.__class__.__name__}, id: {getattr(m, 'id', None)}, path: {path}, " 146 f"version: {m.v}, max version: {m.max_schema_version()}" 147 ) 148 if parsed_json["model_type"] != cls.type_name(): 149 raise ValueError( 150 f"Cannot load from file because the model type is incorrect. Expected {cls.type_name()}, got {parsed_json['model_type']}. " 151 f"Class: {m.__class__.__name__}, id: {getattr(m, 'id', None)}, path: {path}, " 152 f"version: {m.v}, max version: {m.max_schema_version()}" 153 ) 154 return m
Load a model instance from a specific file path.
Args: path (Path): Path to the model file
Returns: T: Instance of the model
Raises: ValueError: If the loaded model is not of the expected type or version FileNotFoundError: If the file does not exist
156 def save_to_file(self) -> None: 157 """Save the model instance to a file. 158 159 Raises: 160 ValueError: If the path is not set 161 """ 162 path = self.build_path() 163 if path is None: 164 raise ValueError( 165 f"Cannot save to file because 'path' is not set. Class: {self.__class__.__name__}, " 166 f"id: {getattr(self, 'id', None)}, path: {path}" 167 ) 168 path.parent.mkdir(parents=True, exist_ok=True) 169 json_data = self.model_dump_json(indent=2, exclude={"path"}) 170 with open(path, "w") as file: 171 file.write(json_data) 172 # save the path so even if something like name changes, the file doesn't move 173 self.path = path
Save the model instance to a file.
Raises: ValueError: If the path is not set
175 def delete(self) -> None: 176 if self.path is None: 177 raise ValueError("Cannot delete model because path is not set") 178 dir_path = self.path.parent if self.path.is_file() else self.path 179 if dir_path is None: 180 raise ValueError("Cannot delete model because path is not set") 181 shutil.rmtree(dir_path) 182 self.path = None
Metadata about the fields defined on the model,
mapping of field names to [FieldInfo
][pydantic.fields.FieldInfo] objects.
This replaces Model.__fields__
from Pydantic V1.
A dictionary of computed field names and their corresponding ComputedFieldInfo
objects.
194class KilnParentedModel(KilnBaseModel, metaclass=ABCMeta): 195 """Base model for Kiln models that have a parent-child relationship. This base class is for child models. 196 197 This class provides functionality for managing hierarchical relationships between models, 198 including parent reference handling and file system organization. 199 200 Attributes: 201 _parent (KilnBaseModel): Reference to the parent model instance 202 """ 203 204 _parent: KilnBaseModel | None = None 205 206 # workaround to tell typechecker that we support the parent property, even though it's not a stock property 207 if TYPE_CHECKING: 208 parent: KilnBaseModel # type: ignore 209 210 def __init__(self, **data): 211 super().__init__(**data) 212 if "parent" in data: 213 self.parent = data["parent"] 214 215 @property 216 def parent(self) -> Optional[KilnBaseModel]: 217 """Get the parent model instance, loading it from disk if necessary. 218 219 Returns: 220 Optional[KilnBaseModel]: The parent model instance or None if not set 221 """ 222 if self._parent is not None: 223 return self._parent 224 # lazy load parent from path 225 if self.path is None: 226 return None 227 # TODO: this only works with base_filename. If we every support custom names, we need to change this. 228 parent_path = ( 229 self.path.parent.parent.parent 230 / self.__class__.parent_type().base_filename() 231 ) 232 if parent_path is None: 233 return None 234 self._parent = self.__class__.parent_type().load_from_file(parent_path) 235 return self._parent 236 237 @parent.setter 238 def parent(self, value: Optional[KilnBaseModel]): 239 if value is not None: 240 expected_parent_type = self.__class__.parent_type() 241 if not isinstance(value, expected_parent_type): 242 raise ValueError( 243 f"Parent must be of type {expected_parent_type}, but was {type(value)}" 244 ) 245 self._parent = value 246 247 # Dynamically implemented by KilnParentModel method injection 248 @classmethod 249 def relationship_name(cls) -> str: 250 raise NotImplementedError("Relationship name must be implemented") 251 252 # Dynamically implemented by KilnParentModel method injection 253 @classmethod 254 def parent_type(cls) -> Type[KilnBaseModel]: 255 raise NotImplementedError("Parent type must be implemented") 256 257 @model_validator(mode="after") 258 def check_parent_type(self) -> Self: 259 if self._parent is not None: 260 expected_parent_type = self.__class__.parent_type() 261 if not isinstance(self._parent, expected_parent_type): 262 raise ValueError( 263 f"Parent must be of type {expected_parent_type}, but was {type(self._parent)}" 264 ) 265 return self 266 267 def build_child_dirname(self) -> Path: 268 # Default implementation for readable folder names. 269 # {id} - {name}/{type}.kiln 270 if self.id is None: 271 # consider generating an ID here. But if it's been cleared, we've already used this without one so raise for now. 272 raise ValueError("ID is not set - can not save or build path") 273 path = self.id 274 name = getattr(self, "name", None) 275 if name is not None: 276 path = f"{path} - {name[:32]}" 277 return Path(path) 278 279 def build_path(self) -> Path | None: 280 # if specifically loaded from an existing path, keep that no matter what 281 # this ensures the file structure is easy to use with git/version control 282 # and that changes to things like name (which impacts default path) don't leave dangling files 283 if self.path is not None: 284 return self.path 285 # Build a path under parent_folder/relationship/file.kiln 286 if self.parent is None: 287 return None 288 parent_path = self.parent.build_path() 289 if parent_path is None: 290 return None 291 parent_folder = parent_path.parent 292 if parent_folder is None: 293 return None 294 return ( 295 parent_folder 296 / self.__class__.relationship_name() 297 / self.build_child_dirname() 298 / self.__class__.base_filename() 299 ) 300 301 @classmethod 302 def all_children_of_parent_path( 303 cls: Type[PT], parent_path: Path | None 304 ) -> list[PT]: 305 if parent_path is None: 306 # children are disk based. If not saved, they don't exist 307 return [] 308 309 # Determine the parent folder 310 if parent_path.is_file(): 311 parent_folder = parent_path.parent 312 else: 313 parent_folder = parent_path 314 315 parent = cls.parent_type().load_from_file(parent_path) 316 if parent is None: 317 raise ValueError("Parent must be set to load children") 318 319 # Ignore type error: this is abstract base class, but children must implement relationship_name 320 relationship_folder = parent_folder / Path(cls.relationship_name()) # type: ignore 321 322 if not relationship_folder.exists() or not relationship_folder.is_dir(): 323 return [] 324 325 # Collect all /relationship/{id}/{base_filename.kiln} files in the relationship folder 326 children = [] 327 for child_file in relationship_folder.glob(f"**/{cls.base_filename()}"): 328 child = cls.load_from_file(child_file) 329 children.append(child) 330 331 return children
Base model for Kiln models that have a parent-child relationship. This base class is for child models.
This class provides functionality for managing hierarchical relationships between models, including parent reference handling and file system organization.
Attributes: _parent (KilnBaseModel): Reference to the parent model instance
210 def __init__(self, **data): 211 super().__init__(**data) 212 if "parent" in data: 213 self.parent = data["parent"]
Create a new model by parsing and validating input data from keyword arguments.
Raises [ValidationError
][pydantic_core.ValidationError] if the input data cannot be
validated to form a valid model.
self
is explicitly positional-only to allow self
as a field name.
215 @property 216 def parent(self) -> Optional[KilnBaseModel]: 217 """Get the parent model instance, loading it from disk if necessary. 218 219 Returns: 220 Optional[KilnBaseModel]: The parent model instance or None if not set 221 """ 222 if self._parent is not None: 223 return self._parent 224 # lazy load parent from path 225 if self.path is None: 226 return None 227 # TODO: this only works with base_filename. If we every support custom names, we need to change this. 228 parent_path = ( 229 self.path.parent.parent.parent 230 / self.__class__.parent_type().base_filename() 231 ) 232 if parent_path is None: 233 return None 234 self._parent = self.__class__.parent_type().load_from_file(parent_path) 235 return self._parent
Get the parent model instance, loading it from disk if necessary.
Returns: Optional[KilnBaseModel]: The parent model instance or None if not set
257 @model_validator(mode="after") 258 def check_parent_type(self) -> Self: 259 if self._parent is not None: 260 expected_parent_type = self.__class__.parent_type() 261 if not isinstance(self._parent, expected_parent_type): 262 raise ValueError( 263 f"Parent must be of type {expected_parent_type}, but was {type(self._parent)}" 264 ) 265 return self
267 def build_child_dirname(self) -> Path: 268 # Default implementation for readable folder names. 269 # {id} - {name}/{type}.kiln 270 if self.id is None: 271 # consider generating an ID here. But if it's been cleared, we've already used this without one so raise for now. 272 raise ValueError("ID is not set - can not save or build path") 273 path = self.id 274 name = getattr(self, "name", None) 275 if name is not None: 276 path = f"{path} - {name[:32]}" 277 return Path(path)
279 def build_path(self) -> Path | None: 280 # if specifically loaded from an existing path, keep that no matter what 281 # this ensures the file structure is easy to use with git/version control 282 # and that changes to things like name (which impacts default path) don't leave dangling files 283 if self.path is not None: 284 return self.path 285 # Build a path under parent_folder/relationship/file.kiln 286 if self.parent is None: 287 return None 288 parent_path = self.parent.build_path() 289 if parent_path is None: 290 return None 291 parent_folder = parent_path.parent 292 if parent_folder is None: 293 return None 294 return ( 295 parent_folder 296 / self.__class__.relationship_name() 297 / self.build_child_dirname() 298 / self.__class__.base_filename() 299 )
301 @classmethod 302 def all_children_of_parent_path( 303 cls: Type[PT], parent_path: Path | None 304 ) -> list[PT]: 305 if parent_path is None: 306 # children are disk based. If not saved, they don't exist 307 return [] 308 309 # Determine the parent folder 310 if parent_path.is_file(): 311 parent_folder = parent_path.parent 312 else: 313 parent_folder = parent_path 314 315 parent = cls.parent_type().load_from_file(parent_path) 316 if parent is None: 317 raise ValueError("Parent must be set to load children") 318 319 # Ignore type error: this is abstract base class, but children must implement relationship_name 320 relationship_folder = parent_folder / Path(cls.relationship_name()) # type: ignore 321 322 if not relationship_folder.exists() or not relationship_folder.is_dir(): 323 return [] 324 325 # Collect all /relationship/{id}/{base_filename.kiln} files in the relationship folder 326 children = [] 327 for child_file in relationship_folder.glob(f"**/{cls.base_filename()}"): 328 child = cls.load_from_file(child_file) 329 children.append(child) 330 331 return children
Configuration for the model, should be a dictionary conforming to [ConfigDict
][pydantic.config.ConfigDict].
Metadata about the fields defined on the model,
mapping of field names to [FieldInfo
][pydantic.fields.FieldInfo] objects.
This replaces Model.__fields__
from Pydantic V1.
A dictionary of computed field names and their corresponding ComputedFieldInfo
objects.
306def init_private_attributes(self: BaseModel, context: Any, /) -> None: 307 """This function is meant to behave like a BaseModel method to initialise private attributes. 308 309 It takes context as an argument since that's what pydantic-core passes when calling it. 310 311 Args: 312 self: The BaseModel instance. 313 context: The context. 314 """ 315 if getattr(self, '__pydantic_private__', None) is None: 316 pydantic_private = {} 317 for name, private_attr in self.__private_attributes__.items(): 318 default = private_attr.get_default() 319 if default is not PydanticUndefined: 320 pydantic_private[name] = default 321 object_setattr(self, '__pydantic_private__', pydantic_private)
This function is meant to behave like a BaseModel method to initialise private attributes.
It takes context as an argument since that's what pydantic-core passes when calling it.
Args: self: The BaseModel instance. context: The context.
336class KilnParentModel(KilnBaseModel, metaclass=ABCMeta): 337 """Base model for Kiln models that can have child models. 338 339 This class provides functionality for managing collections of child models and their persistence. 340 Child relationships must be defined using the parent_of parameter in the class definition. 341 342 Args: 343 parent_of (Dict[str, Type[KilnParentedModel]]): Mapping of relationship names to child model types 344 """ 345 346 @classmethod 347 def _create_child_method( 348 cls, relationship_name: str, child_class: Type[KilnParentedModel] 349 ): 350 def child_method(self) -> list[child_class]: 351 return child_class.all_children_of_parent_path(self.path) 352 353 child_method.__name__ = relationship_name 354 child_method.__annotations__ = {"return": List[child_class]} 355 setattr(cls, relationship_name, child_method) 356 357 @classmethod 358 def _create_parent_methods( 359 cls, targetCls: Type[KilnParentedModel], relationship_name: str 360 ): 361 def parent_class_method() -> Type[KilnParentModel]: 362 return cls 363 364 parent_class_method.__name__ = "parent_type" 365 parent_class_method.__annotations__ = {"return": Type[KilnParentModel]} 366 setattr(targetCls, "parent_type", parent_class_method) 367 368 def relationship_name_method() -> str: 369 return relationship_name 370 371 relationship_name_method.__name__ = "relationship_name" 372 relationship_name_method.__annotations__ = {"return": str} 373 setattr(targetCls, "relationship_name", relationship_name_method) 374 375 @classmethod 376 def __init_subclass__(cls, parent_of: Dict[str, Type[KilnParentedModel]], **kwargs): 377 super().__init_subclass__(**kwargs) 378 cls._parent_of = parent_of 379 for relationship_name, child_class in parent_of.items(): 380 cls._create_child_method(relationship_name, child_class) 381 cls._create_parent_methods(child_class, relationship_name) 382 383 @classmethod 384 def validate_and_save_with_subrelations( 385 cls, 386 data: Dict[str, Any], 387 path: Path | None = None, 388 parent: KilnBaseModel | None = None, 389 ): 390 """Validate and save a model instance along with all its nested child relationships. 391 392 Args: 393 data (Dict[str, Any]): Model data including child relationships 394 path (Path, optional): Path where the model should be saved 395 parent (KilnBaseModel, optional): Parent model instance for parented models 396 397 Returns: 398 KilnParentModel: The validated and saved model instance 399 400 Raises: 401 ValidationError: If validation fails for the model or any of its children 402 """ 403 # Validate first, then save. Don't want error half way through, and partly persisted 404 # TODO P2: save to tmp dir, then move atomically. But need to merge directories so later. 405 cls._validate_nested(data, save=False, path=path, parent=parent) 406 instance = cls._validate_nested(data, save=True, path=path, parent=parent) 407 return instance 408 409 @classmethod 410 def _validate_nested( 411 cls, 412 data: Dict[str, Any], 413 save: bool = False, 414 parent: KilnBaseModel | None = None, 415 path: Path | None = None, 416 ): 417 # Collect all validation errors so we can report them all at once 418 validation_errors = [] 419 420 try: 421 instance = cls.model_validate(data, strict=True) 422 if path is not None: 423 instance.path = path 424 if parent is not None and isinstance(instance, KilnParentedModel): 425 instance.parent = parent 426 if save: 427 instance.save_to_file() 428 except ValidationError as e: 429 instance = None 430 for suberror in e.errors(): 431 validation_errors.append(suberror) 432 433 for key, value_list in data.items(): 434 if key in cls._parent_of: 435 parent_type = cls._parent_of[key] 436 if not isinstance(value_list, list): 437 raise ValueError( 438 f"Expected a list for {key}, but got {type(value_list)}" 439 ) 440 for value_index, value in enumerate(value_list): 441 try: 442 if issubclass(parent_type, KilnParentModel): 443 kwargs = {"data": value, "save": save} 444 if instance is not None: 445 kwargs["parent"] = instance 446 parent_type._validate_nested(**kwargs) 447 elif issubclass(parent_type, KilnParentedModel): 448 # Root node 449 subinstance = parent_type.model_validate(value, strict=True) 450 if instance is not None: 451 subinstance.parent = instance 452 if save: 453 subinstance.save_to_file() 454 else: 455 raise ValueError( 456 f"Invalid type {parent_type}. Should be KilnBaseModel based." 457 ) 458 except ValidationError as e: 459 for suberror in e.errors(): 460 cls._append_loc(suberror, key, value_index) 461 validation_errors.append(suberror) 462 463 if len(validation_errors) > 0: 464 raise ValidationError.from_exception_data( 465 title=f"Validation failed for {cls.__name__}", 466 line_errors=validation_errors, 467 input_type="json", 468 ) 469 470 return instance 471 472 @classmethod 473 def _append_loc( 474 cls, error: ErrorDetails, current_loc: str, value_index: int | None = None 475 ): 476 orig_loc = error["loc"] if "loc" in error else None 477 new_loc: list[str | int] = [current_loc] 478 if value_index is not None: 479 new_loc.append(value_index) 480 if isinstance(orig_loc, tuple): 481 new_loc.extend(list(orig_loc)) 482 elif isinstance(orig_loc, list): 483 new_loc.extend(orig_loc) 484 error["loc"] = tuple(new_loc)
Base model for Kiln models that can have child models.
This class provides functionality for managing collections of child models and their persistence. Child relationships must be defined using the parent_of parameter in the class definition.
Args: parent_of (Dict[str, Type[KilnParentedModel]]): Mapping of relationship names to child model types
383 @classmethod 384 def validate_and_save_with_subrelations( 385 cls, 386 data: Dict[str, Any], 387 path: Path | None = None, 388 parent: KilnBaseModel | None = None, 389 ): 390 """Validate and save a model instance along with all its nested child relationships. 391 392 Args: 393 data (Dict[str, Any]): Model data including child relationships 394 path (Path, optional): Path where the model should be saved 395 parent (KilnBaseModel, optional): Parent model instance for parented models 396 397 Returns: 398 KilnParentModel: The validated and saved model instance 399 400 Raises: 401 ValidationError: If validation fails for the model or any of its children 402 """ 403 # Validate first, then save. Don't want error half way through, and partly persisted 404 # TODO P2: save to tmp dir, then move atomically. But need to merge directories so later. 405 cls._validate_nested(data, save=False, path=path, parent=parent) 406 instance = cls._validate_nested(data, save=True, path=path, parent=parent) 407 return instance
Validate and save a model instance along with all its nested child relationships.
Args: data (Dict[str, Any]): Model data including child relationships path (Path, optional): Path where the model should be saved parent (KilnBaseModel, optional): Parent model instance for parented models
Returns: KilnParentModel: The validated and saved model instance
Raises: ValidationError: If validation fails for the model or any of its children
Configuration for the model, should be a dictionary conforming to [ConfigDict
][pydantic.config.ConfigDict].
Metadata about the fields defined on the model,
mapping of field names to [FieldInfo
][pydantic.fields.FieldInfo] objects.
This replaces Model.__fields__
from Pydantic V1.
A dictionary of computed field names and their corresponding ComputedFieldInfo
objects.