kiln_ai.datamodel.basemodel

  1import json
  2import re
  3import shutil
  4import uuid
  5from abc import ABCMeta
  6from builtins import classmethod
  7from datetime import datetime
  8from pathlib import Path
  9from typing import (
 10    TYPE_CHECKING,
 11    Any,
 12    Dict,
 13    List,
 14    Optional,
 15    Type,
 16    TypeVar,
 17)
 18
 19from pydantic import (
 20    BaseModel,
 21    ConfigDict,
 22    Field,
 23    ValidationError,
 24    computed_field,
 25    model_validator,
 26)
 27from pydantic_core import ErrorDetails
 28from typing_extensions import Self
 29
 30from kiln_ai.utils.config import Config
 31from kiln_ai.utils.formatting import snake_case
 32
 33# ID is a 12 digit random integer string.
 34# Should be unique per item, at least inside the context of a parent/child relationship.
 35# Use integers to make it easier to type for a search function.
 36# Allow none, even though we generate it, because we clear it in the REST API if the object is ephemeral (not persisted to disk)
 37ID_FIELD = Field(default_factory=lambda: str(uuid.uuid4().int)[:12])
 38ID_TYPE = Optional[str]
 39T = TypeVar("T", bound="KilnBaseModel")
 40PT = TypeVar("PT", bound="KilnParentedModel")
 41
 42# Naming conventions:
 43# 1) Names are filename safe as they may be used as file names. They are informational and not to be used in prompts/training/validation.
 44# 2) Descrptions are for Kiln users to describe/understanding the purpose of this object. They must never be used in prompts/training/validation. Use "instruction/requirements" instead.
 45
 46# Filename compatible names
 47NAME_REGEX = r"^[A-Za-z0-9 _-]+$"
 48NAME_FIELD = Field(
 49    min_length=1,
 50    max_length=120,
 51    pattern=NAME_REGEX,
 52    description="A name for this entity.",
 53)
 54SHORT_NAME_FIELD = Field(
 55    min_length=1,
 56    max_length=32,
 57    pattern=NAME_REGEX,
 58    description="A name for this entity",
 59)
 60
 61
 62def string_to_valid_name(name: str) -> str:
 63    # Replace any character not allowed by NAME_REGEX with an underscore
 64    valid_name = re.sub(r"[^A-Za-z0-9 _-]", "_", name)
 65    # Replace consecutive underscores with a single underscore
 66    valid_name = re.sub(r"_+", "_", valid_name)
 67    # Remove leading and trailing underscores or whitespace
 68    return valid_name.strip("_").strip()
 69
 70
 71class KilnBaseModel(BaseModel):
 72    """Base model for all Kiln data models with common functionality for persistence and versioning.
 73
 74    Attributes:
 75        v (int): Schema version number for migration support
 76        id (str): Unique identifier for the model instance
 77        path (Path): File system path where the model is stored
 78        created_at (datetime): Timestamp when the model was created
 79        created_by (str): User ID of the creator
 80    """
 81
 82    model_config = ConfigDict(validate_assignment=True)
 83
 84    v: int = Field(default=1)  # schema_version
 85    id: ID_TYPE = ID_FIELD
 86    path: Optional[Path] = Field(default=None)
 87    created_at: datetime = Field(default_factory=datetime.now)
 88    created_by: str = Field(default_factory=lambda: Config.shared().user_id)
 89
 90    @computed_field()
 91    def model_type(self) -> str:
 92        return self.type_name()
 93
 94    # if changing the model name, should keep the original name here for parsing old files
 95    @classmethod
 96    def type_name(cls) -> str:
 97        return snake_case(cls.__name__)
 98
 99    # used as /obj_folder/base_filename.kiln
100    @classmethod
101    def base_filename(cls) -> str:
102        return cls.type_name() + ".kiln"
103
104    @classmethod
105    def load_from_folder(cls: Type[T], folderPath: Path) -> T:
106        """Load a model instance from a folder using the default filename.
107
108        Args:
109            folderPath (Path): Directory path containing the model file
110
111        Returns:
112            T: Instance of the model
113        """
114        path = folderPath / cls.base_filename()
115        return cls.load_from_file(path)
116
117    @classmethod
118    def load_from_file(cls: Type[T], path: Path) -> T:
119        """Load a model instance from a specific file path.
120
121        Args:
122            path (Path): Path to the model file
123
124        Returns:
125            T: Instance of the model
126
127        Raises:
128            ValueError: If the loaded model is not of the expected type or version
129            FileNotFoundError: If the file does not exist
130        """
131        with open(path, "r") as file:
132            file_data = file.read()
133            # TODO P2 perf: parsing the JSON twice here.
134            # Once for model_type, once for model. Can't call model_validate with parsed json because enum types break; they get strings instead of enums.
135            parsed_json = json.loads(file_data)
136            m = cls.model_validate_json(file_data, strict=True)
137            if not isinstance(m, cls):
138                raise ValueError(f"Loaded model is not of type {cls.__name__}")
139            file_data = None
140        m.path = path
141        if m.v > m.max_schema_version():
142            raise ValueError(
143                f"Cannot load from file because the schema version is higher than the current version. Upgrade kiln to the latest version. "
144                f"Class: {m.__class__.__name__}, id: {getattr(m, 'id', None)}, path: {path}, "
145                f"version: {m.v}, max version: {m.max_schema_version()}"
146            )
147        if parsed_json["model_type"] != cls.type_name():
148            raise ValueError(
149                f"Cannot load from file because the model type is incorrect. Expected {cls.type_name()}, got {parsed_json['model_type']}. "
150                f"Class: {m.__class__.__name__}, id: {getattr(m, 'id', None)}, path: {path}, "
151                f"version: {m.v}, max version: {m.max_schema_version()}"
152            )
153        return m
154
155    def save_to_file(self) -> None:
156        """Save the model instance to a file.
157
158        Raises:
159            ValueError: If the path is not set
160        """
161        path = self.build_path()
162        if path is None:
163            raise ValueError(
164                f"Cannot save to file because 'path' is not set. Class: {self.__class__.__name__}, "
165                f"id: {getattr(self, 'id', None)}, path: {path}"
166            )
167        path.parent.mkdir(parents=True, exist_ok=True)
168        json_data = self.model_dump_json(indent=2, exclude={"path"})
169        with open(path, "w") as file:
170            file.write(json_data)
171        # save the path so even if something like name changes, the file doesn't move
172        self.path = path
173
174    def delete(self) -> None:
175        if self.path is None:
176            raise ValueError("Cannot delete model because path is not set")
177        dir_path = self.path.parent if self.path.is_file() else self.path
178        if dir_path is None:
179            raise ValueError("Cannot delete model because path is not set")
180        shutil.rmtree(dir_path)
181        self.path = None
182
183    def build_path(self) -> Path | None:
184        if self.path is not None:
185            return self.path
186        return None
187
188    # increment for breaking changes
189    def max_schema_version(self) -> int:
190        return 1
191
192
193class KilnParentedModel(KilnBaseModel, metaclass=ABCMeta):
194    """Base model for Kiln models that have a parent-child relationship. This base class is for child models.
195
196    This class provides functionality for managing hierarchical relationships between models,
197    including parent reference handling and file system organization.
198
199    Attributes:
200        _parent (KilnBaseModel): Reference to the parent model instance
201    """
202
203    _parent: KilnBaseModel | None = None
204
205    # workaround to tell typechecker that we support the parent property, even though it's not a stock property
206    if TYPE_CHECKING:
207        parent: KilnBaseModel  # type: ignore
208
209    def __init__(self, **data):
210        super().__init__(**data)
211        if "parent" in data:
212            self.parent = data["parent"]
213
214    @property
215    def parent(self) -> Optional[KilnBaseModel]:
216        """Get the parent model instance, loading it from disk if necessary.
217
218        Returns:
219            Optional[KilnBaseModel]: The parent model instance or None if not set
220        """
221        if self._parent is not None:
222            return self._parent
223        # lazy load parent from path
224        if self.path is None:
225            return None
226        # TODO: this only works with base_filename. If we every support custom names, we need to change this.
227        parent_path = (
228            self.path.parent.parent.parent
229            / self.__class__.parent_type().base_filename()
230        )
231        if parent_path is None:
232            return None
233        self._parent = self.__class__.parent_type().load_from_file(parent_path)
234        return self._parent
235
236    @parent.setter
237    def parent(self, value: Optional[KilnBaseModel]):
238        if value is not None:
239            expected_parent_type = self.__class__.parent_type()
240            if not isinstance(value, expected_parent_type):
241                raise ValueError(
242                    f"Parent must be of type {expected_parent_type}, but was {type(value)}"
243                )
244        self._parent = value
245
246    # Dynamically implemented by KilnParentModel method injection
247    @classmethod
248    def relationship_name(cls) -> str:
249        raise NotImplementedError("Relationship name must be implemented")
250
251    # Dynamically implemented by KilnParentModel method injection
252    @classmethod
253    def parent_type(cls) -> Type[KilnBaseModel]:
254        raise NotImplementedError("Parent type must be implemented")
255
256    @model_validator(mode="after")
257    def check_parent_type(self) -> Self:
258        if self._parent is not None:
259            expected_parent_type = self.__class__.parent_type()
260            if not isinstance(self._parent, expected_parent_type):
261                raise ValueError(
262                    f"Parent must be of type {expected_parent_type}, but was {type(self._parent)}"
263                )
264        return self
265
266    def build_child_dirname(self) -> Path:
267        # Default implementation for readable folder names.
268        # {id} - {name}/{type}.kiln
269        if self.id is None:
270            # consider generating an ID here. But if it's been cleared, we've already used this without one so raise for now.
271            raise ValueError("ID is not set - can not save or build path")
272        path = self.id
273        name = getattr(self, "name", None)
274        if name is not None:
275            path = f"{path} - {name[:32]}"
276        return Path(path)
277
278    def build_path(self) -> Path | None:
279        # if specifically loaded from an existing path, keep that no matter what
280        # this ensures the file structure is easy to use with git/version control
281        # and that changes to things like name (which impacts default path) don't leave dangling files
282        if self.path is not None:
283            return self.path
284        # Build a path under parent_folder/relationship/file.kiln
285        if self.parent is None:
286            return None
287        parent_path = self.parent.build_path()
288        if parent_path is None:
289            return None
290        parent_folder = parent_path.parent
291        if parent_folder is None:
292            return None
293        return (
294            parent_folder
295            / self.__class__.relationship_name()
296            / self.build_child_dirname()
297            / self.__class__.base_filename()
298        )
299
300    @classmethod
301    def all_children_of_parent_path(
302        cls: Type[PT], parent_path: Path | None
303    ) -> list[PT]:
304        if parent_path is None:
305            # children are disk based. If not saved, they don't exist
306            return []
307
308        # Determine the parent folder
309        if parent_path.is_file():
310            parent_folder = parent_path.parent
311        else:
312            parent_folder = parent_path
313
314        parent = cls.parent_type().load_from_file(parent_path)
315        if parent is None:
316            raise ValueError("Parent must be set to load children")
317
318        # Ignore type error: this is abstract base class, but children must implement relationship_name
319        relationship_folder = parent_folder / Path(cls.relationship_name())  # type: ignore
320
321        if not relationship_folder.exists() or not relationship_folder.is_dir():
322            return []
323
324        # Collect all /relationship/{id}/{base_filename.kiln} files in the relationship folder
325        children = []
326        for child_file in relationship_folder.glob(f"**/{cls.base_filename()}"):
327            child = cls.load_from_file(child_file)
328            children.append(child)
329
330        return children
331
332
333# Parent create methods for all child relationships
334# You must pass in parent_of in the subclass definition, defining the child relationships
335class KilnParentModel(KilnBaseModel, metaclass=ABCMeta):
336    """Base model for Kiln models that can have child models.
337
338    This class provides functionality for managing collections of child models and their persistence.
339    Child relationships must be defined using the parent_of parameter in the class definition.
340
341    Args:
342        parent_of (Dict[str, Type[KilnParentedModel]]): Mapping of relationship names to child model types
343    """
344
345    @classmethod
346    def _create_child_method(
347        cls, relationship_name: str, child_class: Type[KilnParentedModel]
348    ):
349        def child_method(self) -> list[child_class]:
350            return child_class.all_children_of_parent_path(self.path)
351
352        child_method.__name__ = relationship_name
353        child_method.__annotations__ = {"return": List[child_class]}
354        setattr(cls, relationship_name, child_method)
355
356    @classmethod
357    def _create_parent_methods(
358        cls, targetCls: Type[KilnParentedModel], relationship_name: str
359    ):
360        def parent_class_method() -> Type[KilnParentModel]:
361            return cls
362
363        parent_class_method.__name__ = "parent_type"
364        parent_class_method.__annotations__ = {"return": Type[KilnParentModel]}
365        setattr(targetCls, "parent_type", parent_class_method)
366
367        def relationship_name_method() -> str:
368            return relationship_name
369
370        relationship_name_method.__name__ = "relationship_name"
371        relationship_name_method.__annotations__ = {"return": str}
372        setattr(targetCls, "relationship_name", relationship_name_method)
373
374    @classmethod
375    def __init_subclass__(cls, parent_of: Dict[str, Type[KilnParentedModel]], **kwargs):
376        super().__init_subclass__(**kwargs)
377        cls._parent_of = parent_of
378        for relationship_name, child_class in parent_of.items():
379            cls._create_child_method(relationship_name, child_class)
380            cls._create_parent_methods(child_class, relationship_name)
381
382    @classmethod
383    def validate_and_save_with_subrelations(
384        cls,
385        data: Dict[str, Any],
386        path: Path | None = None,
387        parent: KilnBaseModel | None = None,
388    ):
389        """Validate and save a model instance along with all its nested child relationships.
390
391        Args:
392            data (Dict[str, Any]): Model data including child relationships
393            path (Path, optional): Path where the model should be saved
394            parent (KilnBaseModel, optional): Parent model instance for parented models
395
396        Returns:
397            KilnParentModel: The validated and saved model instance
398
399        Raises:
400            ValidationError: If validation fails for the model or any of its children
401        """
402        # Validate first, then save. Don't want error half way through, and partly persisted
403        # TODO P2: save to tmp dir, then move atomically. But need to merge directories so later.
404        cls._validate_nested(data, save=False, path=path, parent=parent)
405        instance = cls._validate_nested(data, save=True, path=path, parent=parent)
406        return instance
407
408    @classmethod
409    def _validate_nested(
410        cls,
411        data: Dict[str, Any],
412        save: bool = False,
413        parent: KilnBaseModel | None = None,
414        path: Path | None = None,
415    ):
416        # Collect all validation errors so we can report them all at once
417        validation_errors = []
418
419        try:
420            instance = cls.model_validate(data, strict=True)
421            if path is not None:
422                instance.path = path
423            if parent is not None and isinstance(instance, KilnParentedModel):
424                instance.parent = parent
425            if save:
426                instance.save_to_file()
427        except ValidationError as e:
428            instance = None
429            for suberror in e.errors():
430                validation_errors.append(suberror)
431
432        for key, value_list in data.items():
433            if key in cls._parent_of:
434                parent_type = cls._parent_of[key]
435                if not isinstance(value_list, list):
436                    raise ValueError(
437                        f"Expected a list for {key}, but got {type(value_list)}"
438                    )
439                for value_index, value in enumerate(value_list):
440                    try:
441                        if issubclass(parent_type, KilnParentModel):
442                            kwargs = {"data": value, "save": save}
443                            if instance is not None:
444                                kwargs["parent"] = instance
445                            parent_type._validate_nested(**kwargs)
446                        elif issubclass(parent_type, KilnParentedModel):
447                            # Root node
448                            subinstance = parent_type.model_validate(value, strict=True)
449                            if instance is not None:
450                                subinstance.parent = instance
451                            if save:
452                                subinstance.save_to_file()
453                        else:
454                            raise ValueError(
455                                f"Invalid type {parent_type}. Should be KilnBaseModel based."
456                            )
457                    except ValidationError as e:
458                        for suberror in e.errors():
459                            cls._append_loc(suberror, key, value_index)
460                            validation_errors.append(suberror)
461
462        if len(validation_errors) > 0:
463            raise ValidationError.from_exception_data(
464                title=f"Validation failed for {cls.__name__}",
465                line_errors=validation_errors,
466                input_type="json",
467            )
468
469        return instance
470
471    @classmethod
472    def _append_loc(
473        cls, error: ErrorDetails, current_loc: str, value_index: int | None = None
474    ):
475        orig_loc = error["loc"] if "loc" in error else None
476        new_loc: list[str | int] = [current_loc]
477        if value_index is not None:
478            new_loc.append(value_index)
479        if isinstance(orig_loc, tuple):
480            new_loc.extend(list(orig_loc))
481        elif isinstance(orig_loc, list):
482            new_loc.extend(orig_loc)
483        error["loc"] = tuple(new_loc)
ID_FIELD = FieldInfo(annotation=Union[str, NoneType], required=False, default_factory=<lambda>)
ID_TYPE = typing.Optional[str]
NAME_REGEX = '^[A-Za-z0-9 _-]+$'
NAME_FIELD = FieldInfo(annotation=str, required=True, description='A name for this entity.', metadata=[MinLen(min_length=1), MaxLen(max_length=120), _PydanticGeneralMetadata(pattern='^[A-Za-z0-9 _-]+$')])
SHORT_NAME_FIELD = FieldInfo(annotation=str, required=True, description='A name for this entity', metadata=[MinLen(min_length=1), MaxLen(max_length=32), _PydanticGeneralMetadata(pattern='^[A-Za-z0-9 _-]+$')])
def string_to_valid_name(name: str) -> str:
63def string_to_valid_name(name: str) -> str:
64    # Replace any character not allowed by NAME_REGEX with an underscore
65    valid_name = re.sub(r"[^A-Za-z0-9 _-]", "_", name)
66    # Replace consecutive underscores with a single underscore
67    valid_name = re.sub(r"_+", "_", valid_name)
68    # Remove leading and trailing underscores or whitespace
69    return valid_name.strip("_").strip()
class KilnBaseModel(pydantic.main.BaseModel):
 72class KilnBaseModel(BaseModel):
 73    """Base model for all Kiln data models with common functionality for persistence and versioning.
 74
 75    Attributes:
 76        v (int): Schema version number for migration support
 77        id (str): Unique identifier for the model instance
 78        path (Path): File system path where the model is stored
 79        created_at (datetime): Timestamp when the model was created
 80        created_by (str): User ID of the creator
 81    """
 82
 83    model_config = ConfigDict(validate_assignment=True)
 84
 85    v: int = Field(default=1)  # schema_version
 86    id: ID_TYPE = ID_FIELD
 87    path: Optional[Path] = Field(default=None)
 88    created_at: datetime = Field(default_factory=datetime.now)
 89    created_by: str = Field(default_factory=lambda: Config.shared().user_id)
 90
 91    @computed_field()
 92    def model_type(self) -> str:
 93        return self.type_name()
 94
 95    # if changing the model name, should keep the original name here for parsing old files
 96    @classmethod
 97    def type_name(cls) -> str:
 98        return snake_case(cls.__name__)
 99
100    # used as /obj_folder/base_filename.kiln
101    @classmethod
102    def base_filename(cls) -> str:
103        return cls.type_name() + ".kiln"
104
105    @classmethod
106    def load_from_folder(cls: Type[T], folderPath: Path) -> T:
107        """Load a model instance from a folder using the default filename.
108
109        Args:
110            folderPath (Path): Directory path containing the model file
111
112        Returns:
113            T: Instance of the model
114        """
115        path = folderPath / cls.base_filename()
116        return cls.load_from_file(path)
117
118    @classmethod
119    def load_from_file(cls: Type[T], path: Path) -> T:
120        """Load a model instance from a specific file path.
121
122        Args:
123            path (Path): Path to the model file
124
125        Returns:
126            T: Instance of the model
127
128        Raises:
129            ValueError: If the loaded model is not of the expected type or version
130            FileNotFoundError: If the file does not exist
131        """
132        with open(path, "r") as file:
133            file_data = file.read()
134            # TODO P2 perf: parsing the JSON twice here.
135            # Once for model_type, once for model. Can't call model_validate with parsed json because enum types break; they get strings instead of enums.
136            parsed_json = json.loads(file_data)
137            m = cls.model_validate_json(file_data, strict=True)
138            if not isinstance(m, cls):
139                raise ValueError(f"Loaded model is not of type {cls.__name__}")
140            file_data = None
141        m.path = path
142        if m.v > m.max_schema_version():
143            raise ValueError(
144                f"Cannot load from file because the schema version is higher than the current version. Upgrade kiln to the latest version. "
145                f"Class: {m.__class__.__name__}, id: {getattr(m, 'id', None)}, path: {path}, "
146                f"version: {m.v}, max version: {m.max_schema_version()}"
147            )
148        if parsed_json["model_type"] != cls.type_name():
149            raise ValueError(
150                f"Cannot load from file because the model type is incorrect. Expected {cls.type_name()}, got {parsed_json['model_type']}. "
151                f"Class: {m.__class__.__name__}, id: {getattr(m, 'id', None)}, path: {path}, "
152                f"version: {m.v}, max version: {m.max_schema_version()}"
153            )
154        return m
155
156    def save_to_file(self) -> None:
157        """Save the model instance to a file.
158
159        Raises:
160            ValueError: If the path is not set
161        """
162        path = self.build_path()
163        if path is None:
164            raise ValueError(
165                f"Cannot save to file because 'path' is not set. Class: {self.__class__.__name__}, "
166                f"id: {getattr(self, 'id', None)}, path: {path}"
167            )
168        path.parent.mkdir(parents=True, exist_ok=True)
169        json_data = self.model_dump_json(indent=2, exclude={"path"})
170        with open(path, "w") as file:
171            file.write(json_data)
172        # save the path so even if something like name changes, the file doesn't move
173        self.path = path
174
175    def delete(self) -> None:
176        if self.path is None:
177            raise ValueError("Cannot delete model because path is not set")
178        dir_path = self.path.parent if self.path.is_file() else self.path
179        if dir_path is None:
180            raise ValueError("Cannot delete model because path is not set")
181        shutil.rmtree(dir_path)
182        self.path = None
183
184    def build_path(self) -> Path | None:
185        if self.path is not None:
186            return self.path
187        return None
188
189    # increment for breaking changes
190    def max_schema_version(self) -> int:
191        return 1

Base model for all Kiln data models with common functionality for persistence and versioning.

Attributes: v (int): Schema version number for migration support id (str): Unique identifier for the model instance path (Path): File system path where the model is stored created_at (datetime): Timestamp when the model was created created_by (str): User ID of the creator

model_config = {'validate_assignment': True}

Configuration for the model, should be a dictionary conforming to [ConfigDict][pydantic.config.ConfigDict].

v: int
id: Optional[str]
path: Optional[pathlib.Path]
created_at: datetime.datetime
created_by: str
model_type: str
91    @computed_field()
92    def model_type(self) -> str:
93        return self.type_name()
@classmethod
def type_name(cls) -> str:
96    @classmethod
97    def type_name(cls) -> str:
98        return snake_case(cls.__name__)
@classmethod
def base_filename(cls) -> str:
101    @classmethod
102    def base_filename(cls) -> str:
103        return cls.type_name() + ".kiln"
@classmethod
def load_from_folder(cls: Type[~T], folderPath: pathlib.Path) -> ~T:
105    @classmethod
106    def load_from_folder(cls: Type[T], folderPath: Path) -> T:
107        """Load a model instance from a folder using the default filename.
108
109        Args:
110            folderPath (Path): Directory path containing the model file
111
112        Returns:
113            T: Instance of the model
114        """
115        path = folderPath / cls.base_filename()
116        return cls.load_from_file(path)

Load a model instance from a folder using the default filename.

Args: folderPath (Path): Directory path containing the model file

Returns: T: Instance of the model

@classmethod
def load_from_file(cls: Type[~T], path: pathlib.Path) -> ~T:
118    @classmethod
119    def load_from_file(cls: Type[T], path: Path) -> T:
120        """Load a model instance from a specific file path.
121
122        Args:
123            path (Path): Path to the model file
124
125        Returns:
126            T: Instance of the model
127
128        Raises:
129            ValueError: If the loaded model is not of the expected type or version
130            FileNotFoundError: If the file does not exist
131        """
132        with open(path, "r") as file:
133            file_data = file.read()
134            # TODO P2 perf: parsing the JSON twice here.
135            # Once for model_type, once for model. Can't call model_validate with parsed json because enum types break; they get strings instead of enums.
136            parsed_json = json.loads(file_data)
137            m = cls.model_validate_json(file_data, strict=True)
138            if not isinstance(m, cls):
139                raise ValueError(f"Loaded model is not of type {cls.__name__}")
140            file_data = None
141        m.path = path
142        if m.v > m.max_schema_version():
143            raise ValueError(
144                f"Cannot load from file because the schema version is higher than the current version. Upgrade kiln to the latest version. "
145                f"Class: {m.__class__.__name__}, id: {getattr(m, 'id', None)}, path: {path}, "
146                f"version: {m.v}, max version: {m.max_schema_version()}"
147            )
148        if parsed_json["model_type"] != cls.type_name():
149            raise ValueError(
150                f"Cannot load from file because the model type is incorrect. Expected {cls.type_name()}, got {parsed_json['model_type']}. "
151                f"Class: {m.__class__.__name__}, id: {getattr(m, 'id', None)}, path: {path}, "
152                f"version: {m.v}, max version: {m.max_schema_version()}"
153            )
154        return m

Load a model instance from a specific file path.

Args: path (Path): Path to the model file

Returns: T: Instance of the model

Raises: ValueError: If the loaded model is not of the expected type or version FileNotFoundError: If the file does not exist

def save_to_file(self) -> None:
156    def save_to_file(self) -> None:
157        """Save the model instance to a file.
158
159        Raises:
160            ValueError: If the path is not set
161        """
162        path = self.build_path()
163        if path is None:
164            raise ValueError(
165                f"Cannot save to file because 'path' is not set. Class: {self.__class__.__name__}, "
166                f"id: {getattr(self, 'id', None)}, path: {path}"
167            )
168        path.parent.mkdir(parents=True, exist_ok=True)
169        json_data = self.model_dump_json(indent=2, exclude={"path"})
170        with open(path, "w") as file:
171            file.write(json_data)
172        # save the path so even if something like name changes, the file doesn't move
173        self.path = path

Save the model instance to a file.

Raises: ValueError: If the path is not set

def delete(self) -> None:
175    def delete(self) -> None:
176        if self.path is None:
177            raise ValueError("Cannot delete model because path is not set")
178        dir_path = self.path.parent if self.path.is_file() else self.path
179        if dir_path is None:
180            raise ValueError("Cannot delete model because path is not set")
181        shutil.rmtree(dir_path)
182        self.path = None
def build_path(self) -> pathlib.Path | None:
184    def build_path(self) -> Path | None:
185        if self.path is not None:
186            return self.path
187        return None
def max_schema_version(self) -> int:
190    def max_schema_version(self) -> int:
191        return 1
model_fields: ClassVar[Dict[str, pydantic.fields.FieldInfo]] = {'v': FieldInfo(annotation=int, required=False, default=1), 'id': FieldInfo(annotation=Union[str, NoneType], required=False, default_factory=<lambda>), 'path': FieldInfo(annotation=Union[Path, NoneType], required=False, default=None), 'created_at': FieldInfo(annotation=datetime, required=False, default_factory=builtin_function_or_method), 'created_by': FieldInfo(annotation=str, required=False, default_factory=<lambda>)}

Metadata about the fields defined on the model, mapping of field names to [FieldInfo][pydantic.fields.FieldInfo] objects.

This replaces Model.__fields__ from Pydantic V1.

model_computed_fields: ClassVar[Dict[str, pydantic.fields.ComputedFieldInfo]] = {'model_type': ComputedFieldInfo(wrapped_property=<property object>, return_type=<class 'str'>, alias=None, alias_priority=None, title=None, field_title_generator=None, description=None, deprecated=None, examples=None, json_schema_extra=None, repr=True)}

A dictionary of computed field names and their corresponding ComputedFieldInfo objects.

class KilnParentedModel(KilnBaseModel):
194class KilnParentedModel(KilnBaseModel, metaclass=ABCMeta):
195    """Base model for Kiln models that have a parent-child relationship. This base class is for child models.
196
197    This class provides functionality for managing hierarchical relationships between models,
198    including parent reference handling and file system organization.
199
200    Attributes:
201        _parent (KilnBaseModel): Reference to the parent model instance
202    """
203
204    _parent: KilnBaseModel | None = None
205
206    # workaround to tell typechecker that we support the parent property, even though it's not a stock property
207    if TYPE_CHECKING:
208        parent: KilnBaseModel  # type: ignore
209
210    def __init__(self, **data):
211        super().__init__(**data)
212        if "parent" in data:
213            self.parent = data["parent"]
214
215    @property
216    def parent(self) -> Optional[KilnBaseModel]:
217        """Get the parent model instance, loading it from disk if necessary.
218
219        Returns:
220            Optional[KilnBaseModel]: The parent model instance or None if not set
221        """
222        if self._parent is not None:
223            return self._parent
224        # lazy load parent from path
225        if self.path is None:
226            return None
227        # TODO: this only works with base_filename. If we every support custom names, we need to change this.
228        parent_path = (
229            self.path.parent.parent.parent
230            / self.__class__.parent_type().base_filename()
231        )
232        if parent_path is None:
233            return None
234        self._parent = self.__class__.parent_type().load_from_file(parent_path)
235        return self._parent
236
237    @parent.setter
238    def parent(self, value: Optional[KilnBaseModel]):
239        if value is not None:
240            expected_parent_type = self.__class__.parent_type()
241            if not isinstance(value, expected_parent_type):
242                raise ValueError(
243                    f"Parent must be of type {expected_parent_type}, but was {type(value)}"
244                )
245        self._parent = value
246
247    # Dynamically implemented by KilnParentModel method injection
248    @classmethod
249    def relationship_name(cls) -> str:
250        raise NotImplementedError("Relationship name must be implemented")
251
252    # Dynamically implemented by KilnParentModel method injection
253    @classmethod
254    def parent_type(cls) -> Type[KilnBaseModel]:
255        raise NotImplementedError("Parent type must be implemented")
256
257    @model_validator(mode="after")
258    def check_parent_type(self) -> Self:
259        if self._parent is not None:
260            expected_parent_type = self.__class__.parent_type()
261            if not isinstance(self._parent, expected_parent_type):
262                raise ValueError(
263                    f"Parent must be of type {expected_parent_type}, but was {type(self._parent)}"
264                )
265        return self
266
267    def build_child_dirname(self) -> Path:
268        # Default implementation for readable folder names.
269        # {id} - {name}/{type}.kiln
270        if self.id is None:
271            # consider generating an ID here. But if it's been cleared, we've already used this without one so raise for now.
272            raise ValueError("ID is not set - can not save or build path")
273        path = self.id
274        name = getattr(self, "name", None)
275        if name is not None:
276            path = f"{path} - {name[:32]}"
277        return Path(path)
278
279    def build_path(self) -> Path | None:
280        # if specifically loaded from an existing path, keep that no matter what
281        # this ensures the file structure is easy to use with git/version control
282        # and that changes to things like name (which impacts default path) don't leave dangling files
283        if self.path is not None:
284            return self.path
285        # Build a path under parent_folder/relationship/file.kiln
286        if self.parent is None:
287            return None
288        parent_path = self.parent.build_path()
289        if parent_path is None:
290            return None
291        parent_folder = parent_path.parent
292        if parent_folder is None:
293            return None
294        return (
295            parent_folder
296            / self.__class__.relationship_name()
297            / self.build_child_dirname()
298            / self.__class__.base_filename()
299        )
300
301    @classmethod
302    def all_children_of_parent_path(
303        cls: Type[PT], parent_path: Path | None
304    ) -> list[PT]:
305        if parent_path is None:
306            # children are disk based. If not saved, they don't exist
307            return []
308
309        # Determine the parent folder
310        if parent_path.is_file():
311            parent_folder = parent_path.parent
312        else:
313            parent_folder = parent_path
314
315        parent = cls.parent_type().load_from_file(parent_path)
316        if parent is None:
317            raise ValueError("Parent must be set to load children")
318
319        # Ignore type error: this is abstract base class, but children must implement relationship_name
320        relationship_folder = parent_folder / Path(cls.relationship_name())  # type: ignore
321
322        if not relationship_folder.exists() or not relationship_folder.is_dir():
323            return []
324
325        # Collect all /relationship/{id}/{base_filename.kiln} files in the relationship folder
326        children = []
327        for child_file in relationship_folder.glob(f"**/{cls.base_filename()}"):
328            child = cls.load_from_file(child_file)
329            children.append(child)
330
331        return children

Base model for Kiln models that have a parent-child relationship. This base class is for child models.

This class provides functionality for managing hierarchical relationships between models, including parent reference handling and file system organization.

Attributes: _parent (KilnBaseModel): Reference to the parent model instance

KilnParentedModel(**data)
210    def __init__(self, **data):
211        super().__init__(**data)
212        if "parent" in data:
213            self.parent = data["parent"]

Create a new model by parsing and validating input data from keyword arguments.

Raises [ValidationError][pydantic_core.ValidationError] if the input data cannot be validated to form a valid model.

self is explicitly positional-only to allow self as a field name.

parent: Optional[KilnBaseModel]
215    @property
216    def parent(self) -> Optional[KilnBaseModel]:
217        """Get the parent model instance, loading it from disk if necessary.
218
219        Returns:
220            Optional[KilnBaseModel]: The parent model instance or None if not set
221        """
222        if self._parent is not None:
223            return self._parent
224        # lazy load parent from path
225        if self.path is None:
226            return None
227        # TODO: this only works with base_filename. If we every support custom names, we need to change this.
228        parent_path = (
229            self.path.parent.parent.parent
230            / self.__class__.parent_type().base_filename()
231        )
232        if parent_path is None:
233            return None
234        self._parent = self.__class__.parent_type().load_from_file(parent_path)
235        return self._parent

Get the parent model instance, loading it from disk if necessary.

Returns: Optional[KilnBaseModel]: The parent model instance or None if not set

@classmethod
def relationship_name(cls) -> str:
248    @classmethod
249    def relationship_name(cls) -> str:
250        raise NotImplementedError("Relationship name must be implemented")
@classmethod
def parent_type(cls) -> Type[KilnBaseModel]:
253    @classmethod
254    def parent_type(cls) -> Type[KilnBaseModel]:
255        raise NotImplementedError("Parent type must be implemented")
@model_validator(mode='after')
def check_parent_type(self) -> Self:
257    @model_validator(mode="after")
258    def check_parent_type(self) -> Self:
259        if self._parent is not None:
260            expected_parent_type = self.__class__.parent_type()
261            if not isinstance(self._parent, expected_parent_type):
262                raise ValueError(
263                    f"Parent must be of type {expected_parent_type}, but was {type(self._parent)}"
264                )
265        return self
def build_child_dirname(self) -> pathlib.Path:
267    def build_child_dirname(self) -> Path:
268        # Default implementation for readable folder names.
269        # {id} - {name}/{type}.kiln
270        if self.id is None:
271            # consider generating an ID here. But if it's been cleared, we've already used this without one so raise for now.
272            raise ValueError("ID is not set - can not save or build path")
273        path = self.id
274        name = getattr(self, "name", None)
275        if name is not None:
276            path = f"{path} - {name[:32]}"
277        return Path(path)
def build_path(self) -> pathlib.Path | None:
279    def build_path(self) -> Path | None:
280        # if specifically loaded from an existing path, keep that no matter what
281        # this ensures the file structure is easy to use with git/version control
282        # and that changes to things like name (which impacts default path) don't leave dangling files
283        if self.path is not None:
284            return self.path
285        # Build a path under parent_folder/relationship/file.kiln
286        if self.parent is None:
287            return None
288        parent_path = self.parent.build_path()
289        if parent_path is None:
290            return None
291        parent_folder = parent_path.parent
292        if parent_folder is None:
293            return None
294        return (
295            parent_folder
296            / self.__class__.relationship_name()
297            / self.build_child_dirname()
298            / self.__class__.base_filename()
299        )
@classmethod
def all_children_of_parent_path(cls: Type[~PT], parent_path: pathlib.Path | None) -> list[~PT]:
301    @classmethod
302    def all_children_of_parent_path(
303        cls: Type[PT], parent_path: Path | None
304    ) -> list[PT]:
305        if parent_path is None:
306            # children are disk based. If not saved, they don't exist
307            return []
308
309        # Determine the parent folder
310        if parent_path.is_file():
311            parent_folder = parent_path.parent
312        else:
313            parent_folder = parent_path
314
315        parent = cls.parent_type().load_from_file(parent_path)
316        if parent is None:
317            raise ValueError("Parent must be set to load children")
318
319        # Ignore type error: this is abstract base class, but children must implement relationship_name
320        relationship_folder = parent_folder / Path(cls.relationship_name())  # type: ignore
321
322        if not relationship_folder.exists() or not relationship_folder.is_dir():
323            return []
324
325        # Collect all /relationship/{id}/{base_filename.kiln} files in the relationship folder
326        children = []
327        for child_file in relationship_folder.glob(f"**/{cls.base_filename()}"):
328            child = cls.load_from_file(child_file)
329            children.append(child)
330
331        return children
model_config = {'validate_assignment': True}

Configuration for the model, should be a dictionary conforming to [ConfigDict][pydantic.config.ConfigDict].

model_fields: ClassVar[Dict[str, pydantic.fields.FieldInfo]] = {'v': FieldInfo(annotation=int, required=False, default=1), 'id': FieldInfo(annotation=Union[str, NoneType], required=False, default_factory=<lambda>), 'path': FieldInfo(annotation=Union[Path, NoneType], required=False, default=None), 'created_at': FieldInfo(annotation=datetime, required=False, default_factory=builtin_function_or_method), 'created_by': FieldInfo(annotation=str, required=False, default_factory=<lambda>)}

Metadata about the fields defined on the model, mapping of field names to [FieldInfo][pydantic.fields.FieldInfo] objects.

This replaces Model.__fields__ from Pydantic V1.

model_computed_fields: ClassVar[Dict[str, pydantic.fields.ComputedFieldInfo]] = {'model_type': ComputedFieldInfo(wrapped_property=<property object>, return_type=<class 'str'>, alias=None, alias_priority=None, title=None, field_title_generator=None, description=None, deprecated=None, examples=None, json_schema_extra=None, repr=True)}

A dictionary of computed field names and their corresponding ComputedFieldInfo objects.

def model_post_init(self: pydantic.main.BaseModel, context: Any, /) -> None:
306def init_private_attributes(self: BaseModel, context: Any, /) -> None:
307    """This function is meant to behave like a BaseModel method to initialise private attributes.
308
309    It takes context as an argument since that's what pydantic-core passes when calling it.
310
311    Args:
312        self: The BaseModel instance.
313        context: The context.
314    """
315    if getattr(self, '__pydantic_private__', None) is None:
316        pydantic_private = {}
317        for name, private_attr in self.__private_attributes__.items():
318            default = private_attr.get_default()
319            if default is not PydanticUndefined:
320                pydantic_private[name] = default
321        object_setattr(self, '__pydantic_private__', pydantic_private)

This function is meant to behave like a BaseModel method to initialise private attributes.

It takes context as an argument since that's what pydantic-core passes when calling it.

Args: self: The BaseModel instance. context: The context.

class KilnParentModel(KilnBaseModel):
336class KilnParentModel(KilnBaseModel, metaclass=ABCMeta):
337    """Base model for Kiln models that can have child models.
338
339    This class provides functionality for managing collections of child models and their persistence.
340    Child relationships must be defined using the parent_of parameter in the class definition.
341
342    Args:
343        parent_of (Dict[str, Type[KilnParentedModel]]): Mapping of relationship names to child model types
344    """
345
346    @classmethod
347    def _create_child_method(
348        cls, relationship_name: str, child_class: Type[KilnParentedModel]
349    ):
350        def child_method(self) -> list[child_class]:
351            return child_class.all_children_of_parent_path(self.path)
352
353        child_method.__name__ = relationship_name
354        child_method.__annotations__ = {"return": List[child_class]}
355        setattr(cls, relationship_name, child_method)
356
357    @classmethod
358    def _create_parent_methods(
359        cls, targetCls: Type[KilnParentedModel], relationship_name: str
360    ):
361        def parent_class_method() -> Type[KilnParentModel]:
362            return cls
363
364        parent_class_method.__name__ = "parent_type"
365        parent_class_method.__annotations__ = {"return": Type[KilnParentModel]}
366        setattr(targetCls, "parent_type", parent_class_method)
367
368        def relationship_name_method() -> str:
369            return relationship_name
370
371        relationship_name_method.__name__ = "relationship_name"
372        relationship_name_method.__annotations__ = {"return": str}
373        setattr(targetCls, "relationship_name", relationship_name_method)
374
375    @classmethod
376    def __init_subclass__(cls, parent_of: Dict[str, Type[KilnParentedModel]], **kwargs):
377        super().__init_subclass__(**kwargs)
378        cls._parent_of = parent_of
379        for relationship_name, child_class in parent_of.items():
380            cls._create_child_method(relationship_name, child_class)
381            cls._create_parent_methods(child_class, relationship_name)
382
383    @classmethod
384    def validate_and_save_with_subrelations(
385        cls,
386        data: Dict[str, Any],
387        path: Path | None = None,
388        parent: KilnBaseModel | None = None,
389    ):
390        """Validate and save a model instance along with all its nested child relationships.
391
392        Args:
393            data (Dict[str, Any]): Model data including child relationships
394            path (Path, optional): Path where the model should be saved
395            parent (KilnBaseModel, optional): Parent model instance for parented models
396
397        Returns:
398            KilnParentModel: The validated and saved model instance
399
400        Raises:
401            ValidationError: If validation fails for the model or any of its children
402        """
403        # Validate first, then save. Don't want error half way through, and partly persisted
404        # TODO P2: save to tmp dir, then move atomically. But need to merge directories so later.
405        cls._validate_nested(data, save=False, path=path, parent=parent)
406        instance = cls._validate_nested(data, save=True, path=path, parent=parent)
407        return instance
408
409    @classmethod
410    def _validate_nested(
411        cls,
412        data: Dict[str, Any],
413        save: bool = False,
414        parent: KilnBaseModel | None = None,
415        path: Path | None = None,
416    ):
417        # Collect all validation errors so we can report them all at once
418        validation_errors = []
419
420        try:
421            instance = cls.model_validate(data, strict=True)
422            if path is not None:
423                instance.path = path
424            if parent is not None and isinstance(instance, KilnParentedModel):
425                instance.parent = parent
426            if save:
427                instance.save_to_file()
428        except ValidationError as e:
429            instance = None
430            for suberror in e.errors():
431                validation_errors.append(suberror)
432
433        for key, value_list in data.items():
434            if key in cls._parent_of:
435                parent_type = cls._parent_of[key]
436                if not isinstance(value_list, list):
437                    raise ValueError(
438                        f"Expected a list for {key}, but got {type(value_list)}"
439                    )
440                for value_index, value in enumerate(value_list):
441                    try:
442                        if issubclass(parent_type, KilnParentModel):
443                            kwargs = {"data": value, "save": save}
444                            if instance is not None:
445                                kwargs["parent"] = instance
446                            parent_type._validate_nested(**kwargs)
447                        elif issubclass(parent_type, KilnParentedModel):
448                            # Root node
449                            subinstance = parent_type.model_validate(value, strict=True)
450                            if instance is not None:
451                                subinstance.parent = instance
452                            if save:
453                                subinstance.save_to_file()
454                        else:
455                            raise ValueError(
456                                f"Invalid type {parent_type}. Should be KilnBaseModel based."
457                            )
458                    except ValidationError as e:
459                        for suberror in e.errors():
460                            cls._append_loc(suberror, key, value_index)
461                            validation_errors.append(suberror)
462
463        if len(validation_errors) > 0:
464            raise ValidationError.from_exception_data(
465                title=f"Validation failed for {cls.__name__}",
466                line_errors=validation_errors,
467                input_type="json",
468            )
469
470        return instance
471
472    @classmethod
473    def _append_loc(
474        cls, error: ErrorDetails, current_loc: str, value_index: int | None = None
475    ):
476        orig_loc = error["loc"] if "loc" in error else None
477        new_loc: list[str | int] = [current_loc]
478        if value_index is not None:
479            new_loc.append(value_index)
480        if isinstance(orig_loc, tuple):
481            new_loc.extend(list(orig_loc))
482        elif isinstance(orig_loc, list):
483            new_loc.extend(orig_loc)
484        error["loc"] = tuple(new_loc)

Base model for Kiln models that can have child models.

This class provides functionality for managing collections of child models and their persistence. Child relationships must be defined using the parent_of parameter in the class definition.

Args: parent_of (Dict[str, Type[KilnParentedModel]]): Mapping of relationship names to child model types

@classmethod
def validate_and_save_with_subrelations( cls, data: Dict[str, Any], path: pathlib.Path | None = None, parent: KilnBaseModel | None = None):
383    @classmethod
384    def validate_and_save_with_subrelations(
385        cls,
386        data: Dict[str, Any],
387        path: Path | None = None,
388        parent: KilnBaseModel | None = None,
389    ):
390        """Validate and save a model instance along with all its nested child relationships.
391
392        Args:
393            data (Dict[str, Any]): Model data including child relationships
394            path (Path, optional): Path where the model should be saved
395            parent (KilnBaseModel, optional): Parent model instance for parented models
396
397        Returns:
398            KilnParentModel: The validated and saved model instance
399
400        Raises:
401            ValidationError: If validation fails for the model or any of its children
402        """
403        # Validate first, then save. Don't want error half way through, and partly persisted
404        # TODO P2: save to tmp dir, then move atomically. But need to merge directories so later.
405        cls._validate_nested(data, save=False, path=path, parent=parent)
406        instance = cls._validate_nested(data, save=True, path=path, parent=parent)
407        return instance

Validate and save a model instance along with all its nested child relationships.

Args: data (Dict[str, Any]): Model data including child relationships path (Path, optional): Path where the model should be saved parent (KilnBaseModel, optional): Parent model instance for parented models

Returns: KilnParentModel: The validated and saved model instance

Raises: ValidationError: If validation fails for the model or any of its children

model_config = {'validate_assignment': True}

Configuration for the model, should be a dictionary conforming to [ConfigDict][pydantic.config.ConfigDict].

model_fields: ClassVar[Dict[str, pydantic.fields.FieldInfo]] = {'v': FieldInfo(annotation=int, required=False, default=1), 'id': FieldInfo(annotation=Union[str, NoneType], required=False, default_factory=<lambda>), 'path': FieldInfo(annotation=Union[Path, NoneType], required=False, default=None), 'created_at': FieldInfo(annotation=datetime, required=False, default_factory=builtin_function_or_method), 'created_by': FieldInfo(annotation=str, required=False, default_factory=<lambda>)}

Metadata about the fields defined on the model, mapping of field names to [FieldInfo][pydantic.fields.FieldInfo] objects.

This replaces Model.__fields__ from Pydantic V1.

model_computed_fields: ClassVar[Dict[str, pydantic.fields.ComputedFieldInfo]] = {'model_type': ComputedFieldInfo(wrapped_property=<property object>, return_type=<class 'str'>, alias=None, alias_priority=None, title=None, field_title_generator=None, description=None, deprecated=None, examples=None, json_schema_extra=None, repr=True)}

A dictionary of computed field names and their corresponding ComputedFieldInfo objects.