kiln_ai.datamodel.basemodel

  1import json
  2import os
  3import re
  4import shutil
  5import uuid
  6from abc import ABCMeta
  7from builtins import classmethod
  8from datetime import datetime
  9from pathlib import Path
 10from typing import (
 11    Any,
 12    Dict,
 13    List,
 14    Optional,
 15    Type,
 16    TypeVar,
 17)
 18
 19from pydantic import (
 20    BaseModel,
 21    ConfigDict,
 22    Field,
 23    ValidationError,
 24    ValidationInfo,
 25    computed_field,
 26    model_validator,
 27)
 28from pydantic_core import ErrorDetails
 29from typing_extensions import Self
 30
 31from kiln_ai.datamodel.model_cache import ModelCache
 32from kiln_ai.utils.config import Config
 33from kiln_ai.utils.formatting import snake_case
 34
 35# ID is a 12 digit random integer string.
 36# Should be unique per item, at least inside the context of a parent/child relationship.
 37# Use integers to make it easier to type for a search function.
 38# Allow none, even though we generate it, because we clear it in the REST API if the object is ephemeral (not persisted to disk)
 39ID_FIELD = Field(default_factory=lambda: str(uuid.uuid4().int)[:12])
 40ID_TYPE = Optional[str]
 41T = TypeVar("T", bound="KilnBaseModel")
 42PT = TypeVar("PT", bound="KilnParentedModel")
 43
 44
 45# Naming conventions:
 46# 1) Names are filename safe as they may be used as file names. They are informational and not to be used in prompts/training/validation.
 47# 2) Descrptions are for Kiln users to describe/understanding the purpose of this object. They must never be used in prompts/training/validation. Use "instruction/requirements" instead.
 48
 49# Filename compatible names
 50NAME_REGEX = r"^[A-Za-z0-9 _-]+$"
 51NAME_FIELD = Field(
 52    min_length=1,
 53    max_length=120,
 54    pattern=NAME_REGEX,
 55    description="A name for this entity.",
 56)
 57SHORT_NAME_FIELD = Field(
 58    min_length=1,
 59    max_length=32,
 60    pattern=NAME_REGEX,
 61    description="A name for this entity",
 62)
 63
 64
 65def string_to_valid_name(name: str) -> str:
 66    # Replace any character not allowed by NAME_REGEX with an underscore
 67    valid_name = re.sub(r"[^A-Za-z0-9 _-]", "_", name)
 68    # Replace consecutive underscores with a single underscore
 69    valid_name = re.sub(r"_+", "_", valid_name)
 70    # Remove leading and trailing underscores or whitespace
 71    return valid_name.strip("_").strip()
 72
 73
 74class KilnBaseModel(BaseModel):
 75    """Base model for all Kiln data models with common functionality for persistence and versioning.
 76
 77    Attributes:
 78        v (int): Schema version number for migration support
 79        id (str): Unique identifier for the model instance
 80        path (Path): File system path where the model is stored
 81        created_at (datetime): Timestamp when the model was created
 82        created_by (str): User ID of the creator
 83    """
 84
 85    model_config = ConfigDict(validate_assignment=True)
 86
 87    v: int = Field(default=1)  # schema_version
 88    id: ID_TYPE = ID_FIELD
 89    path: Optional[Path] = Field(default=None)
 90    created_at: datetime = Field(default_factory=datetime.now)
 91    created_by: str = Field(default_factory=lambda: Config.shared().user_id)
 92
 93    _loaded_from_file: bool = False
 94
 95    @computed_field()
 96    def model_type(self) -> str:
 97        return self.type_name()
 98
 99    # if changing the model name, should keep the original name here for parsing old files
100    @classmethod
101    def type_name(cls) -> str:
102        return snake_case(cls.__name__)
103
104    # used as /obj_folder/base_filename.kiln
105    @classmethod
106    def base_filename(cls) -> str:
107        return cls.type_name() + ".kiln"
108
109    @classmethod
110    def load_from_folder(cls: Type[T], folderPath: Path) -> T:
111        """Load a model instance from a folder using the default filename.
112
113        Args:
114            folderPath (Path): Directory path containing the model file
115
116        Returns:
117            T: Instance of the model
118        """
119        path = folderPath / cls.base_filename()
120        return cls.load_from_file(path)
121
122    @classmethod
123    def load_from_file(cls: Type[T], path: Path | str, readonly: bool = False) -> T:
124        """Load a model instance from a specific file path.
125
126        Args:
127            path (Path): Path to the model file
128            readonly (bool): If True, the model will be returned in readonly mode (cached instance, not a copy, not safe to mutate)
129
130        Returns:
131            T: Instance of the model
132
133        Raises:
134            ValueError: If the loaded model is not of the expected type or version
135            FileNotFoundError: If the file does not exist
136        """
137        if isinstance(path, str):
138            path = Path(path)
139        cached_model = ModelCache.shared().get_model(path, cls, readonly=readonly)
140        if cached_model is not None:
141            return cached_model
142        with open(path, "r", encoding="utf-8") as file:
143            # modified time of file for cache invalidation. From file descriptor so it's atomic w read.
144            mtime_ns = os.fstat(file.fileno()).st_mtime_ns
145            file_data = file.read()
146            parsed_json = json.loads(file_data)
147            m = cls.model_validate(parsed_json, context={"loading_from_file": True})
148            if not isinstance(m, cls):
149                raise ValueError(f"Loaded model is not of type {cls.__name__}")
150            m._loaded_from_file = True
151            file_data = None
152        m.path = path
153        if m.v > m.max_schema_version():
154            raise ValueError(
155                f"Cannot load from file because the schema version is higher than the current version. Upgrade kiln to the latest version. "
156                f"Class: {m.__class__.__name__}, id: {getattr(m, 'id', None)}, path: {path}, "
157                f"version: {m.v}, max version: {m.max_schema_version()}"
158            )
159        if parsed_json["model_type"] != cls.type_name():
160            raise ValueError(
161                f"Cannot load from file because the model type is incorrect. Expected {cls.type_name()}, got {parsed_json['model_type']}. "
162                f"Class: {m.__class__.__name__}, id: {getattr(m, 'id', None)}, path: {path}, "
163                f"version: {m.v}, max version: {m.max_schema_version()}"
164            )
165        ModelCache.shared().set_model(path, m, mtime_ns)
166        return m
167
168    def loaded_from_file(self, info: ValidationInfo | None = None) -> bool:
169        # Two methods of indicated it's loaded from file:
170        # 1) info.context.get("loading_from_file") -> During actual loading, before we can set _loaded_from_file
171        # 2) self._loaded_from_file -> After loading, set by the loader
172        if self.loading_from_file(info):
173            return True
174        return self._loaded_from_file
175
176    # indicates the model is currently being loaded from file (not mutating it after)
177    def loading_from_file(self, info: ValidationInfo | None = None) -> bool:
178        # info.context.get("loading_from_file") -> During actual loading, before we can set _loaded_from_file
179        if (
180            info is not None
181            and info.context is not None
182            and info.context.get("loading_from_file", False)
183        ):
184            return True
185        return False
186
187    def save_to_file(self) -> None:
188        """Save the model instance to a file.
189
190        Raises:
191            ValueError: If the path is not set
192        """
193        path = self.build_path()
194        if path is None:
195            raise ValueError(
196                f"Cannot save to file because 'path' is not set. Class: {self.__class__.__name__}, "
197                f"id: {getattr(self, 'id', None)}, path: {path}"
198            )
199        path.parent.mkdir(parents=True, exist_ok=True)
200        json_data = self.model_dump_json(indent=2, exclude={"path"})
201        with open(path, "w", encoding="utf-8") as file:
202            file.write(json_data)
203        # save the path so even if something like name changes, the file doesn't move
204        self.path = path
205        # We could save, but invalidating will trigger load on next use.
206        # This ensures everything in cache is loaded from disk, and the cache perfectly reflects what's on disk
207        ModelCache.shared().invalidate(path)
208
209    def delete(self) -> None:
210        if self.path is None:
211            raise ValueError("Cannot delete model because path is not set")
212        dir_path = self.path.parent if self.path.is_file() else self.path
213        if dir_path is None:
214            raise ValueError("Cannot delete model because path is not set")
215        shutil.rmtree(dir_path)
216        ModelCache.shared().invalidate(self.path)
217        self.path = None
218
219    def build_path(self) -> Path | None:
220        if self.path is not None:
221            return self.path
222        return None
223
224    # increment for breaking changes
225    def max_schema_version(self) -> int:
226        return 1
227
228
229class KilnParentedModel(KilnBaseModel, metaclass=ABCMeta):
230    """Base model for Kiln models that have a parent-child relationship. This base class is for child models.
231
232    This class provides functionality for managing hierarchical relationships between models,
233    including parent reference handling and file system organization.
234
235    Attributes:
236        parent (KilnBaseModel): Reference to the parent model instance. Not persisted, just in memory.
237    """
238
239    # Parent is an in memory only reference to parent. If it's set we use that. If not we'll try to load it from disk based on the path.
240    # We don't persist the parent reference to disk. See the accessors below for how we make it a clean api (parent accessor will lazy load from disk)
241    parent: Optional[KilnBaseModel] = Field(default=None, exclude=True)
242
243    def __getattribute__(self, name: str) -> Any:
244        if name == "parent":
245            return self.load_parent()
246        return super().__getattribute__(name)
247
248    def cached_parent(self) -> Optional[KilnBaseModel]:
249        return object.__getattribute__(self, "parent")
250
251    def load_parent(self) -> Optional[KilnBaseModel]:
252        """Get the parent model instance, loading it from disk if necessary.
253
254        Returns:
255            Optional[KilnBaseModel]: The parent model instance or None if not set
256        """
257        cached_parent = self.cached_parent()
258        if cached_parent is not None:
259            return cached_parent
260
261        # lazy load parent from path
262        if self.path is None:
263            return None
264        # Note: this only works with base_filename. If we every support custom names, we need to change this.
265        parent_path = (
266            self.path.parent.parent.parent
267            / self.__class__.parent_type().base_filename()
268        )
269        if parent_path is None:
270            return None
271        loaded_parent = self.__class__.parent_type().load_from_file(parent_path)
272        self.parent = loaded_parent
273        return loaded_parent
274
275    # Dynamically implemented by KilnParentModel method injection
276    @classmethod
277    def relationship_name(cls) -> str:
278        raise NotImplementedError("Relationship name must be implemented")
279
280    # Dynamically implemented by KilnParentModel method injection
281    @classmethod
282    def parent_type(cls) -> Type[KilnBaseModel]:
283        raise NotImplementedError("Parent type must be implemented")
284
285    @model_validator(mode="after")
286    def check_parent_type(self) -> Self:
287        cached_parent = self.cached_parent()
288        if cached_parent is not None:
289            expected_parent_type = self.__class__.parent_type()
290            if not isinstance(cached_parent, expected_parent_type):
291                raise ValueError(
292                    f"Parent must be of type {expected_parent_type}, but was {type(cached_parent)}"
293                )
294        return self
295
296    def build_child_dirname(self) -> Path:
297        # Default implementation for readable folder names.
298        # {id} - {name}/{type}.kiln
299        if self.id is None:
300            # consider generating an ID here. But if it's been cleared, we've already used this without one so raise for now.
301            raise ValueError("ID is not set - can not save or build path")
302        path = self.id
303        name = getattr(self, "name", None)
304        if name is not None:
305            path = f"{path} - {name[:32]}"
306        return Path(path)
307
308    def build_path(self) -> Path | None:
309        # if specifically loaded from an existing path, keep that no matter what
310        # this ensures the file structure is easy to use with git/version control
311        # and that changes to things like name (which impacts default path) don't leave dangling files
312        if self.path is not None:
313            return self.path
314        # Build a path under parent_folder/relationship/file.kiln
315        if self.parent is None:
316            return None
317        parent_path = self.parent.build_path()
318        if parent_path is None:
319            return None
320        parent_folder = parent_path.parent
321        if parent_folder is None:
322            return None
323        return (
324            parent_folder
325            / self.__class__.relationship_name()
326            / self.build_child_dirname()
327            / self.__class__.base_filename()
328        )
329
330    @classmethod
331    def iterate_children_paths_of_parent_path(cls: Type[PT], parent_path: Path | None):
332        if parent_path is None:
333            # children are disk based. If not saved, they don't exist
334            return []
335
336        # Determine the parent folder
337        if parent_path.is_file():
338            parent_folder = parent_path.parent
339        else:
340            parent_folder = parent_path
341
342        parent = cls.parent_type().load_from_file(parent_path)
343        if parent is None:
344            raise ValueError("Parent must be set to load children")
345
346        # Ignore type error: this is abstract base class, but children must implement relationship_name
347        relationship_folder = parent_folder / Path(cls.relationship_name())  # type: ignore
348
349        if not relationship_folder.exists() or not relationship_folder.is_dir():
350            return []
351
352        # Collect all /relationship/{id}/{base_filename.kiln} files in the relationship folder
353        # manual code instead of glob for performance (5x speedup over glob)
354
355        base_filename = cls.base_filename()
356        # Iterate through immediate subdirectories using scandir for better performance
357        # Benchmark: scandir is 10x faster than glob, so worth the extra code
358        with os.scandir(relationship_folder) as entries:
359            for entry in entries:
360                if not entry.is_dir():
361                    continue
362
363                child_file = Path(entry.path) / base_filename
364                if child_file.is_file():
365                    yield child_file
366
367    @classmethod
368    def all_children_of_parent_path(
369        cls: Type[PT], parent_path: Path | None, readonly: bool = False
370    ) -> list[PT]:
371        children = []
372        for child_path in cls.iterate_children_paths_of_parent_path(parent_path):
373            item = cls.load_from_file(child_path, readonly=readonly)
374            children.append(item)
375        return children
376
377    @classmethod
378    def from_id_and_parent_path(
379        cls: Type[PT], id: str, parent_path: Path | None
380    ) -> PT | None:
381        """
382        Fast search by ID using the cache. Avoids the model_copy overhead on all but the exact match.
383
384        Uses cache so still slow on first load.
385        """
386        if parent_path is None:
387            return None
388
389        # Note: we're using the in-file ID. We could make this faster using the path-ID if this becomes perf bottleneck, but it's better to have 1 source of truth.
390        for child_path in cls.iterate_children_paths_of_parent_path(parent_path):
391            child_id = ModelCache.shared().get_model_id(child_path, cls)
392            if child_id == id:
393                return cls.load_from_file(child_path)
394            if child_id is None:
395                child = cls.load_from_file(child_path)
396                if child.id == id:
397                    return child
398        return None
399
400
401# Parent create methods for all child relationships
402# You must pass in parent_of in the subclass definition, defining the child relationships
403class KilnParentModel(KilnBaseModel, metaclass=ABCMeta):
404    """Base model for Kiln models that can have child models.
405
406    This class provides functionality for managing collections of child models and their persistence.
407    Child relationships must be defined using the parent_of parameter in the class definition.
408
409    Args:
410        parent_of (Dict[str, Type[KilnParentedModel]]): Mapping of relationship names to child model types
411    """
412
413    @classmethod
414    def _create_child_method(
415        cls, relationship_name: str, child_class: Type[KilnParentedModel]
416    ):
417        def child_method(self, readonly: bool = False) -> list[child_class]:
418            return child_class.all_children_of_parent_path(self.path, readonly=readonly)
419
420        child_method.__name__ = relationship_name
421        child_method.__annotations__ = {"return": List[child_class]}
422        setattr(cls, relationship_name, child_method)
423
424    @classmethod
425    def _create_parent_methods(
426        cls, targetCls: Type[KilnParentedModel], relationship_name: str
427    ):
428        def parent_class_method() -> Type[KilnParentModel]:
429            return cls
430
431        parent_class_method.__name__ = "parent_type"
432        parent_class_method.__annotations__ = {"return": Type[KilnParentModel]}
433        setattr(targetCls, "parent_type", parent_class_method)
434
435        def relationship_name_method() -> str:
436            return relationship_name
437
438        relationship_name_method.__name__ = "relationship_name"
439        relationship_name_method.__annotations__ = {"return": str}
440        setattr(targetCls, "relationship_name", relationship_name_method)
441
442    @classmethod
443    def __init_subclass__(cls, parent_of: Dict[str, Type[KilnParentedModel]], **kwargs):
444        super().__init_subclass__(**kwargs)
445        cls._parent_of = parent_of
446        for relationship_name, child_class in parent_of.items():
447            cls._create_child_method(relationship_name, child_class)
448            cls._create_parent_methods(child_class, relationship_name)
449
450    @classmethod
451    def validate_and_save_with_subrelations(
452        cls,
453        data: Dict[str, Any],
454        path: Path | None = None,
455        parent: KilnBaseModel | None = None,
456    ):
457        """Validate and save a model instance along with all its nested child relationships.
458
459        Args:
460            data (Dict[str, Any]): Model data including child relationships
461            path (Path, optional): Path where the model should be saved
462            parent (KilnBaseModel, optional): Parent model instance for parented models
463
464        Returns:
465            KilnParentModel: The validated and saved model instance
466
467        Raises:
468            ValidationError: If validation fails for the model or any of its children
469        """
470        # Validate first, then save. Don't want error half way through, and partly persisted
471        # TODO P2: save to tmp dir, then move atomically. But need to merge directories so later.
472        cls._validate_nested(data, save=False, path=path, parent=parent)
473        instance = cls._validate_nested(data, save=True, path=path, parent=parent)
474        return instance
475
476    @classmethod
477    def _validate_nested(
478        cls,
479        data: Dict[str, Any],
480        save: bool = False,
481        parent: KilnBaseModel | None = None,
482        path: Path | None = None,
483    ):
484        # Collect all validation errors so we can report them all at once
485        validation_errors = []
486
487        try:
488            instance = cls.model_validate(data)
489            if path is not None:
490                instance.path = path
491            if parent is not None and isinstance(instance, KilnParentedModel):
492                instance.parent = parent
493            if save:
494                instance.save_to_file()
495        except ValidationError as e:
496            instance = None
497            for suberror in e.errors():
498                validation_errors.append(suberror)
499
500        for key, value_list in data.items():
501            if key in cls._parent_of:
502                parent_type = cls._parent_of[key]
503                if not isinstance(value_list, list):
504                    raise ValueError(
505                        f"Expected a list for {key}, but got {type(value_list)}"
506                    )
507                for value_index, value in enumerate(value_list):
508                    try:
509                        if issubclass(parent_type, KilnParentModel):
510                            kwargs = {"data": value, "save": save}
511                            if instance is not None:
512                                kwargs["parent"] = instance
513                            parent_type._validate_nested(**kwargs)
514                        elif issubclass(parent_type, KilnParentedModel):
515                            # Root node
516                            subinstance = parent_type.model_validate(value)
517                            if instance is not None:
518                                subinstance.parent = instance
519                            if save:
520                                subinstance.save_to_file()
521                        else:
522                            raise ValueError(
523                                f"Invalid type {parent_type}. Should be KilnBaseModel based."
524                            )
525                    except ValidationError as e:
526                        for suberror in e.errors():
527                            cls._append_loc(suberror, key, value_index)
528                            validation_errors.append(suberror)
529
530        if len(validation_errors) > 0:
531            raise ValidationError.from_exception_data(
532                title=f"Validation failed for {cls.__name__}",
533                line_errors=validation_errors,
534                input_type="json",
535            )
536
537        return instance
538
539    @classmethod
540    def _append_loc(
541        cls, error: ErrorDetails, current_loc: str, value_index: int | None = None
542    ):
543        orig_loc = error["loc"] if "loc" in error else None
544        new_loc: list[str | int] = [current_loc]
545        if value_index is not None:
546            new_loc.append(value_index)
547        if isinstance(orig_loc, tuple):
548            new_loc.extend(list(orig_loc))
549        elif isinstance(orig_loc, list):
550            new_loc.extend(orig_loc)
551        error["loc"] = tuple(new_loc)
ID_FIELD = FieldInfo(annotation=Union[str, NoneType], required=False, default_factory=<lambda>)
ID_TYPE = typing.Optional[str]
NAME_REGEX = '^[A-Za-z0-9 _-]+$'
NAME_FIELD = FieldInfo(annotation=str, required=True, description='A name for this entity.', metadata=[MinLen(min_length=1), MaxLen(max_length=120), _PydanticGeneralMetadata(pattern='^[A-Za-z0-9 _-]+$')])
SHORT_NAME_FIELD = FieldInfo(annotation=str, required=True, description='A name for this entity', metadata=[MinLen(min_length=1), MaxLen(max_length=32), _PydanticGeneralMetadata(pattern='^[A-Za-z0-9 _-]+$')])
def string_to_valid_name(name: str) -> str:
66def string_to_valid_name(name: str) -> str:
67    # Replace any character not allowed by NAME_REGEX with an underscore
68    valid_name = re.sub(r"[^A-Za-z0-9 _-]", "_", name)
69    # Replace consecutive underscores with a single underscore
70    valid_name = re.sub(r"_+", "_", valid_name)
71    # Remove leading and trailing underscores or whitespace
72    return valid_name.strip("_").strip()
class KilnBaseModel(pydantic.main.BaseModel):
 75class KilnBaseModel(BaseModel):
 76    """Base model for all Kiln data models with common functionality for persistence and versioning.
 77
 78    Attributes:
 79        v (int): Schema version number for migration support
 80        id (str): Unique identifier for the model instance
 81        path (Path): File system path where the model is stored
 82        created_at (datetime): Timestamp when the model was created
 83        created_by (str): User ID of the creator
 84    """
 85
 86    model_config = ConfigDict(validate_assignment=True)
 87
 88    v: int = Field(default=1)  # schema_version
 89    id: ID_TYPE = ID_FIELD
 90    path: Optional[Path] = Field(default=None)
 91    created_at: datetime = Field(default_factory=datetime.now)
 92    created_by: str = Field(default_factory=lambda: Config.shared().user_id)
 93
 94    _loaded_from_file: bool = False
 95
 96    @computed_field()
 97    def model_type(self) -> str:
 98        return self.type_name()
 99
100    # if changing the model name, should keep the original name here for parsing old files
101    @classmethod
102    def type_name(cls) -> str:
103        return snake_case(cls.__name__)
104
105    # used as /obj_folder/base_filename.kiln
106    @classmethod
107    def base_filename(cls) -> str:
108        return cls.type_name() + ".kiln"
109
110    @classmethod
111    def load_from_folder(cls: Type[T], folderPath: Path) -> T:
112        """Load a model instance from a folder using the default filename.
113
114        Args:
115            folderPath (Path): Directory path containing the model file
116
117        Returns:
118            T: Instance of the model
119        """
120        path = folderPath / cls.base_filename()
121        return cls.load_from_file(path)
122
123    @classmethod
124    def load_from_file(cls: Type[T], path: Path | str, readonly: bool = False) -> T:
125        """Load a model instance from a specific file path.
126
127        Args:
128            path (Path): Path to the model file
129            readonly (bool): If True, the model will be returned in readonly mode (cached instance, not a copy, not safe to mutate)
130
131        Returns:
132            T: Instance of the model
133
134        Raises:
135            ValueError: If the loaded model is not of the expected type or version
136            FileNotFoundError: If the file does not exist
137        """
138        if isinstance(path, str):
139            path = Path(path)
140        cached_model = ModelCache.shared().get_model(path, cls, readonly=readonly)
141        if cached_model is not None:
142            return cached_model
143        with open(path, "r", encoding="utf-8") as file:
144            # modified time of file for cache invalidation. From file descriptor so it's atomic w read.
145            mtime_ns = os.fstat(file.fileno()).st_mtime_ns
146            file_data = file.read()
147            parsed_json = json.loads(file_data)
148            m = cls.model_validate(parsed_json, context={"loading_from_file": True})
149            if not isinstance(m, cls):
150                raise ValueError(f"Loaded model is not of type {cls.__name__}")
151            m._loaded_from_file = True
152            file_data = None
153        m.path = path
154        if m.v > m.max_schema_version():
155            raise ValueError(
156                f"Cannot load from file because the schema version is higher than the current version. Upgrade kiln to the latest version. "
157                f"Class: {m.__class__.__name__}, id: {getattr(m, 'id', None)}, path: {path}, "
158                f"version: {m.v}, max version: {m.max_schema_version()}"
159            )
160        if parsed_json["model_type"] != cls.type_name():
161            raise ValueError(
162                f"Cannot load from file because the model type is incorrect. Expected {cls.type_name()}, got {parsed_json['model_type']}. "
163                f"Class: {m.__class__.__name__}, id: {getattr(m, 'id', None)}, path: {path}, "
164                f"version: {m.v}, max version: {m.max_schema_version()}"
165            )
166        ModelCache.shared().set_model(path, m, mtime_ns)
167        return m
168
169    def loaded_from_file(self, info: ValidationInfo | None = None) -> bool:
170        # Two methods of indicated it's loaded from file:
171        # 1) info.context.get("loading_from_file") -> During actual loading, before we can set _loaded_from_file
172        # 2) self._loaded_from_file -> After loading, set by the loader
173        if self.loading_from_file(info):
174            return True
175        return self._loaded_from_file
176
177    # indicates the model is currently being loaded from file (not mutating it after)
178    def loading_from_file(self, info: ValidationInfo | None = None) -> bool:
179        # info.context.get("loading_from_file") -> During actual loading, before we can set _loaded_from_file
180        if (
181            info is not None
182            and info.context is not None
183            and info.context.get("loading_from_file", False)
184        ):
185            return True
186        return False
187
188    def save_to_file(self) -> None:
189        """Save the model instance to a file.
190
191        Raises:
192            ValueError: If the path is not set
193        """
194        path = self.build_path()
195        if path is None:
196            raise ValueError(
197                f"Cannot save to file because 'path' is not set. Class: {self.__class__.__name__}, "
198                f"id: {getattr(self, 'id', None)}, path: {path}"
199            )
200        path.parent.mkdir(parents=True, exist_ok=True)
201        json_data = self.model_dump_json(indent=2, exclude={"path"})
202        with open(path, "w", encoding="utf-8") as file:
203            file.write(json_data)
204        # save the path so even if something like name changes, the file doesn't move
205        self.path = path
206        # We could save, but invalidating will trigger load on next use.
207        # This ensures everything in cache is loaded from disk, and the cache perfectly reflects what's on disk
208        ModelCache.shared().invalidate(path)
209
210    def delete(self) -> None:
211        if self.path is None:
212            raise ValueError("Cannot delete model because path is not set")
213        dir_path = self.path.parent if self.path.is_file() else self.path
214        if dir_path is None:
215            raise ValueError("Cannot delete model because path is not set")
216        shutil.rmtree(dir_path)
217        ModelCache.shared().invalidate(self.path)
218        self.path = None
219
220    def build_path(self) -> Path | None:
221        if self.path is not None:
222            return self.path
223        return None
224
225    # increment for breaking changes
226    def max_schema_version(self) -> int:
227        return 1

Base model for all Kiln data models with common functionality for persistence and versioning.

Attributes: v (int): Schema version number for migration support id (str): Unique identifier for the model instance path (Path): File system path where the model is stored created_at (datetime): Timestamp when the model was created created_by (str): User ID of the creator

model_config = {'validate_assignment': True}

Configuration for the model, should be a dictionary conforming to [ConfigDict][pydantic.config.ConfigDict].

v: int
id: Optional[str]
path: Optional[pathlib._local.Path]
created_at: datetime.datetime
created_by: str
model_type: str
96    @computed_field()
97    def model_type(self) -> str:
98        return self.type_name()
@classmethod
def type_name(cls) -> str:
101    @classmethod
102    def type_name(cls) -> str:
103        return snake_case(cls.__name__)
@classmethod
def base_filename(cls) -> str:
106    @classmethod
107    def base_filename(cls) -> str:
108        return cls.type_name() + ".kiln"
@classmethod
def load_from_folder(cls: Type[~T], folderPath: pathlib._local.Path) -> ~T:
110    @classmethod
111    def load_from_folder(cls: Type[T], folderPath: Path) -> T:
112        """Load a model instance from a folder using the default filename.
113
114        Args:
115            folderPath (Path): Directory path containing the model file
116
117        Returns:
118            T: Instance of the model
119        """
120        path = folderPath / cls.base_filename()
121        return cls.load_from_file(path)

Load a model instance from a folder using the default filename.

Args: folderPath (Path): Directory path containing the model file

Returns: T: Instance of the model

@classmethod
def load_from_file( cls: Type[~T], path: pathlib._local.Path | str, readonly: bool = False) -> ~T:
123    @classmethod
124    def load_from_file(cls: Type[T], path: Path | str, readonly: bool = False) -> T:
125        """Load a model instance from a specific file path.
126
127        Args:
128            path (Path): Path to the model file
129            readonly (bool): If True, the model will be returned in readonly mode (cached instance, not a copy, not safe to mutate)
130
131        Returns:
132            T: Instance of the model
133
134        Raises:
135            ValueError: If the loaded model is not of the expected type or version
136            FileNotFoundError: If the file does not exist
137        """
138        if isinstance(path, str):
139            path = Path(path)
140        cached_model = ModelCache.shared().get_model(path, cls, readonly=readonly)
141        if cached_model is not None:
142            return cached_model
143        with open(path, "r", encoding="utf-8") as file:
144            # modified time of file for cache invalidation. From file descriptor so it's atomic w read.
145            mtime_ns = os.fstat(file.fileno()).st_mtime_ns
146            file_data = file.read()
147            parsed_json = json.loads(file_data)
148            m = cls.model_validate(parsed_json, context={"loading_from_file": True})
149            if not isinstance(m, cls):
150                raise ValueError(f"Loaded model is not of type {cls.__name__}")
151            m._loaded_from_file = True
152            file_data = None
153        m.path = path
154        if m.v > m.max_schema_version():
155            raise ValueError(
156                f"Cannot load from file because the schema version is higher than the current version. Upgrade kiln to the latest version. "
157                f"Class: {m.__class__.__name__}, id: {getattr(m, 'id', None)}, path: {path}, "
158                f"version: {m.v}, max version: {m.max_schema_version()}"
159            )
160        if parsed_json["model_type"] != cls.type_name():
161            raise ValueError(
162                f"Cannot load from file because the model type is incorrect. Expected {cls.type_name()}, got {parsed_json['model_type']}. "
163                f"Class: {m.__class__.__name__}, id: {getattr(m, 'id', None)}, path: {path}, "
164                f"version: {m.v}, max version: {m.max_schema_version()}"
165            )
166        ModelCache.shared().set_model(path, m, mtime_ns)
167        return m

Load a model instance from a specific file path.

Args: path (Path): Path to the model file readonly (bool): If True, the model will be returned in readonly mode (cached instance, not a copy, not safe to mutate)

Returns: T: Instance of the model

Raises: ValueError: If the loaded model is not of the expected type or version FileNotFoundError: If the file does not exist

def loaded_from_file( self, info: pydantic_core.core_schema.ValidationInfo | None = None) -> bool:
169    def loaded_from_file(self, info: ValidationInfo | None = None) -> bool:
170        # Two methods of indicated it's loaded from file:
171        # 1) info.context.get("loading_from_file") -> During actual loading, before we can set _loaded_from_file
172        # 2) self._loaded_from_file -> After loading, set by the loader
173        if self.loading_from_file(info):
174            return True
175        return self._loaded_from_file
def loading_from_file( self, info: pydantic_core.core_schema.ValidationInfo | None = None) -> bool:
178    def loading_from_file(self, info: ValidationInfo | None = None) -> bool:
179        # info.context.get("loading_from_file") -> During actual loading, before we can set _loaded_from_file
180        if (
181            info is not None
182            and info.context is not None
183            and info.context.get("loading_from_file", False)
184        ):
185            return True
186        return False
def save_to_file(self) -> None:
188    def save_to_file(self) -> None:
189        """Save the model instance to a file.
190
191        Raises:
192            ValueError: If the path is not set
193        """
194        path = self.build_path()
195        if path is None:
196            raise ValueError(
197                f"Cannot save to file because 'path' is not set. Class: {self.__class__.__name__}, "
198                f"id: {getattr(self, 'id', None)}, path: {path}"
199            )
200        path.parent.mkdir(parents=True, exist_ok=True)
201        json_data = self.model_dump_json(indent=2, exclude={"path"})
202        with open(path, "w", encoding="utf-8") as file:
203            file.write(json_data)
204        # save the path so even if something like name changes, the file doesn't move
205        self.path = path
206        # We could save, but invalidating will trigger load on next use.
207        # This ensures everything in cache is loaded from disk, and the cache perfectly reflects what's on disk
208        ModelCache.shared().invalidate(path)

Save the model instance to a file.

Raises: ValueError: If the path is not set

def delete(self) -> None:
210    def delete(self) -> None:
211        if self.path is None:
212            raise ValueError("Cannot delete model because path is not set")
213        dir_path = self.path.parent if self.path.is_file() else self.path
214        if dir_path is None:
215            raise ValueError("Cannot delete model because path is not set")
216        shutil.rmtree(dir_path)
217        ModelCache.shared().invalidate(self.path)
218        self.path = None
def build_path(self) -> pathlib._local.Path | None:
220    def build_path(self) -> Path | None:
221        if self.path is not None:
222            return self.path
223        return None
def max_schema_version(self) -> int:
226    def max_schema_version(self) -> int:
227        return 1
def model_post_init(self: pydantic.main.BaseModel, context: Any, /) -> None:
384def init_private_attributes(self: BaseModel, context: Any, /) -> None:
385    """This function is meant to behave like a BaseModel method to initialise private attributes.
386
387    It takes context as an argument since that's what pydantic-core passes when calling it.
388
389    Args:
390        self: The BaseModel instance.
391        context: The context.
392    """
393    if getattr(self, '__pydantic_private__', None) is None:
394        pydantic_private = {}
395        for name, private_attr in self.__private_attributes__.items():
396            default = private_attr.get_default()
397            if default is not PydanticUndefined:
398                pydantic_private[name] = default
399        object_setattr(self, '__pydantic_private__', pydantic_private)

This function is meant to behave like a BaseModel method to initialise private attributes.

It takes context as an argument since that's what pydantic-core passes when calling it.

Args: self: The BaseModel instance. context: The context.

class KilnParentedModel(KilnBaseModel):
230class KilnParentedModel(KilnBaseModel, metaclass=ABCMeta):
231    """Base model for Kiln models that have a parent-child relationship. This base class is for child models.
232
233    This class provides functionality for managing hierarchical relationships between models,
234    including parent reference handling and file system organization.
235
236    Attributes:
237        parent (KilnBaseModel): Reference to the parent model instance. Not persisted, just in memory.
238    """
239
240    # Parent is an in memory only reference to parent. If it's set we use that. If not we'll try to load it from disk based on the path.
241    # We don't persist the parent reference to disk. See the accessors below for how we make it a clean api (parent accessor will lazy load from disk)
242    parent: Optional[KilnBaseModel] = Field(default=None, exclude=True)
243
244    def __getattribute__(self, name: str) -> Any:
245        if name == "parent":
246            return self.load_parent()
247        return super().__getattribute__(name)
248
249    def cached_parent(self) -> Optional[KilnBaseModel]:
250        return object.__getattribute__(self, "parent")
251
252    def load_parent(self) -> Optional[KilnBaseModel]:
253        """Get the parent model instance, loading it from disk if necessary.
254
255        Returns:
256            Optional[KilnBaseModel]: The parent model instance or None if not set
257        """
258        cached_parent = self.cached_parent()
259        if cached_parent is not None:
260            return cached_parent
261
262        # lazy load parent from path
263        if self.path is None:
264            return None
265        # Note: this only works with base_filename. If we every support custom names, we need to change this.
266        parent_path = (
267            self.path.parent.parent.parent
268            / self.__class__.parent_type().base_filename()
269        )
270        if parent_path is None:
271            return None
272        loaded_parent = self.__class__.parent_type().load_from_file(parent_path)
273        self.parent = loaded_parent
274        return loaded_parent
275
276    # Dynamically implemented by KilnParentModel method injection
277    @classmethod
278    def relationship_name(cls) -> str:
279        raise NotImplementedError("Relationship name must be implemented")
280
281    # Dynamically implemented by KilnParentModel method injection
282    @classmethod
283    def parent_type(cls) -> Type[KilnBaseModel]:
284        raise NotImplementedError("Parent type must be implemented")
285
286    @model_validator(mode="after")
287    def check_parent_type(self) -> Self:
288        cached_parent = self.cached_parent()
289        if cached_parent is not None:
290            expected_parent_type = self.__class__.parent_type()
291            if not isinstance(cached_parent, expected_parent_type):
292                raise ValueError(
293                    f"Parent must be of type {expected_parent_type}, but was {type(cached_parent)}"
294                )
295        return self
296
297    def build_child_dirname(self) -> Path:
298        # Default implementation for readable folder names.
299        # {id} - {name}/{type}.kiln
300        if self.id is None:
301            # consider generating an ID here. But if it's been cleared, we've already used this without one so raise for now.
302            raise ValueError("ID is not set - can not save or build path")
303        path = self.id
304        name = getattr(self, "name", None)
305        if name is not None:
306            path = f"{path} - {name[:32]}"
307        return Path(path)
308
309    def build_path(self) -> Path | None:
310        # if specifically loaded from an existing path, keep that no matter what
311        # this ensures the file structure is easy to use with git/version control
312        # and that changes to things like name (which impacts default path) don't leave dangling files
313        if self.path is not None:
314            return self.path
315        # Build a path under parent_folder/relationship/file.kiln
316        if self.parent is None:
317            return None
318        parent_path = self.parent.build_path()
319        if parent_path is None:
320            return None
321        parent_folder = parent_path.parent
322        if parent_folder is None:
323            return None
324        return (
325            parent_folder
326            / self.__class__.relationship_name()
327            / self.build_child_dirname()
328            / self.__class__.base_filename()
329        )
330
331    @classmethod
332    def iterate_children_paths_of_parent_path(cls: Type[PT], parent_path: Path | None):
333        if parent_path is None:
334            # children are disk based. If not saved, they don't exist
335            return []
336
337        # Determine the parent folder
338        if parent_path.is_file():
339            parent_folder = parent_path.parent
340        else:
341            parent_folder = parent_path
342
343        parent = cls.parent_type().load_from_file(parent_path)
344        if parent is None:
345            raise ValueError("Parent must be set to load children")
346
347        # Ignore type error: this is abstract base class, but children must implement relationship_name
348        relationship_folder = parent_folder / Path(cls.relationship_name())  # type: ignore
349
350        if not relationship_folder.exists() or not relationship_folder.is_dir():
351            return []
352
353        # Collect all /relationship/{id}/{base_filename.kiln} files in the relationship folder
354        # manual code instead of glob for performance (5x speedup over glob)
355
356        base_filename = cls.base_filename()
357        # Iterate through immediate subdirectories using scandir for better performance
358        # Benchmark: scandir is 10x faster than glob, so worth the extra code
359        with os.scandir(relationship_folder) as entries:
360            for entry in entries:
361                if not entry.is_dir():
362                    continue
363
364                child_file = Path(entry.path) / base_filename
365                if child_file.is_file():
366                    yield child_file
367
368    @classmethod
369    def all_children_of_parent_path(
370        cls: Type[PT], parent_path: Path | None, readonly: bool = False
371    ) -> list[PT]:
372        children = []
373        for child_path in cls.iterate_children_paths_of_parent_path(parent_path):
374            item = cls.load_from_file(child_path, readonly=readonly)
375            children.append(item)
376        return children
377
378    @classmethod
379    def from_id_and_parent_path(
380        cls: Type[PT], id: str, parent_path: Path | None
381    ) -> PT | None:
382        """
383        Fast search by ID using the cache. Avoids the model_copy overhead on all but the exact match.
384
385        Uses cache so still slow on first load.
386        """
387        if parent_path is None:
388            return None
389
390        # Note: we're using the in-file ID. We could make this faster using the path-ID if this becomes perf bottleneck, but it's better to have 1 source of truth.
391        for child_path in cls.iterate_children_paths_of_parent_path(parent_path):
392            child_id = ModelCache.shared().get_model_id(child_path, cls)
393            if child_id == id:
394                return cls.load_from_file(child_path)
395            if child_id is None:
396                child = cls.load_from_file(child_path)
397                if child.id == id:
398                    return child
399        return None

Base model for Kiln models that have a parent-child relationship. This base class is for child models.

This class provides functionality for managing hierarchical relationships between models, including parent reference handling and file system organization.

Attributes: parent (KilnBaseModel): Reference to the parent model instance. Not persisted, just in memory.

parent: Optional[KilnBaseModel]
def cached_parent(self) -> Optional[KilnBaseModel]:
249    def cached_parent(self) -> Optional[KilnBaseModel]:
250        return object.__getattribute__(self, "parent")
def load_parent(self) -> Optional[KilnBaseModel]:
252    def load_parent(self) -> Optional[KilnBaseModel]:
253        """Get the parent model instance, loading it from disk if necessary.
254
255        Returns:
256            Optional[KilnBaseModel]: The parent model instance or None if not set
257        """
258        cached_parent = self.cached_parent()
259        if cached_parent is not None:
260            return cached_parent
261
262        # lazy load parent from path
263        if self.path is None:
264            return None
265        # Note: this only works with base_filename. If we every support custom names, we need to change this.
266        parent_path = (
267            self.path.parent.parent.parent
268            / self.__class__.parent_type().base_filename()
269        )
270        if parent_path is None:
271            return None
272        loaded_parent = self.__class__.parent_type().load_from_file(parent_path)
273        self.parent = loaded_parent
274        return loaded_parent

Get the parent model instance, loading it from disk if necessary.

Returns: Optional[KilnBaseModel]: The parent model instance or None if not set

@classmethod
def relationship_name(cls) -> str:
277    @classmethod
278    def relationship_name(cls) -> str:
279        raise NotImplementedError("Relationship name must be implemented")
@classmethod
def parent_type(cls) -> Type[KilnBaseModel]:
282    @classmethod
283    def parent_type(cls) -> Type[KilnBaseModel]:
284        raise NotImplementedError("Parent type must be implemented")
@model_validator(mode='after')
def check_parent_type(self) -> Self:
286    @model_validator(mode="after")
287    def check_parent_type(self) -> Self:
288        cached_parent = self.cached_parent()
289        if cached_parent is not None:
290            expected_parent_type = self.__class__.parent_type()
291            if not isinstance(cached_parent, expected_parent_type):
292                raise ValueError(
293                    f"Parent must be of type {expected_parent_type}, but was {type(cached_parent)}"
294                )
295        return self
def build_child_dirname(self) -> pathlib._local.Path:
297    def build_child_dirname(self) -> Path:
298        # Default implementation for readable folder names.
299        # {id} - {name}/{type}.kiln
300        if self.id is None:
301            # consider generating an ID here. But if it's been cleared, we've already used this without one so raise for now.
302            raise ValueError("ID is not set - can not save or build path")
303        path = self.id
304        name = getattr(self, "name", None)
305        if name is not None:
306            path = f"{path} - {name[:32]}"
307        return Path(path)
def build_path(self) -> pathlib._local.Path | None:
309    def build_path(self) -> Path | None:
310        # if specifically loaded from an existing path, keep that no matter what
311        # this ensures the file structure is easy to use with git/version control
312        # and that changes to things like name (which impacts default path) don't leave dangling files
313        if self.path is not None:
314            return self.path
315        # Build a path under parent_folder/relationship/file.kiln
316        if self.parent is None:
317            return None
318        parent_path = self.parent.build_path()
319        if parent_path is None:
320            return None
321        parent_folder = parent_path.parent
322        if parent_folder is None:
323            return None
324        return (
325            parent_folder
326            / self.__class__.relationship_name()
327            / self.build_child_dirname()
328            / self.__class__.base_filename()
329        )
@classmethod
def iterate_children_paths_of_parent_path(cls: Type[~PT], parent_path: pathlib._local.Path | None):
331    @classmethod
332    def iterate_children_paths_of_parent_path(cls: Type[PT], parent_path: Path | None):
333        if parent_path is None:
334            # children are disk based. If not saved, they don't exist
335            return []
336
337        # Determine the parent folder
338        if parent_path.is_file():
339            parent_folder = parent_path.parent
340        else:
341            parent_folder = parent_path
342
343        parent = cls.parent_type().load_from_file(parent_path)
344        if parent is None:
345            raise ValueError("Parent must be set to load children")
346
347        # Ignore type error: this is abstract base class, but children must implement relationship_name
348        relationship_folder = parent_folder / Path(cls.relationship_name())  # type: ignore
349
350        if not relationship_folder.exists() or not relationship_folder.is_dir():
351            return []
352
353        # Collect all /relationship/{id}/{base_filename.kiln} files in the relationship folder
354        # manual code instead of glob for performance (5x speedup over glob)
355
356        base_filename = cls.base_filename()
357        # Iterate through immediate subdirectories using scandir for better performance
358        # Benchmark: scandir is 10x faster than glob, so worth the extra code
359        with os.scandir(relationship_folder) as entries:
360            for entry in entries:
361                if not entry.is_dir():
362                    continue
363
364                child_file = Path(entry.path) / base_filename
365                if child_file.is_file():
366                    yield child_file
@classmethod
def all_children_of_parent_path( cls: Type[~PT], parent_path: pathlib._local.Path | None, readonly: bool = False) -> list[~PT]:
368    @classmethod
369    def all_children_of_parent_path(
370        cls: Type[PT], parent_path: Path | None, readonly: bool = False
371    ) -> list[PT]:
372        children = []
373        for child_path in cls.iterate_children_paths_of_parent_path(parent_path):
374            item = cls.load_from_file(child_path, readonly=readonly)
375            children.append(item)
376        return children
@classmethod
def from_id_and_parent_path( cls: Type[~PT], id: str, parent_path: pathlib._local.Path | None) -> Optional[~PT]:
378    @classmethod
379    def from_id_and_parent_path(
380        cls: Type[PT], id: str, parent_path: Path | None
381    ) -> PT | None:
382        """
383        Fast search by ID using the cache. Avoids the model_copy overhead on all but the exact match.
384
385        Uses cache so still slow on first load.
386        """
387        if parent_path is None:
388            return None
389
390        # Note: we're using the in-file ID. We could make this faster using the path-ID if this becomes perf bottleneck, but it's better to have 1 source of truth.
391        for child_path in cls.iterate_children_paths_of_parent_path(parent_path):
392            child_id = ModelCache.shared().get_model_id(child_path, cls)
393            if child_id == id:
394                return cls.load_from_file(child_path)
395            if child_id is None:
396                child = cls.load_from_file(child_path)
397                if child.id == id:
398                    return child
399        return None

Fast search by ID using the cache. Avoids the model_copy overhead on all but the exact match.

Uses cache so still slow on first load.

model_config = {'validate_assignment': True}

Configuration for the model, should be a dictionary conforming to [ConfigDict][pydantic.config.ConfigDict].

def model_post_init(self: pydantic.main.BaseModel, context: Any, /) -> None:
122                    def wrapped_model_post_init(self: BaseModel, context: Any, /) -> None:
123                        """We need to both initialize private attributes and call the user-defined model_post_init
124                        method.
125                        """
126                        init_private_attributes(self, context)
127                        original_model_post_init(self, context)

We need to both initialize private attributes and call the user-defined model_post_init method.

class KilnParentModel(KilnBaseModel):
404class KilnParentModel(KilnBaseModel, metaclass=ABCMeta):
405    """Base model for Kiln models that can have child models.
406
407    This class provides functionality for managing collections of child models and their persistence.
408    Child relationships must be defined using the parent_of parameter in the class definition.
409
410    Args:
411        parent_of (Dict[str, Type[KilnParentedModel]]): Mapping of relationship names to child model types
412    """
413
414    @classmethod
415    def _create_child_method(
416        cls, relationship_name: str, child_class: Type[KilnParentedModel]
417    ):
418        def child_method(self, readonly: bool = False) -> list[child_class]:
419            return child_class.all_children_of_parent_path(self.path, readonly=readonly)
420
421        child_method.__name__ = relationship_name
422        child_method.__annotations__ = {"return": List[child_class]}
423        setattr(cls, relationship_name, child_method)
424
425    @classmethod
426    def _create_parent_methods(
427        cls, targetCls: Type[KilnParentedModel], relationship_name: str
428    ):
429        def parent_class_method() -> Type[KilnParentModel]:
430            return cls
431
432        parent_class_method.__name__ = "parent_type"
433        parent_class_method.__annotations__ = {"return": Type[KilnParentModel]}
434        setattr(targetCls, "parent_type", parent_class_method)
435
436        def relationship_name_method() -> str:
437            return relationship_name
438
439        relationship_name_method.__name__ = "relationship_name"
440        relationship_name_method.__annotations__ = {"return": str}
441        setattr(targetCls, "relationship_name", relationship_name_method)
442
443    @classmethod
444    def __init_subclass__(cls, parent_of: Dict[str, Type[KilnParentedModel]], **kwargs):
445        super().__init_subclass__(**kwargs)
446        cls._parent_of = parent_of
447        for relationship_name, child_class in parent_of.items():
448            cls._create_child_method(relationship_name, child_class)
449            cls._create_parent_methods(child_class, relationship_name)
450
451    @classmethod
452    def validate_and_save_with_subrelations(
453        cls,
454        data: Dict[str, Any],
455        path: Path | None = None,
456        parent: KilnBaseModel | None = None,
457    ):
458        """Validate and save a model instance along with all its nested child relationships.
459
460        Args:
461            data (Dict[str, Any]): Model data including child relationships
462            path (Path, optional): Path where the model should be saved
463            parent (KilnBaseModel, optional): Parent model instance for parented models
464
465        Returns:
466            KilnParentModel: The validated and saved model instance
467
468        Raises:
469            ValidationError: If validation fails for the model or any of its children
470        """
471        # Validate first, then save. Don't want error half way through, and partly persisted
472        # TODO P2: save to tmp dir, then move atomically. But need to merge directories so later.
473        cls._validate_nested(data, save=False, path=path, parent=parent)
474        instance = cls._validate_nested(data, save=True, path=path, parent=parent)
475        return instance
476
477    @classmethod
478    def _validate_nested(
479        cls,
480        data: Dict[str, Any],
481        save: bool = False,
482        parent: KilnBaseModel | None = None,
483        path: Path | None = None,
484    ):
485        # Collect all validation errors so we can report them all at once
486        validation_errors = []
487
488        try:
489            instance = cls.model_validate(data)
490            if path is not None:
491                instance.path = path
492            if parent is not None and isinstance(instance, KilnParentedModel):
493                instance.parent = parent
494            if save:
495                instance.save_to_file()
496        except ValidationError as e:
497            instance = None
498            for suberror in e.errors():
499                validation_errors.append(suberror)
500
501        for key, value_list in data.items():
502            if key in cls._parent_of:
503                parent_type = cls._parent_of[key]
504                if not isinstance(value_list, list):
505                    raise ValueError(
506                        f"Expected a list for {key}, but got {type(value_list)}"
507                    )
508                for value_index, value in enumerate(value_list):
509                    try:
510                        if issubclass(parent_type, KilnParentModel):
511                            kwargs = {"data": value, "save": save}
512                            if instance is not None:
513                                kwargs["parent"] = instance
514                            parent_type._validate_nested(**kwargs)
515                        elif issubclass(parent_type, KilnParentedModel):
516                            # Root node
517                            subinstance = parent_type.model_validate(value)
518                            if instance is not None:
519                                subinstance.parent = instance
520                            if save:
521                                subinstance.save_to_file()
522                        else:
523                            raise ValueError(
524                                f"Invalid type {parent_type}. Should be KilnBaseModel based."
525                            )
526                    except ValidationError as e:
527                        for suberror in e.errors():
528                            cls._append_loc(suberror, key, value_index)
529                            validation_errors.append(suberror)
530
531        if len(validation_errors) > 0:
532            raise ValidationError.from_exception_data(
533                title=f"Validation failed for {cls.__name__}",
534                line_errors=validation_errors,
535                input_type="json",
536            )
537
538        return instance
539
540    @classmethod
541    def _append_loc(
542        cls, error: ErrorDetails, current_loc: str, value_index: int | None = None
543    ):
544        orig_loc = error["loc"] if "loc" in error else None
545        new_loc: list[str | int] = [current_loc]
546        if value_index is not None:
547            new_loc.append(value_index)
548        if isinstance(orig_loc, tuple):
549            new_loc.extend(list(orig_loc))
550        elif isinstance(orig_loc, list):
551            new_loc.extend(orig_loc)
552        error["loc"] = tuple(new_loc)

Base model for Kiln models that can have child models.

This class provides functionality for managing collections of child models and their persistence. Child relationships must be defined using the parent_of parameter in the class definition.

Args: parent_of (Dict[str, Type[KilnParentedModel]]): Mapping of relationship names to child model types

@classmethod
def validate_and_save_with_subrelations( cls, data: Dict[str, Any], path: pathlib._local.Path | None = None, parent: KilnBaseModel | None = None):
451    @classmethod
452    def validate_and_save_with_subrelations(
453        cls,
454        data: Dict[str, Any],
455        path: Path | None = None,
456        parent: KilnBaseModel | None = None,
457    ):
458        """Validate and save a model instance along with all its nested child relationships.
459
460        Args:
461            data (Dict[str, Any]): Model data including child relationships
462            path (Path, optional): Path where the model should be saved
463            parent (KilnBaseModel, optional): Parent model instance for parented models
464
465        Returns:
466            KilnParentModel: The validated and saved model instance
467
468        Raises:
469            ValidationError: If validation fails for the model or any of its children
470        """
471        # Validate first, then save. Don't want error half way through, and partly persisted
472        # TODO P2: save to tmp dir, then move atomically. But need to merge directories so later.
473        cls._validate_nested(data, save=False, path=path, parent=parent)
474        instance = cls._validate_nested(data, save=True, path=path, parent=parent)
475        return instance

Validate and save a model instance along with all its nested child relationships.

Args: data (Dict[str, Any]): Model data including child relationships path (Path, optional): Path where the model should be saved parent (KilnBaseModel, optional): Parent model instance for parented models

Returns: KilnParentModel: The validated and saved model instance

Raises: ValidationError: If validation fails for the model or any of its children

model_config = {'validate_assignment': True}

Configuration for the model, should be a dictionary conforming to [ConfigDict][pydantic.config.ConfigDict].

def model_post_init(self: pydantic.main.BaseModel, context: Any, /) -> None:
122                    def wrapped_model_post_init(self: BaseModel, context: Any, /) -> None:
123                        """We need to both initialize private attributes and call the user-defined model_post_init
124                        method.
125                        """
126                        init_private_attributes(self, context)
127                        original_model_post_init(self, context)

We need to both initialize private attributes and call the user-defined model_post_init method.