Coverage for /Users/davegaeddert/Developer/dropseed/plain/plain-models/plain/models/backends/sqlite3/schema.py: 18%
212 statements
« prev ^ index » next coverage.py v7.6.9, created at 2024-12-23 11:16 -0600
« prev ^ index » next coverage.py v7.6.9, created at 2024-12-23 11:16 -0600
1import copy
2from decimal import Decimal
4from plain.models.backends.base.schema import BaseDatabaseSchemaEditor
5from plain.models.backends.ddl_references import Statement
6from plain.models.backends.utils import strip_quotes
7from plain.models.constraints import UniqueConstraint
8from plain.models.db import NotSupportedError
9from plain.models.transaction import atomic
10from plain.packages.registry import Packages
13class DatabaseSchemaEditor(BaseDatabaseSchemaEditor):
14 sql_delete_table = "DROP TABLE %(table)s"
15 sql_create_fk = None
16 sql_create_inline_fk = (
17 "REFERENCES %(to_table)s (%(to_column)s) DEFERRABLE INITIALLY DEFERRED"
18 )
19 sql_create_column_inline_fk = sql_create_inline_fk
20 sql_delete_column = "ALTER TABLE %(table)s DROP COLUMN %(column)s"
21 sql_create_unique = "CREATE UNIQUE INDEX %(name)s ON %(table)s (%(columns)s)"
22 sql_delete_unique = "DROP INDEX %(name)s"
24 def __enter__(self):
25 # Some SQLite schema alterations need foreign key constraints to be
26 # disabled. Enforce it here for the duration of the schema edition.
27 if not self.connection.disable_constraint_checking():
28 raise NotSupportedError(
29 "SQLite schema editor cannot be used while foreign key "
30 "constraint checks are enabled. Make sure to disable them "
31 "before entering a transaction.atomic() context because "
32 "SQLite does not support disabling them in the middle of "
33 "a multi-statement transaction."
34 )
35 return super().__enter__()
37 def __exit__(self, exc_type, exc_value, traceback):
38 self.connection.check_constraints()
39 super().__exit__(exc_type, exc_value, traceback)
40 self.connection.enable_constraint_checking()
42 def quote_value(self, value):
43 # The backend "mostly works" without this function and there are use
44 # cases for compiling Python without the sqlite3 libraries (e.g.
45 # security hardening).
46 try:
47 import sqlite3
49 value = sqlite3.adapt(value)
50 except ImportError:
51 pass
52 except sqlite3.ProgrammingError:
53 pass
54 # Manual emulation of SQLite parameter quoting
55 if isinstance(value, bool):
56 return str(int(value))
57 elif isinstance(value, Decimal | float | int):
58 return str(value)
59 elif isinstance(value, str):
60 return "'{}'".format(value.replace("'", "''"))
61 elif value is None:
62 return "NULL"
63 elif isinstance(value, bytes | bytearray | memoryview):
64 # Bytes are only allowed for BLOB fields, encoded as string
65 # literals containing hexadecimal data and preceded by a single "X"
66 # character.
67 return f"X'{value.hex()}'"
68 else:
69 raise ValueError(
70 f"Cannot quote parameter value {value!r} of type {type(value)}"
71 )
73 def prepare_default(self, value):
74 return self.quote_value(value)
76 def _is_referenced_by_fk_constraint(
77 self, table_name, column_name=None, ignore_self=False
78 ):
79 """
80 Return whether or not the provided table name is referenced by another
81 one. If `column_name` is specified, only references pointing to that
82 column are considered. If `ignore_self` is True, self-referential
83 constraints are ignored.
84 """
85 with self.connection.cursor() as cursor:
86 for other_table in self.connection.introspection.get_table_list(cursor):
87 if ignore_self and other_table.name == table_name:
88 continue
89 relations = self.connection.introspection.get_relations(
90 cursor, other_table.name
91 )
92 for constraint_column, constraint_table in relations.values():
93 if constraint_table == table_name and (
94 column_name is None or constraint_column == column_name
95 ):
96 return True
97 return False
99 def alter_db_table(
100 self, model, old_db_table, new_db_table, disable_constraints=True
101 ):
102 if (
103 not self.connection.features.supports_atomic_references_rename
104 and disable_constraints
105 and self._is_referenced_by_fk_constraint(old_db_table)
106 ):
107 if self.connection.in_atomic_block:
108 raise NotSupportedError(
109 f"Renaming the {old_db_table!r} table while in a transaction is not "
110 "supported on SQLite < 3.26 because it would break referential "
111 "integrity. Try adding `atomic = False` to the Migration class."
112 )
113 self.connection.enable_constraint_checking()
114 super().alter_db_table(model, old_db_table, new_db_table)
115 self.connection.disable_constraint_checking()
116 else:
117 super().alter_db_table(model, old_db_table, new_db_table)
119 def alter_field(self, model, old_field, new_field, strict=False):
120 if not self._field_should_be_altered(old_field, new_field):
121 return
122 old_field_name = old_field.name
123 table_name = model._meta.db_table
124 _, old_column_name = old_field.get_attname_column()
125 if (
126 new_field.name != old_field_name
127 and not self.connection.features.supports_atomic_references_rename
128 and self._is_referenced_by_fk_constraint(
129 table_name, old_column_name, ignore_self=True
130 )
131 ):
132 if self.connection.in_atomic_block:
133 raise NotSupportedError(
134 f"Renaming the {model._meta.db_table!r}.{old_field_name!r} column while in a transaction is not "
135 "supported on SQLite < 3.26 because it would break referential "
136 "integrity. Try adding `atomic = False` to the Migration class."
137 )
138 with atomic(self.connection.alias):
139 super().alter_field(model, old_field, new_field, strict=strict)
140 # Follow SQLite's documented procedure for performing changes
141 # that don't affect the on-disk content.
142 # https://sqlite.org/lang_altertable.html#otheralter
143 with self.connection.cursor() as cursor:
144 schema_version = cursor.execute("PRAGMA schema_version").fetchone()[
145 0
146 ]
147 cursor.execute("PRAGMA writable_schema = 1")
148 references_template = f' REFERENCES "{table_name}" ("%s") '
149 new_column_name = new_field.get_attname_column()[1]
150 search = references_template % old_column_name
151 replacement = references_template % new_column_name
152 cursor.execute(
153 "UPDATE sqlite_master SET sql = replace(sql, %s, %s)",
154 (search, replacement),
155 )
156 cursor.execute("PRAGMA schema_version = %d" % (schema_version + 1))
157 cursor.execute("PRAGMA writable_schema = 0")
158 # The integrity check will raise an exception and rollback
159 # the transaction if the sqlite_master updates corrupt the
160 # database.
161 cursor.execute("PRAGMA integrity_check")
162 # Perform a VACUUM to refresh the database representation from
163 # the sqlite_master table.
164 with self.connection.cursor() as cursor:
165 cursor.execute("VACUUM")
166 else:
167 super().alter_field(model, old_field, new_field, strict=strict)
169 def _remake_table(
170 self, model, create_field=None, delete_field=None, alter_fields=None
171 ):
172 """
173 Shortcut to transform a model from old_model into new_model
175 This follows the correct procedure to perform non-rename or column
176 addition operations based on SQLite's documentation
178 https://www.sqlite.org/lang_altertable.html#caution
180 The essential steps are:
181 1. Create a table with the updated definition called "new__app_model"
182 2. Copy the data from the existing "app_model" table to the new table
183 3. Drop the "app_model" table
184 4. Rename the "new__app_model" table to "app_model"
185 5. Restore any index of the previous "app_model" table.
186 """
188 # Self-referential fields must be recreated rather than copied from
189 # the old model to ensure their remote_field.field_name doesn't refer
190 # to an altered field.
191 def is_self_referential(f):
192 return f.is_relation and f.remote_field.model is model
194 # Work out the new fields dict / mapping
195 body = {
196 f.name: f.clone() if is_self_referential(f) else f
197 for f in model._meta.local_concrete_fields
198 }
199 # Since mapping might mix column names and default values,
200 # its values must be already quoted.
201 mapping = {
202 f.column: self.quote_name(f.column)
203 for f in model._meta.local_concrete_fields
204 }
205 # If any of the new or altered fields is introducing a new PK,
206 # remove the old one
207 restore_pk_field = None
208 alter_fields = alter_fields or []
209 if getattr(create_field, "primary_key", False) or any(
210 getattr(new_field, "primary_key", False) for _, new_field in alter_fields
211 ):
212 for name, field in list(body.items()):
213 if field.primary_key and not any(
214 # Do not remove the old primary key when an altered field
215 # that introduces a primary key is the same field.
216 name == new_field.name
217 for _, new_field in alter_fields
218 ):
219 field.primary_key = False
220 restore_pk_field = field
221 if field.auto_created:
222 del body[name]
223 del mapping[field.column]
224 # Add in any created fields
225 if create_field:
226 body[create_field.name] = create_field
227 # Choose a default and insert it into the copy map
228 if not create_field.many_to_many and create_field.concrete:
229 mapping[create_field.column] = self.prepare_default(
230 self.effective_default(create_field),
231 )
232 # Add in any altered fields
233 for alter_field in alter_fields:
234 old_field, new_field = alter_field
235 body.pop(old_field.name, None)
236 mapping.pop(old_field.column, None)
237 body[new_field.name] = new_field
238 if old_field.null and not new_field.null:
239 case_sql = f"coalesce({self.quote_name(old_field.column)}, {self.prepare_default(self.effective_default(new_field))})"
240 mapping[new_field.column] = case_sql
241 else:
242 mapping[new_field.column] = self.quote_name(old_field.column)
243 # Remove any deleted fields
244 if delete_field:
245 del body[delete_field.name]
246 del mapping[delete_field.column]
247 # Remove any implicit M2M tables
248 if (
249 delete_field.many_to_many
250 and delete_field.remote_field.through._meta.auto_created
251 ):
252 return self.delete_model(delete_field.remote_field.through)
253 # Work inside a new app registry
254 packages = Packages()
256 indexes = model._meta.indexes
257 if delete_field:
258 indexes = [
259 index for index in indexes if delete_field.name not in index.fields
260 ]
262 constraints = list(model._meta.constraints)
264 # Provide isolated instances of the fields to the new model body so
265 # that the existing model's internals aren't interfered with when
266 # the dummy model is constructed.
267 body_copy = copy.deepcopy(body)
269 # Construct a new model with the new fields to allow self referential
270 # primary key to resolve to. This model won't ever be materialized as a
271 # table and solely exists for foreign key reference resolution purposes.
272 # This wouldn't be required if the schema editor was operating on model
273 # states instead of rendered models.
274 meta_contents = {
275 "package_label": model._meta.package_label,
276 "db_table": model._meta.db_table,
277 "indexes": indexes,
278 "constraints": constraints,
279 "packages": packages,
280 }
281 meta = type("Meta", (), meta_contents)
282 body_copy["Meta"] = meta
283 body_copy["__module__"] = model.__module__
284 type(model._meta.object_name, model.__bases__, body_copy)
286 # Construct a model with a renamed table name.
287 body_copy = copy.deepcopy(body)
288 meta_contents = {
289 "package_label": model._meta.package_label,
290 "db_table": f"new__{strip_quotes(model._meta.db_table)}",
291 "indexes": indexes,
292 "constraints": constraints,
293 "packages": packages,
294 }
295 meta = type("Meta", (), meta_contents)
296 body_copy["Meta"] = meta
297 body_copy["__module__"] = model.__module__
298 new_model = type(f"New{model._meta.object_name}", model.__bases__, body_copy)
300 # Create a new table with the updated schema.
301 self.create_model(new_model)
303 # Copy data from the old table into the new table
304 self.execute(
305 "INSERT INTO {} ({}) SELECT {} FROM {}".format(
306 self.quote_name(new_model._meta.db_table),
307 ", ".join(self.quote_name(x) for x in mapping),
308 ", ".join(mapping.values()),
309 self.quote_name(model._meta.db_table),
310 )
311 )
313 # Delete the old table to make way for the new
314 self.delete_model(model, handle_autom2m=False)
316 # Rename the new table to take way for the old
317 self.alter_db_table(
318 new_model,
319 new_model._meta.db_table,
320 model._meta.db_table,
321 disable_constraints=False,
322 )
324 # Run deferred SQL on correct table
325 for sql in self.deferred_sql:
326 self.execute(sql)
327 self.deferred_sql = []
328 # Fix any PK-removed field
329 if restore_pk_field:
330 restore_pk_field.primary_key = True
332 def delete_model(self, model, handle_autom2m=True):
333 if handle_autom2m:
334 super().delete_model(model)
335 else:
336 # Delete the table (and only that)
337 self.execute(
338 self.sql_delete_table
339 % {
340 "table": self.quote_name(model._meta.db_table),
341 }
342 )
343 # Remove all deferred statements referencing the deleted table.
344 for sql in list(self.deferred_sql):
345 if isinstance(sql, Statement) and sql.references_table(
346 model._meta.db_table
347 ):
348 self.deferred_sql.remove(sql)
350 def add_field(self, model, field):
351 """Create a field on a model."""
352 # Special-case implicit M2M tables.
353 if field.many_to_many and field.remote_field.through._meta.auto_created:
354 self.create_model(field.remote_field.through)
355 elif (
356 # Primary keys and unique fields are not supported in ALTER TABLE
357 # ADD COLUMN.
358 field.primary_key
359 or field.unique
360 or
361 # Fields with default values cannot by handled by ALTER TABLE ADD
362 # COLUMN statement because DROP DEFAULT is not supported in
363 # ALTER TABLE.
364 not field.null
365 or self.effective_default(field) is not None
366 ):
367 self._remake_table(model, create_field=field)
368 else:
369 super().add_field(model, field)
371 def remove_field(self, model, field):
372 """
373 Remove a field from a model. Usually involves deleting a column,
374 but for M2Ms may involve deleting a table.
375 """
376 # M2M fields are a special case
377 if field.many_to_many:
378 # For implicit M2M tables, delete the auto-created table
379 if field.remote_field.through._meta.auto_created:
380 self.delete_model(field.remote_field.through)
381 # For explicit "through" M2M fields, do nothing
382 elif (
383 self.connection.features.can_alter_table_drop_column
384 # Primary keys, unique fields, indexed fields, and foreign keys are
385 # not supported in ALTER TABLE DROP COLUMN.
386 and not field.primary_key
387 and not field.unique
388 and not field.db_index
389 and not (field.remote_field and field.db_constraint)
390 ):
391 super().remove_field(model, field)
392 # For everything else, remake.
393 else:
394 # It might not actually have a column behind it
395 if field.db_parameters(connection=self.connection)["type"] is None:
396 return
397 self._remake_table(model, delete_field=field)
399 def _alter_field(
400 self,
401 model,
402 old_field,
403 new_field,
404 old_type,
405 new_type,
406 old_db_params,
407 new_db_params,
408 strict=False,
409 ):
410 """Perform a "physical" (non-ManyToMany) field update."""
411 # Use "ALTER TABLE ... RENAME COLUMN" if only the column name
412 # changed and there aren't any constraints.
413 if (
414 self.connection.features.can_alter_table_rename_column
415 and old_field.column != new_field.column
416 and self.column_sql(model, old_field) == self.column_sql(model, new_field)
417 and not (
418 old_field.remote_field
419 and old_field.db_constraint
420 or new_field.remote_field
421 and new_field.db_constraint
422 )
423 ):
424 return self.execute(
425 self._rename_field_sql(
426 model._meta.db_table, old_field, new_field, new_type
427 )
428 )
429 # Alter by remaking table
430 self._remake_table(model, alter_fields=[(old_field, new_field)])
431 # Rebuild tables with FKs pointing to this field.
432 old_collation = old_db_params.get("collation")
433 new_collation = new_db_params.get("collation")
434 if new_field.unique and (
435 old_type != new_type or old_collation != new_collation
436 ):
437 related_models = set()
438 opts = new_field.model._meta
439 for remote_field in opts.related_objects:
440 # Ignore self-relationship since the table was already rebuilt.
441 if remote_field.related_model == model:
442 continue
443 if not remote_field.many_to_many:
444 if remote_field.field_name == new_field.name:
445 related_models.add(remote_field.related_model)
446 elif new_field.primary_key and remote_field.through._meta.auto_created:
447 related_models.add(remote_field.through)
448 if new_field.primary_key:
449 for many_to_many in opts.many_to_many:
450 # Ignore self-relationship since the table was already rebuilt.
451 if many_to_many.related_model == model:
452 continue
453 if many_to_many.remote_field.through._meta.auto_created:
454 related_models.add(many_to_many.remote_field.through)
455 for related_model in related_models:
456 self._remake_table(related_model)
458 def _alter_many_to_many(self, model, old_field, new_field, strict):
459 """Alter M2Ms to repoint their to= endpoints."""
460 if (
461 old_field.remote_field.through._meta.db_table
462 == new_field.remote_field.through._meta.db_table
463 ):
464 # The field name didn't change, but some options did, so we have to
465 # propagate this altering.
466 self._remake_table(
467 old_field.remote_field.through,
468 alter_fields=[
469 (
470 # The field that points to the target model is needed,
471 # so that table can be remade with the new m2m field -
472 # this is m2m_reverse_field_name().
473 old_field.remote_field.through._meta.get_field(
474 old_field.m2m_reverse_field_name()
475 ),
476 new_field.remote_field.through._meta.get_field(
477 new_field.m2m_reverse_field_name()
478 ),
479 ),
480 (
481 # The field that points to the model itself is needed,
482 # so that table can be remade with the new self field -
483 # this is m2m_field_name().
484 old_field.remote_field.through._meta.get_field(
485 old_field.m2m_field_name()
486 ),
487 new_field.remote_field.through._meta.get_field(
488 new_field.m2m_field_name()
489 ),
490 ),
491 ],
492 )
493 return
495 # Make a new through table
496 self.create_model(new_field.remote_field.through)
497 # Copy the data across
498 self.execute(
499 "INSERT INTO {} ({}) SELECT {} FROM {}".format(
500 self.quote_name(new_field.remote_field.through._meta.db_table),
501 ", ".join(
502 [
503 "id",
504 new_field.m2m_column_name(),
505 new_field.m2m_reverse_name(),
506 ]
507 ),
508 ", ".join(
509 [
510 "id",
511 old_field.m2m_column_name(),
512 old_field.m2m_reverse_name(),
513 ]
514 ),
515 self.quote_name(old_field.remote_field.through._meta.db_table),
516 )
517 )
518 # Delete the old through table
519 self.delete_model(old_field.remote_field.through)
521 def add_constraint(self, model, constraint):
522 if isinstance(constraint, UniqueConstraint) and (
523 constraint.condition
524 or constraint.contains_expressions
525 or constraint.include
526 or constraint.deferrable
527 ):
528 super().add_constraint(model, constraint)
529 else:
530 self._remake_table(model)
532 def remove_constraint(self, model, constraint):
533 if isinstance(constraint, UniqueConstraint) and (
534 constraint.condition
535 or constraint.contains_expressions
536 or constraint.include
537 or constraint.deferrable
538 ):
539 super().remove_constraint(model, constraint)
540 else:
541 self._remake_table(model)
543 def _collate_sql(self, collation):
544 return "COLLATE " + collation