Coverage for /Users/davegaeddert/Developer/dropseed/plain/plain-models/plain/models/backends/sqlite3/schema.py: 18%

212 statements  

« prev     ^ index     » next       coverage.py v7.6.9, created at 2024-12-23 11:16 -0600

1import copy 

2from decimal import Decimal 

3 

4from plain.models.backends.base.schema import BaseDatabaseSchemaEditor 

5from plain.models.backends.ddl_references import Statement 

6from plain.models.backends.utils import strip_quotes 

7from plain.models.constraints import UniqueConstraint 

8from plain.models.db import NotSupportedError 

9from plain.models.transaction import atomic 

10from plain.packages.registry import Packages 

11 

12 

13class DatabaseSchemaEditor(BaseDatabaseSchemaEditor): 

14 sql_delete_table = "DROP TABLE %(table)s" 

15 sql_create_fk = None 

16 sql_create_inline_fk = ( 

17 "REFERENCES %(to_table)s (%(to_column)s) DEFERRABLE INITIALLY DEFERRED" 

18 ) 

19 sql_create_column_inline_fk = sql_create_inline_fk 

20 sql_delete_column = "ALTER TABLE %(table)s DROP COLUMN %(column)s" 

21 sql_create_unique = "CREATE UNIQUE INDEX %(name)s ON %(table)s (%(columns)s)" 

22 sql_delete_unique = "DROP INDEX %(name)s" 

23 

24 def __enter__(self): 

25 # Some SQLite schema alterations need foreign key constraints to be 

26 # disabled. Enforce it here for the duration of the schema edition. 

27 if not self.connection.disable_constraint_checking(): 

28 raise NotSupportedError( 

29 "SQLite schema editor cannot be used while foreign key " 

30 "constraint checks are enabled. Make sure to disable them " 

31 "before entering a transaction.atomic() context because " 

32 "SQLite does not support disabling them in the middle of " 

33 "a multi-statement transaction." 

34 ) 

35 return super().__enter__() 

36 

37 def __exit__(self, exc_type, exc_value, traceback): 

38 self.connection.check_constraints() 

39 super().__exit__(exc_type, exc_value, traceback) 

40 self.connection.enable_constraint_checking() 

41 

42 def quote_value(self, value): 

43 # The backend "mostly works" without this function and there are use 

44 # cases for compiling Python without the sqlite3 libraries (e.g. 

45 # security hardening). 

46 try: 

47 import sqlite3 

48 

49 value = sqlite3.adapt(value) 

50 except ImportError: 

51 pass 

52 except sqlite3.ProgrammingError: 

53 pass 

54 # Manual emulation of SQLite parameter quoting 

55 if isinstance(value, bool): 

56 return str(int(value)) 

57 elif isinstance(value, Decimal | float | int): 

58 return str(value) 

59 elif isinstance(value, str): 

60 return "'{}'".format(value.replace("'", "''")) 

61 elif value is None: 

62 return "NULL" 

63 elif isinstance(value, bytes | bytearray | memoryview): 

64 # Bytes are only allowed for BLOB fields, encoded as string 

65 # literals containing hexadecimal data and preceded by a single "X" 

66 # character. 

67 return f"X'{value.hex()}'" 

68 else: 

69 raise ValueError( 

70 f"Cannot quote parameter value {value!r} of type {type(value)}" 

71 ) 

72 

73 def prepare_default(self, value): 

74 return self.quote_value(value) 

75 

76 def _is_referenced_by_fk_constraint( 

77 self, table_name, column_name=None, ignore_self=False 

78 ): 

79 """ 

80 Return whether or not the provided table name is referenced by another 

81 one. If `column_name` is specified, only references pointing to that 

82 column are considered. If `ignore_self` is True, self-referential 

83 constraints are ignored. 

84 """ 

85 with self.connection.cursor() as cursor: 

86 for other_table in self.connection.introspection.get_table_list(cursor): 

87 if ignore_self and other_table.name == table_name: 

88 continue 

89 relations = self.connection.introspection.get_relations( 

90 cursor, other_table.name 

91 ) 

92 for constraint_column, constraint_table in relations.values(): 

93 if constraint_table == table_name and ( 

94 column_name is None or constraint_column == column_name 

95 ): 

96 return True 

97 return False 

98 

99 def alter_db_table( 

100 self, model, old_db_table, new_db_table, disable_constraints=True 

101 ): 

102 if ( 

103 not self.connection.features.supports_atomic_references_rename 

104 and disable_constraints 

105 and self._is_referenced_by_fk_constraint(old_db_table) 

106 ): 

107 if self.connection.in_atomic_block: 

108 raise NotSupportedError( 

109 f"Renaming the {old_db_table!r} table while in a transaction is not " 

110 "supported on SQLite < 3.26 because it would break referential " 

111 "integrity. Try adding `atomic = False` to the Migration class." 

112 ) 

113 self.connection.enable_constraint_checking() 

114 super().alter_db_table(model, old_db_table, new_db_table) 

115 self.connection.disable_constraint_checking() 

116 else: 

117 super().alter_db_table(model, old_db_table, new_db_table) 

118 

119 def alter_field(self, model, old_field, new_field, strict=False): 

120 if not self._field_should_be_altered(old_field, new_field): 

121 return 

122 old_field_name = old_field.name 

123 table_name = model._meta.db_table 

124 _, old_column_name = old_field.get_attname_column() 

125 if ( 

126 new_field.name != old_field_name 

127 and not self.connection.features.supports_atomic_references_rename 

128 and self._is_referenced_by_fk_constraint( 

129 table_name, old_column_name, ignore_self=True 

130 ) 

131 ): 

132 if self.connection.in_atomic_block: 

133 raise NotSupportedError( 

134 f"Renaming the {model._meta.db_table!r}.{old_field_name!r} column while in a transaction is not " 

135 "supported on SQLite < 3.26 because it would break referential " 

136 "integrity. Try adding `atomic = False` to the Migration class." 

137 ) 

138 with atomic(self.connection.alias): 

139 super().alter_field(model, old_field, new_field, strict=strict) 

140 # Follow SQLite's documented procedure for performing changes 

141 # that don't affect the on-disk content. 

142 # https://sqlite.org/lang_altertable.html#otheralter 

143 with self.connection.cursor() as cursor: 

144 schema_version = cursor.execute("PRAGMA schema_version").fetchone()[ 

145 0 

146 ] 

147 cursor.execute("PRAGMA writable_schema = 1") 

148 references_template = f' REFERENCES "{table_name}" ("%s") ' 

149 new_column_name = new_field.get_attname_column()[1] 

150 search = references_template % old_column_name 

151 replacement = references_template % new_column_name 

152 cursor.execute( 

153 "UPDATE sqlite_master SET sql = replace(sql, %s, %s)", 

154 (search, replacement), 

155 ) 

156 cursor.execute("PRAGMA schema_version = %d" % (schema_version + 1)) 

157 cursor.execute("PRAGMA writable_schema = 0") 

158 # The integrity check will raise an exception and rollback 

159 # the transaction if the sqlite_master updates corrupt the 

160 # database. 

161 cursor.execute("PRAGMA integrity_check") 

162 # Perform a VACUUM to refresh the database representation from 

163 # the sqlite_master table. 

164 with self.connection.cursor() as cursor: 

165 cursor.execute("VACUUM") 

166 else: 

167 super().alter_field(model, old_field, new_field, strict=strict) 

168 

169 def _remake_table( 

170 self, model, create_field=None, delete_field=None, alter_fields=None 

171 ): 

172 """ 

173 Shortcut to transform a model from old_model into new_model 

174 

175 This follows the correct procedure to perform non-rename or column 

176 addition operations based on SQLite's documentation 

177 

178 https://www.sqlite.org/lang_altertable.html#caution 

179 

180 The essential steps are: 

181 1. Create a table with the updated definition called "new__app_model" 

182 2. Copy the data from the existing "app_model" table to the new table 

183 3. Drop the "app_model" table 

184 4. Rename the "new__app_model" table to "app_model" 

185 5. Restore any index of the previous "app_model" table. 

186 """ 

187 

188 # Self-referential fields must be recreated rather than copied from 

189 # the old model to ensure their remote_field.field_name doesn't refer 

190 # to an altered field. 

191 def is_self_referential(f): 

192 return f.is_relation and f.remote_field.model is model 

193 

194 # Work out the new fields dict / mapping 

195 body = { 

196 f.name: f.clone() if is_self_referential(f) else f 

197 for f in model._meta.local_concrete_fields 

198 } 

199 # Since mapping might mix column names and default values, 

200 # its values must be already quoted. 

201 mapping = { 

202 f.column: self.quote_name(f.column) 

203 for f in model._meta.local_concrete_fields 

204 } 

205 # If any of the new or altered fields is introducing a new PK, 

206 # remove the old one 

207 restore_pk_field = None 

208 alter_fields = alter_fields or [] 

209 if getattr(create_field, "primary_key", False) or any( 

210 getattr(new_field, "primary_key", False) for _, new_field in alter_fields 

211 ): 

212 for name, field in list(body.items()): 

213 if field.primary_key and not any( 

214 # Do not remove the old primary key when an altered field 

215 # that introduces a primary key is the same field. 

216 name == new_field.name 

217 for _, new_field in alter_fields 

218 ): 

219 field.primary_key = False 

220 restore_pk_field = field 

221 if field.auto_created: 

222 del body[name] 

223 del mapping[field.column] 

224 # Add in any created fields 

225 if create_field: 

226 body[create_field.name] = create_field 

227 # Choose a default and insert it into the copy map 

228 if not create_field.many_to_many and create_field.concrete: 

229 mapping[create_field.column] = self.prepare_default( 

230 self.effective_default(create_field), 

231 ) 

232 # Add in any altered fields 

233 for alter_field in alter_fields: 

234 old_field, new_field = alter_field 

235 body.pop(old_field.name, None) 

236 mapping.pop(old_field.column, None) 

237 body[new_field.name] = new_field 

238 if old_field.null and not new_field.null: 

239 case_sql = f"coalesce({self.quote_name(old_field.column)}, {self.prepare_default(self.effective_default(new_field))})" 

240 mapping[new_field.column] = case_sql 

241 else: 

242 mapping[new_field.column] = self.quote_name(old_field.column) 

243 # Remove any deleted fields 

244 if delete_field: 

245 del body[delete_field.name] 

246 del mapping[delete_field.column] 

247 # Remove any implicit M2M tables 

248 if ( 

249 delete_field.many_to_many 

250 and delete_field.remote_field.through._meta.auto_created 

251 ): 

252 return self.delete_model(delete_field.remote_field.through) 

253 # Work inside a new app registry 

254 packages = Packages() 

255 

256 indexes = model._meta.indexes 

257 if delete_field: 

258 indexes = [ 

259 index for index in indexes if delete_field.name not in index.fields 

260 ] 

261 

262 constraints = list(model._meta.constraints) 

263 

264 # Provide isolated instances of the fields to the new model body so 

265 # that the existing model's internals aren't interfered with when 

266 # the dummy model is constructed. 

267 body_copy = copy.deepcopy(body) 

268 

269 # Construct a new model with the new fields to allow self referential 

270 # primary key to resolve to. This model won't ever be materialized as a 

271 # table and solely exists for foreign key reference resolution purposes. 

272 # This wouldn't be required if the schema editor was operating on model 

273 # states instead of rendered models. 

274 meta_contents = { 

275 "package_label": model._meta.package_label, 

276 "db_table": model._meta.db_table, 

277 "indexes": indexes, 

278 "constraints": constraints, 

279 "packages": packages, 

280 } 

281 meta = type("Meta", (), meta_contents) 

282 body_copy["Meta"] = meta 

283 body_copy["__module__"] = model.__module__ 

284 type(model._meta.object_name, model.__bases__, body_copy) 

285 

286 # Construct a model with a renamed table name. 

287 body_copy = copy.deepcopy(body) 

288 meta_contents = { 

289 "package_label": model._meta.package_label, 

290 "db_table": f"new__{strip_quotes(model._meta.db_table)}", 

291 "indexes": indexes, 

292 "constraints": constraints, 

293 "packages": packages, 

294 } 

295 meta = type("Meta", (), meta_contents) 

296 body_copy["Meta"] = meta 

297 body_copy["__module__"] = model.__module__ 

298 new_model = type(f"New{model._meta.object_name}", model.__bases__, body_copy) 

299 

300 # Create a new table with the updated schema. 

301 self.create_model(new_model) 

302 

303 # Copy data from the old table into the new table 

304 self.execute( 

305 "INSERT INTO {} ({}) SELECT {} FROM {}".format( 

306 self.quote_name(new_model._meta.db_table), 

307 ", ".join(self.quote_name(x) for x in mapping), 

308 ", ".join(mapping.values()), 

309 self.quote_name(model._meta.db_table), 

310 ) 

311 ) 

312 

313 # Delete the old table to make way for the new 

314 self.delete_model(model, handle_autom2m=False) 

315 

316 # Rename the new table to take way for the old 

317 self.alter_db_table( 

318 new_model, 

319 new_model._meta.db_table, 

320 model._meta.db_table, 

321 disable_constraints=False, 

322 ) 

323 

324 # Run deferred SQL on correct table 

325 for sql in self.deferred_sql: 

326 self.execute(sql) 

327 self.deferred_sql = [] 

328 # Fix any PK-removed field 

329 if restore_pk_field: 

330 restore_pk_field.primary_key = True 

331 

332 def delete_model(self, model, handle_autom2m=True): 

333 if handle_autom2m: 

334 super().delete_model(model) 

335 else: 

336 # Delete the table (and only that) 

337 self.execute( 

338 self.sql_delete_table 

339 % { 

340 "table": self.quote_name(model._meta.db_table), 

341 } 

342 ) 

343 # Remove all deferred statements referencing the deleted table. 

344 for sql in list(self.deferred_sql): 

345 if isinstance(sql, Statement) and sql.references_table( 

346 model._meta.db_table 

347 ): 

348 self.deferred_sql.remove(sql) 

349 

350 def add_field(self, model, field): 

351 """Create a field on a model.""" 

352 # Special-case implicit M2M tables. 

353 if field.many_to_many and field.remote_field.through._meta.auto_created: 

354 self.create_model(field.remote_field.through) 

355 elif ( 

356 # Primary keys and unique fields are not supported in ALTER TABLE 

357 # ADD COLUMN. 

358 field.primary_key 

359 or field.unique 

360 or 

361 # Fields with default values cannot by handled by ALTER TABLE ADD 

362 # COLUMN statement because DROP DEFAULT is not supported in 

363 # ALTER TABLE. 

364 not field.null 

365 or self.effective_default(field) is not None 

366 ): 

367 self._remake_table(model, create_field=field) 

368 else: 

369 super().add_field(model, field) 

370 

371 def remove_field(self, model, field): 

372 """ 

373 Remove a field from a model. Usually involves deleting a column, 

374 but for M2Ms may involve deleting a table. 

375 """ 

376 # M2M fields are a special case 

377 if field.many_to_many: 

378 # For implicit M2M tables, delete the auto-created table 

379 if field.remote_field.through._meta.auto_created: 

380 self.delete_model(field.remote_field.through) 

381 # For explicit "through" M2M fields, do nothing 

382 elif ( 

383 self.connection.features.can_alter_table_drop_column 

384 # Primary keys, unique fields, indexed fields, and foreign keys are 

385 # not supported in ALTER TABLE DROP COLUMN. 

386 and not field.primary_key 

387 and not field.unique 

388 and not field.db_index 

389 and not (field.remote_field and field.db_constraint) 

390 ): 

391 super().remove_field(model, field) 

392 # For everything else, remake. 

393 else: 

394 # It might not actually have a column behind it 

395 if field.db_parameters(connection=self.connection)["type"] is None: 

396 return 

397 self._remake_table(model, delete_field=field) 

398 

399 def _alter_field( 

400 self, 

401 model, 

402 old_field, 

403 new_field, 

404 old_type, 

405 new_type, 

406 old_db_params, 

407 new_db_params, 

408 strict=False, 

409 ): 

410 """Perform a "physical" (non-ManyToMany) field update.""" 

411 # Use "ALTER TABLE ... RENAME COLUMN" if only the column name 

412 # changed and there aren't any constraints. 

413 if ( 

414 self.connection.features.can_alter_table_rename_column 

415 and old_field.column != new_field.column 

416 and self.column_sql(model, old_field) == self.column_sql(model, new_field) 

417 and not ( 

418 old_field.remote_field 

419 and old_field.db_constraint 

420 or new_field.remote_field 

421 and new_field.db_constraint 

422 ) 

423 ): 

424 return self.execute( 

425 self._rename_field_sql( 

426 model._meta.db_table, old_field, new_field, new_type 

427 ) 

428 ) 

429 # Alter by remaking table 

430 self._remake_table(model, alter_fields=[(old_field, new_field)]) 

431 # Rebuild tables with FKs pointing to this field. 

432 old_collation = old_db_params.get("collation") 

433 new_collation = new_db_params.get("collation") 

434 if new_field.unique and ( 

435 old_type != new_type or old_collation != new_collation 

436 ): 

437 related_models = set() 

438 opts = new_field.model._meta 

439 for remote_field in opts.related_objects: 

440 # Ignore self-relationship since the table was already rebuilt. 

441 if remote_field.related_model == model: 

442 continue 

443 if not remote_field.many_to_many: 

444 if remote_field.field_name == new_field.name: 

445 related_models.add(remote_field.related_model) 

446 elif new_field.primary_key and remote_field.through._meta.auto_created: 

447 related_models.add(remote_field.through) 

448 if new_field.primary_key: 

449 for many_to_many in opts.many_to_many: 

450 # Ignore self-relationship since the table was already rebuilt. 

451 if many_to_many.related_model == model: 

452 continue 

453 if many_to_many.remote_field.through._meta.auto_created: 

454 related_models.add(many_to_many.remote_field.through) 

455 for related_model in related_models: 

456 self._remake_table(related_model) 

457 

458 def _alter_many_to_many(self, model, old_field, new_field, strict): 

459 """Alter M2Ms to repoint their to= endpoints.""" 

460 if ( 

461 old_field.remote_field.through._meta.db_table 

462 == new_field.remote_field.through._meta.db_table 

463 ): 

464 # The field name didn't change, but some options did, so we have to 

465 # propagate this altering. 

466 self._remake_table( 

467 old_field.remote_field.through, 

468 alter_fields=[ 

469 ( 

470 # The field that points to the target model is needed, 

471 # so that table can be remade with the new m2m field - 

472 # this is m2m_reverse_field_name(). 

473 old_field.remote_field.through._meta.get_field( 

474 old_field.m2m_reverse_field_name() 

475 ), 

476 new_field.remote_field.through._meta.get_field( 

477 new_field.m2m_reverse_field_name() 

478 ), 

479 ), 

480 ( 

481 # The field that points to the model itself is needed, 

482 # so that table can be remade with the new self field - 

483 # this is m2m_field_name(). 

484 old_field.remote_field.through._meta.get_field( 

485 old_field.m2m_field_name() 

486 ), 

487 new_field.remote_field.through._meta.get_field( 

488 new_field.m2m_field_name() 

489 ), 

490 ), 

491 ], 

492 ) 

493 return 

494 

495 # Make a new through table 

496 self.create_model(new_field.remote_field.through) 

497 # Copy the data across 

498 self.execute( 

499 "INSERT INTO {} ({}) SELECT {} FROM {}".format( 

500 self.quote_name(new_field.remote_field.through._meta.db_table), 

501 ", ".join( 

502 [ 

503 "id", 

504 new_field.m2m_column_name(), 

505 new_field.m2m_reverse_name(), 

506 ] 

507 ), 

508 ", ".join( 

509 [ 

510 "id", 

511 old_field.m2m_column_name(), 

512 old_field.m2m_reverse_name(), 

513 ] 

514 ), 

515 self.quote_name(old_field.remote_field.through._meta.db_table), 

516 ) 

517 ) 

518 # Delete the old through table 

519 self.delete_model(old_field.remote_field.through) 

520 

521 def add_constraint(self, model, constraint): 

522 if isinstance(constraint, UniqueConstraint) and ( 

523 constraint.condition 

524 or constraint.contains_expressions 

525 or constraint.include 

526 or constraint.deferrable 

527 ): 

528 super().add_constraint(model, constraint) 

529 else: 

530 self._remake_table(model) 

531 

532 def remove_constraint(self, model, constraint): 

533 if isinstance(constraint, UniqueConstraint) and ( 

534 constraint.condition 

535 or constraint.contains_expressions 

536 or constraint.include 

537 or constraint.deferrable 

538 ): 

539 super().remove_constraint(model, constraint) 

540 else: 

541 self._remake_table(model) 

542 

543 def _collate_sql(self, collation): 

544 return "COLLATE " + collation