Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1"""Provide the 'autogenerate' feature which can produce migration operations 

2automatically.""" 

3 

4import contextlib 

5 

6from sqlalchemy import inspect 

7 

8from . import compare 

9from . import render 

10from .. import util 

11from ..operations import ops 

12 

13 

14def compare_metadata(context, metadata): 

15 """Compare a database schema to that given in a 

16 :class:`~sqlalchemy.schema.MetaData` instance. 

17 

18 The database connection is presented in the context 

19 of a :class:`.MigrationContext` object, which 

20 provides database connectivity as well as optional 

21 comparison functions to use for datatypes and 

22 server defaults - see the "autogenerate" arguments 

23 at :meth:`.EnvironmentContext.configure` 

24 for details on these. 

25 

26 The return format is a list of "diff" directives, 

27 each representing individual differences:: 

28 

29 from alembic.migration import MigrationContext 

30 from alembic.autogenerate import compare_metadata 

31 from sqlalchemy.schema import SchemaItem 

32 from sqlalchemy.types import TypeEngine 

33 from sqlalchemy import (create_engine, MetaData, Column, 

34 Integer, String, Table) 

35 import pprint 

36 

37 engine = create_engine("sqlite://") 

38 

39 engine.execute(''' 

40 create table foo ( 

41 id integer not null primary key, 

42 old_data varchar, 

43 x integer 

44 )''') 

45 

46 engine.execute(''' 

47 create table bar ( 

48 data varchar 

49 )''') 

50 

51 metadata = MetaData() 

52 Table('foo', metadata, 

53 Column('id', Integer, primary_key=True), 

54 Column('data', Integer), 

55 Column('x', Integer, nullable=False) 

56 ) 

57 Table('bat', metadata, 

58 Column('info', String) 

59 ) 

60 

61 mc = MigrationContext.configure(engine.connect()) 

62 

63 diff = compare_metadata(mc, metadata) 

64 pprint.pprint(diff, indent=2, width=20) 

65 

66 Output:: 

67 

68 [ ( 'add_table', 

69 Table('bat', MetaData(bind=None), 

70 Column('info', String(), table=<bat>), schema=None)), 

71 ( 'remove_table', 

72 Table(u'bar', MetaData(bind=None), 

73 Column(u'data', VARCHAR(), table=<bar>), schema=None)), 

74 ( 'add_column', 

75 None, 

76 'foo', 

77 Column('data', Integer(), table=<foo>)), 

78 ( 'remove_column', 

79 None, 

80 'foo', 

81 Column(u'old_data', VARCHAR(), table=None)), 

82 [ ( 'modify_nullable', 

83 None, 

84 'foo', 

85 u'x', 

86 { 'existing_server_default': None, 

87 'existing_type': INTEGER()}, 

88 True, 

89 False)]] 

90 

91 

92 :param context: a :class:`.MigrationContext` 

93 instance. 

94 :param metadata: a :class:`~sqlalchemy.schema.MetaData` 

95 instance. 

96 

97 .. seealso:: 

98 

99 :func:`.produce_migrations` - produces a :class:`.MigrationScript` 

100 structure based on metadata comparison. 

101 

102 """ 

103 

104 migration_script = produce_migrations(context, metadata) 

105 return migration_script.upgrade_ops.as_diffs() 

106 

107 

108def produce_migrations(context, metadata): 

109 """Produce a :class:`.MigrationScript` structure based on schema 

110 comparison. 

111 

112 This function does essentially what :func:`.compare_metadata` does, 

113 but then runs the resulting list of diffs to produce the full 

114 :class:`.MigrationScript` object. For an example of what this looks like, 

115 see the example in :ref:`customizing_revision`. 

116 

117 .. versionadded:: 0.8.0 

118 

119 .. seealso:: 

120 

121 :func:`.compare_metadata` - returns more fundamental "diff" 

122 data from comparing a schema. 

123 

124 """ 

125 

126 autogen_context = AutogenContext(context, metadata=metadata) 

127 

128 migration_script = ops.MigrationScript( 

129 rev_id=None, 

130 upgrade_ops=ops.UpgradeOps([]), 

131 downgrade_ops=ops.DowngradeOps([]), 

132 ) 

133 

134 compare._populate_migration_script(autogen_context, migration_script) 

135 

136 return migration_script 

137 

138 

139def render_python_code( 

140 up_or_down_op, 

141 sqlalchemy_module_prefix="sa.", 

142 alembic_module_prefix="op.", 

143 render_as_batch=False, 

144 imports=(), 

145 render_item=None, 

146 migration_context=None, 

147): 

148 """Render Python code given an :class:`.UpgradeOps` or 

149 :class:`.DowngradeOps` object. 

150 

151 This is a convenience function that can be used to test the 

152 autogenerate output of a user-defined :class:`.MigrationScript` structure. 

153 

154 """ 

155 opts = { 

156 "sqlalchemy_module_prefix": sqlalchemy_module_prefix, 

157 "alembic_module_prefix": alembic_module_prefix, 

158 "render_item": render_item, 

159 "render_as_batch": render_as_batch, 

160 } 

161 

162 if migration_context is None: 

163 from ..runtime.migration import MigrationContext 

164 from sqlalchemy.engine.default import DefaultDialect 

165 

166 migration_context = MigrationContext.configure( 

167 dialect=DefaultDialect() 

168 ) 

169 

170 autogen_context = AutogenContext(migration_context, opts=opts) 

171 autogen_context.imports = set(imports) 

172 return render._indent( 

173 render._render_cmd_body(up_or_down_op, autogen_context) 

174 ) 

175 

176 

177def _render_migration_diffs(context, template_args): 

178 """legacy, used by test_autogen_composition at the moment""" 

179 

180 autogen_context = AutogenContext(context) 

181 

182 upgrade_ops = ops.UpgradeOps([]) 

183 compare._produce_net_changes(autogen_context, upgrade_ops) 

184 

185 migration_script = ops.MigrationScript( 

186 rev_id=None, 

187 upgrade_ops=upgrade_ops, 

188 downgrade_ops=upgrade_ops.reverse(), 

189 ) 

190 

191 render._render_python_into_templatevars( 

192 autogen_context, migration_script, template_args 

193 ) 

194 

195 

196class AutogenContext(object): 

197 """Maintains configuration and state that's specific to an 

198 autogenerate operation.""" 

199 

200 metadata = None 

201 """The :class:`~sqlalchemy.schema.MetaData` object 

202 representing the destination. 

203 

204 This object is the one that is passed within ``env.py`` 

205 to the :paramref:`.EnvironmentContext.configure.target_metadata` 

206 parameter. It represents the structure of :class:`.Table` and other 

207 objects as stated in the current database model, and represents the 

208 destination structure for the database being examined. 

209 

210 While the :class:`~sqlalchemy.schema.MetaData` object is primarily 

211 known as a collection of :class:`~sqlalchemy.schema.Table` objects, 

212 it also has an :attr:`~sqlalchemy.schema.MetaData.info` dictionary 

213 that may be used by end-user schemes to store additional schema-level 

214 objects that are to be compared in custom autogeneration schemes. 

215 

216 """ 

217 

218 connection = None 

219 """The :class:`~sqlalchemy.engine.base.Connection` object currently 

220 connected to the database backend being compared. 

221 

222 This is obtained from the :attr:`.MigrationContext.bind` and is 

223 utimately set up in the ``env.py`` script. 

224 

225 """ 

226 

227 dialect = None 

228 """The :class:`~sqlalchemy.engine.Dialect` object currently in use. 

229 

230 This is normally obtained from the 

231 :attr:`~sqlalchemy.engine.base.Connection.dialect` attribute. 

232 

233 """ 

234 

235 imports = None 

236 """A ``set()`` which contains string Python import directives. 

237 

238 The directives are to be rendered into the ``${imports}`` section 

239 of a script template. The set is normally empty and can be modified 

240 within hooks such as the 

241 :paramref:`.EnvironmentContext.configure.render_item` hook. 

242 

243 .. versionadded:: 0.8.3 

244 

245 .. seealso:: 

246 

247 :ref:`autogen_render_types` 

248 

249 """ 

250 

251 migration_context = None 

252 """The :class:`.MigrationContext` established by the ``env.py`` script.""" 

253 

254 def __init__( 

255 self, migration_context, metadata=None, opts=None, autogenerate=True 

256 ): 

257 

258 if ( 

259 autogenerate 

260 and migration_context is not None 

261 and migration_context.as_sql 

262 ): 

263 raise util.CommandError( 

264 "autogenerate can't use as_sql=True as it prevents querying " 

265 "the database for schema information" 

266 ) 

267 

268 if opts is None: 

269 opts = migration_context.opts 

270 

271 self.metadata = metadata = ( 

272 opts.get("target_metadata", None) if metadata is None else metadata 

273 ) 

274 

275 if ( 

276 autogenerate 

277 and metadata is None 

278 and migration_context is not None 

279 and migration_context.script is not None 

280 ): 

281 raise util.CommandError( 

282 "Can't proceed with --autogenerate option; environment " 

283 "script %s does not provide " 

284 "a MetaData object or sequence of objects to the context." 

285 % (migration_context.script.env_py_location) 

286 ) 

287 

288 include_symbol = opts.get("include_symbol", None) 

289 include_object = opts.get("include_object", None) 

290 

291 object_filters = [] 

292 if include_symbol: 

293 

294 def include_symbol_filter( 

295 object_, name, type_, reflected, compare_to 

296 ): 

297 if type_ == "table": 

298 return include_symbol(name, object_.schema) 

299 else: 

300 return True 

301 

302 object_filters.append(include_symbol_filter) 

303 if include_object: 

304 object_filters.append(include_object) 

305 

306 self._object_filters = object_filters 

307 

308 self.migration_context = migration_context 

309 if self.migration_context is not None: 

310 self.connection = self.migration_context.bind 

311 self.dialect = self.migration_context.dialect 

312 

313 self.imports = set() 

314 self.opts = opts 

315 self._has_batch = False 

316 

317 @util.memoized_property 

318 def inspector(self): 

319 return inspect(self.connection) 

320 

321 @contextlib.contextmanager 

322 def _within_batch(self): 

323 self._has_batch = True 

324 yield 

325 self._has_batch = False 

326 

327 def run_filters(self, object_, name, type_, reflected, compare_to): 

328 """Run the context's object filters and return True if the targets 

329 should be part of the autogenerate operation. 

330 

331 This method should be run for every kind of object encountered within 

332 an autogenerate operation, giving the environment the chance 

333 to filter what objects should be included in the comparison. 

334 The filters here are produced directly via the 

335 :paramref:`.EnvironmentContext.configure.include_object` 

336 and :paramref:`.EnvironmentContext.configure.include_symbol` 

337 functions, if present. 

338 

339 """ 

340 for fn in self._object_filters: 

341 if not fn(object_, name, type_, reflected, compare_to): 

342 return False 

343 else: 

344 return True 

345 

346 @util.memoized_property 

347 def sorted_tables(self): 

348 """Return an aggregate of the :attr:`.MetaData.sorted_tables` collection(s). 

349 

350 For a sequence of :class:`.MetaData` objects, this 

351 concatenates the :attr:`.MetaData.sorted_tables` collection 

352 for each individual :class:`.MetaData` in the order of the 

353 sequence. It does **not** collate the sorted tables collections. 

354 

355 .. versionadded:: 0.9.0 

356 

357 """ 

358 result = [] 

359 for m in util.to_list(self.metadata): 

360 result.extend(m.sorted_tables) 

361 return result 

362 

363 @util.memoized_property 

364 def table_key_to_table(self): 

365 """Return an aggregate of the :attr:`.MetaData.tables` dictionaries. 

366 

367 The :attr:`.MetaData.tables` collection is a dictionary of table key 

368 to :class:`.Table`; this method aggregates the dictionary across 

369 multiple :class:`.MetaData` objects into one dictionary. 

370 

371 Duplicate table keys are **not** supported; if two :class:`.MetaData` 

372 objects contain the same table key, an exception is raised. 

373 

374 .. versionadded:: 0.9.0 

375 

376 """ 

377 result = {} 

378 for m in util.to_list(self.metadata): 

379 intersect = set(result).intersection(set(m.tables)) 

380 if intersect: 

381 raise ValueError( 

382 "Duplicate table keys across multiple " 

383 "MetaData objects: %s" 

384 % (", ".join('"%s"' % key for key in sorted(intersect))) 

385 ) 

386 

387 result.update(m.tables) 

388 return result 

389 

390 

391class RevisionContext(object): 

392 """Maintains configuration and state that's specific to a revision 

393 file generation operation.""" 

394 

395 def __init__( 

396 self, 

397 config, 

398 script_directory, 

399 command_args, 

400 process_revision_directives=None, 

401 ): 

402 self.config = config 

403 self.script_directory = script_directory 

404 self.command_args = command_args 

405 self.process_revision_directives = process_revision_directives 

406 self.template_args = { 

407 "config": config # Let templates use config for 

408 # e.g. multiple databases 

409 } 

410 self.generated_revisions = [self._default_revision()] 

411 

412 def _to_script(self, migration_script): 

413 template_args = {} 

414 for k, v in self.template_args.items(): 

415 template_args.setdefault(k, v) 

416 

417 if getattr(migration_script, "_needs_render", False): 

418 autogen_context = self._last_autogen_context 

419 

420 # clear out existing imports if we are doing multiple 

421 # renders 

422 autogen_context.imports = set() 

423 if migration_script.imports: 

424 autogen_context.imports.update(migration_script.imports) 

425 render._render_python_into_templatevars( 

426 autogen_context, migration_script, template_args 

427 ) 

428 

429 return self.script_directory.generate_revision( 

430 migration_script.rev_id, 

431 migration_script.message, 

432 refresh=True, 

433 head=migration_script.head, 

434 splice=migration_script.splice, 

435 branch_labels=migration_script.branch_label, 

436 version_path=migration_script.version_path, 

437 depends_on=migration_script.depends_on, 

438 **template_args 

439 ) 

440 

441 def run_autogenerate(self, rev, migration_context): 

442 self._run_environment(rev, migration_context, True) 

443 

444 def run_no_autogenerate(self, rev, migration_context): 

445 self._run_environment(rev, migration_context, False) 

446 

447 def _run_environment(self, rev, migration_context, autogenerate): 

448 if autogenerate: 

449 if self.command_args["sql"]: 

450 raise util.CommandError( 

451 "Using --sql with --autogenerate does not make any sense" 

452 ) 

453 if set(self.script_directory.get_revisions(rev)) != set( 

454 self.script_directory.get_revisions("heads") 

455 ): 

456 raise util.CommandError("Target database is not up to date.") 

457 

458 upgrade_token = migration_context.opts["upgrade_token"] 

459 downgrade_token = migration_context.opts["downgrade_token"] 

460 

461 migration_script = self.generated_revisions[-1] 

462 if not getattr(migration_script, "_needs_render", False): 

463 migration_script.upgrade_ops_list[-1].upgrade_token = upgrade_token 

464 migration_script.downgrade_ops_list[ 

465 -1 

466 ].downgrade_token = downgrade_token 

467 migration_script._needs_render = True 

468 else: 

469 migration_script._upgrade_ops.append( 

470 ops.UpgradeOps([], upgrade_token=upgrade_token) 

471 ) 

472 migration_script._downgrade_ops.append( 

473 ops.DowngradeOps([], downgrade_token=downgrade_token) 

474 ) 

475 

476 self._last_autogen_context = autogen_context = AutogenContext( 

477 migration_context, autogenerate=autogenerate 

478 ) 

479 

480 if autogenerate: 

481 compare._populate_migration_script( 

482 autogen_context, migration_script 

483 ) 

484 

485 if self.process_revision_directives: 

486 self.process_revision_directives( 

487 migration_context, rev, self.generated_revisions 

488 ) 

489 

490 hook = migration_context.opts["process_revision_directives"] 

491 if hook: 

492 hook(migration_context, rev, self.generated_revisions) 

493 

494 for migration_script in self.generated_revisions: 

495 migration_script._needs_render = True 

496 

497 def _default_revision(self): 

498 op = ops.MigrationScript( 

499 rev_id=self.command_args["rev_id"] or util.rev_id(), 

500 message=self.command_args["message"], 

501 upgrade_ops=ops.UpgradeOps([]), 

502 downgrade_ops=ops.DowngradeOps([]), 

503 head=self.command_args["head"], 

504 splice=self.command_args["splice"], 

505 branch_label=self.command_args["branch_label"], 

506 version_path=self.command_args["version_path"], 

507 depends_on=self.command_args["depends_on"], 

508 ) 

509 return op 

510 

511 def generate_scripts(self): 

512 for generated_revision in self.generated_revisions: 

513 yield self._to_script(generated_revision)