Source code for metapack_build.cli.new

# Copyright (c) 2017 Civic Knowledge. This file is licensed under the terms of the
# Revised BSD License, included in this distribution as LICENSE

"""
Metapack CLI program for creating new metapack package directories
"""

import argparse
from pathlib import Path
from shutil import copyfile

import requests
from metapack.cli.core import MetapackCliMemo as _MetapackCliMemo
from metapack.package import Downloader

downloader = Downloader.get_instance()

TEMPLATE_NOTEBOOK = "https://raw.githubusercontent.com/Metatab/exploratory-data-analysis/master/metadata.ipynb"


[docs]class MetapackCliMemo(_MetapackCliMemo): def __init__(self, args, downloader): super().__init__(args, downloader)
[docs]def new_args(subparsers): """ The `mp new` command creates source package directories with a proper name, a `.gitignore` file, and optionally, example data, entries and code. Typical usage, for creating a new package with most of the example options, is :: mp new -o metatab.org -d tutorial -T "Quickstart Example Package" The ``-C`` option will set a configuration file, which is a Metatab file that with terms that are copied into the `metadata.csv` file of the new package. Currently, it copies a limited number of terms, including: - Terms in the Contacts section - Root.Space - Root.Time - Root.Grain - Root.Variant - Root.Version """ parser = subparsers.add_parser( 'new', help='Create new Metatab packages', description=new_args.__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.set_defaults(run_command=new_cmd) parser.add_argument('-o', '--origin', help="Dataset origin, usually a second-level domain name. Required") parser.add_argument('-d', '--dataset', help="Main dataset name. Required", required=True) parser.add_argument('-t', '--time', help="Temporal extents, usually a year, ISO8601 time, or interval. ") parser.add_argument('-s', '--space', help="Space, geographic extent, such as a name of a state or a Census geoid") parser.add_argument('-g', '--grain', help="Grain, the type of entity a row represents") parser.add_argument('-v', '--variant', help="Variant, any distinguishing string") parser.add_argument('-r', '--revision', help="Version, defaults to 1", default=1) parser.add_argument('-T', '--title', help="Set the title") parser.add_argument('-L', '--pylib', help="Configure a pylib directory for Python code extensions", action='store_true') parser.add_argument('-E', '--example', help="Add examples of resources", action='store_true') try: import metapack_jupyter # noqa parser.add_argument('-J', '--jupyter', help="Create a Jupyter notebook source package", action='store_true') except ImportError: pass parser.add_argument('--template', help="Metatab file template, defaults to 'metatab' ", default='metatab') parser.add_argument('-C', '--config', help="Path to config file. " "Defaults to ~/.metapack-defaults.csv or value of METAPACK_DEFAULTS env var." "Sets defaults for specia root terms and the Contacts section.") return parser
[docs]def doc_parser(): from metapack.cli.mp import base_parser return new_args(base_parser())
[docs]class EmptyTerm(object): value = None
et = EmptyTerm()
[docs]def new_cmd(args): from metapack import MetapackDoc from metapack.util import make_metatab_file, datetime_now, ensure_dir from metapack.cli.core import write_doc, prt, err from os.path import exists, join, expanduser, dirname from metatab import DEFAULT_METATAB_FILE from os import getenv import metapack.support as support_dir if args.config: config_file = args.config elif getenv("METAPACK_CONFIG"): config_file = getenv("METAPACK_DEFAULTS") elif expanduser('~/.metapack-default.csv'): config_file = expanduser('~/.metapack-defaults.csv') else: config_file = None if config_file and exists(config_file): prt(f"Using defaults file '{config_file}'") config = MetapackDoc(config_file) else: config = MetapackDoc() if args.jupyter: import tempfile with tempfile.NamedTemporaryFile(suffix='.ipynb', delete=False) as fp: r = requests.get(TEMPLATE_NOTEBOOK, allow_redirects=True) r.raise_for_status() fp.write(r.content) nb_path = Path(fp.name) doc = MetapackDoc(str(nb_path)) else: doc = make_metatab_file(args.template) doc['Root']['Created'] = datetime_now() origin = args.origin or config.get_value('Root.Origin') if not origin: err("Must specify a value for origin, either on command line or in defaults file") (doc['Root'].find_first('Root.Origin') or et).value = origin (doc['Root'].find_first('Root.Dataset') or et).value = args.dataset (doc['Root'].find_first('Root.Space') or et).value = args.space or config.get_value('Root.Space') (doc['Root'].find_first('Root.Time') or et).value = args.time or config.get_value('Root.Time') (doc['Root'].find_first('Root.Grain') or et).value = args.grain or config.get_value('Root.Grain') (doc['Root'].find_first('Root.Variant') or et).value = args.variant or config.get_value('Root.Variant') (doc['Root'].find_first('Root.Version') or et).value = args.revision or config.get_value('Root.Version') # Copy contacts in if 'Contacts' in config: for c in config['Contacts']: doc['Contacts'].add_term(c) if args.title: doc['Root'].find_first('Root.Title').value = args.title.strip() nv_name = doc.as_version(None) if args.example: doc['Resources'].new_term('Root.Datafile', 'http://public.source.civicknowledge.com/example.com/sources/random-names.csv', name='random_names') doc['Documentation'].new_term('Root.Homepage', 'http://metatab.org', title='Metatab Home Page') doc.ensure_identifier() doc.update_name(create_term=True) if getattr(args, 'jupyter'): # b/c maybe metatab_jupyter is not installed from metapack_jupyter.convert import write_metatab_notebook from metapack_jupyter.core import edit_notebook, set_cell_source, get_cell_source new_nb_path = Path(f'{nv_name}.ipynb') doc['Resources'].new_term('Root.Datafile', './' + str(new_nb_path) + "#df", name='local_dataframe', description='Example of using a local Dataframe') if new_nb_path.exists(): err(f"Directory {nb_path} already exists") copyfile(nb_path, new_nb_path) write_metatab_notebook(doc, new_nb_path) with edit_notebook(new_nb_path) as nb: init = get_cell_source(nb, 'init') init += f"\nthis_package_name = '{str(new_nb_path.name)}'" set_cell_source(nb, 'init', init) nb_path.unlink() else: doc['Documentation'].new_term('Root.Documentation', 'file:README.md', title='README') if exists(nv_name): err(f"Directory {nv_name} already exists") ensure_dir(nv_name) if True: # args.pylib: from metapack.support import pylib pylib_dir = join(nv_name, 'pylib') ensure_dir(pylib_dir) with open(join(pylib_dir, '__init__.py'), 'w') as f_out, open(pylib.__file__) as f_in: f_out.write(f_in.read()) if args.example: doc['Resources'].new_term('Root.Datafile', 'python:pylib#row_generator', name='row_generator') prt(f"Writing to '{nv_name}'") write_doc(doc, join(nv_name, DEFAULT_METATAB_FILE)) with open(join(dirname(support_dir.__file__), 'gitignore')) as f: gitignore = f.read() with open(join(nv_name, '.gitignore'), 'w') as f: f.write(gitignore) if args.title: readme = '# {}\n'.format(args.title) else: readme = '# {}\n'.format(doc.get_value('Root.Name')) with open(join(nv_name, 'README.md'), 'w') as f: f.write(readme)