Skip to content

CLI

CLI

The Trendify CLI allows the code to be run from a commandline interface.

DataProductsFileName

Defines arguments parsed from command line

add_argument classmethod

add_argument(parser: ArgumentParser)

Defines the argument parsing from command line

Source code in src/trendify/CLI.py
@classmethod
def add_argument(cls, parser: argparse.ArgumentParser):
    """Defines the argument parsing from command line"""
    parser.add_argument(
        '-f', 
        f'--{cls._NAME}', 
        type=str,
        default=API.DATA_PRODUCTS_FNAME_DEFAULT,
        help=(
            f'Sepcify the data file name to be used (defaults to {API.DATA_PRODUCTS_FNAME_DEFAULT})'
        )
    )

process_argument staticmethod

process_argument(arg: str) -> ProductGenerator

Processes input data from command line flag value

Parameters:

Name Type Description Default
arg str

File name to be type-cast to string

required

Returns:

Type Description
Callable

String (file name to be used for generated data products)

Source code in src/trendify/CLI.py
@staticmethod
def process_argument(arg: str) -> API.ProductGenerator:
    """
    Processes input data from command line flag value

    Args:
        arg (str): File name to be type-cast to string

    Returns:
        (Callable): String (file name to be used for generated data products)
    """
    return str(arg)

FileManager dataclass

FileManager(output_dir: Path)

Determines the folder setup for trendify directory

InputDirectories

Parses the --input-directories argument from CLI

process_argument staticmethod

process_argument(arg: str) -> List[Path]

Converts CLI input to list of directories over which user-specified data product generator method will be mapped.

Parameters:

Name Type Description Default
arg str

Directories or glob string from CLI

required

Returns:

Type Description
List[Path]

List of directories over which to map the user-specified product generator

Source code in src/trendify/CLI.py
@staticmethod
def process_argument(arg: str) -> List[Path]:
    """
    Converts CLI input to list of directories over which user-specified data product generator method will be mapped.

    Args:
        arg (str): Directories or glob string from CLI

    Returns:
        (List[Path]): List of directories over which to map the user-specified product generator
    """
    if isinstance(arg, str):
        return [
            Path(p).parent.resolve() if Path(p).is_file() else Path(p).resolve()
            for p 
            in glob(arg, root_dir=os.getcwd(), recursive=True)
        ]
    else:
        assert isinstance(arg, Iterable) and not isinstance(arg, str)
        paths = []
        for i in arg:
            for p in glob(i, root_dir=os.getcwd(), recursive=True):
                paths.append(Path(p).parent.resolve() if Path(p).is_file() else Path(p).resolve())
        return paths 

NProcs

Determines the number of processors to use in parallel for running trendify commands

add_argument classmethod

add_argument(parser: ArgumentParser)

Defines the argument parsing from command line

Source code in src/trendify/CLI.py
@classmethod
def add_argument(cls, parser: argparse.ArgumentParser):
    """Defines the argument parsing from command line"""
    parser.add_argument(
        '-n', 
        f'--{cls._NAME}',
        default=1,
        help=(
            'Specify the number of parallel processes to use for product generation, product sorting, and asset creation.'
            '\nParallelization reduces wall time for computationally expensive processes.'
            '\nThe number of parallel processes will be limited to a maximum of 5 times the number of available cores'
            'as a precaution not to crash the machine.'
        ),
    )

process_argument staticmethod

process_argument(arg: str)

Type-casts input to int and caps value at 5*os.cpu_count().

Parameters:

Name Type Description Default
arg int

Desired number of processes

required

Returns; (int): Number of processes capped to 5*os.cpu_count()

Source code in src/trendify/CLI.py
@staticmethod
def process_argument(arg: str):
    """
    Type-casts input to `int` and caps value at `5*os.cpu_count()`.

    Args:
        arg (int): Desired number of processes

    Returns;
        (int): Number of processes capped to `5*os.cpu_count()`
    """
    arg = int(arg)
    max_processes = 5*os.cpu_count()
    if arg > max_processes:
        logger.info(
            f'User-specified ({arg = }) exceeds ({max_processes = }).'
            f'Process count will be set to ({max_processes = })'
        )
        # print(
        #     f'User-specified ({arg = }) exceeds ({max_processes = }).'
        #     f'Process count will be set to ({max_processes = })'
        # )
    return min(arg, max_processes)

TrendifyDirectory

TrendifyDirectory(short_flag: str, full_flag: str)

Parses the --trendify-directory argument from CLI

Source code in src/trendify/CLI.py
def __init__(self, short_flag: str, full_flag: str):
    self._short_flag = short_flag
    self._full_flag = full_flag

process_argument

process_argument(arg: str) -> List[Path]

Converts CLI input to list of directories over which user-specified data product generator method will be mapped.

Parameters:

Name Type Description Default
arg str

Directories or glob string from CLI

required

Returns:

Type Description
FileManager

List of directories over which to map the user-specified product generator

Source code in src/trendify/CLI.py
def process_argument(self, arg: str) -> List[Path]:
    """
    Converts CLI input to list of directories over which user-specified data product generator method will be mapped.

    Args:
        arg (str): Directories or glob string from CLI

    Returns:
        (FileManager): List of directories over which to map the user-specified product generator
    """
    return FileManager(arg)

UserMethod

Defines arguments parsed from command line

add_argument classmethod

add_argument(parser: ArgumentParser)

Defines the argument parsing from command line

Source code in src/trendify/CLI.py
@classmethod
def add_argument(cls, parser: argparse.ArgumentParser):
    """Defines the argument parsing from command line"""
    parser.add_argument(
        '-g', 
        f'--{cls._NAME}', 
        required=True,
        help=(
            'Sepcify method `product_generator(workdir: Path) -> List[DataProduct]` method to map over input directories.'
            '\n\t\tUse the following formats:'
            '\n\t\tpackage.module,'
            '\n\t\tpackage.module:method,'
            '\n\t\tpackage.module:Class.method,'
            '\n\t\t/absolute/path/to/module.py,'
            '\n\t\t/absolute/path/to/module.py:method,'
            '\n\t\t/absolute/path/to/module.py:Class.method,'
            '\n\t\t./relative/path/to/module.py,'
            '\n\t\t./relative/path/to/module:method,'
            '\n\t\t./relative/path/to/module:Class.method'
        )
    )

process_argument staticmethod

process_argument(arg: str) -> ProductGenerator

Imports python method based on user CLI input

Parameters:

Name Type Description Default
arg str

Method to be imported in the form package.module:method or file/path.py:method. method can be replaced be Class.method. File path can be either relative or absolute.

required

Returns:

Type Description
Callable

User-specified method to be mapped over raw data directories.

Source code in src/trendify/CLI.py
@staticmethod
def process_argument(arg: str) -> API.ProductGenerator:
    """
    Imports python method based on user CLI input

    Args:
        arg (str): Method to be imported in the form `package.module:method` or `file/path.py:method`.
            `method` can be replaced be `Class.method`.  File path can be either relative or absolute.

    Returns:
        (Callable): User-specified method to be mapped over raw data directories.
    """
    msplit = arg.split(':')
    assert 1 <= len(msplit) <= 2
    module_path = msplit[0]
    method_name = msplit[1] if len(msplit) == 2 else None

    if Path(module_path).exists():
        module = _import_from_path(Path(module_path).name, Path(module_path))
    else:
        module = importlib.import_module(name=module_path)

    obj = module
    for arg in method_name.split('.'):
        obj = getattr(obj, arg)
    return obj

trendify

trendify(*pargs)

Defines the command line interface script installed with python package.

Run the help via trendify -h.

Parameters:

Name Type Description Default
*pargs list[Any]

List of flags and arguments to pass to commandline. Simulates running from commandline in pythong script.

()
Source code in src/trendify/CLI.py
def trendify(*pargs):
    """
    Defines the command line interface script installed with python package.

    Run the help via `trendify -h`.

    Args:
        *pargs (list[Any]): List of flags and arguments to pass to commandline.
            Simulates running from commandline in pythong script.
    """

    # Main parser
    parser = argparse.ArgumentParser(
        prog='trendify', 
        usage='Generate visual data products and static/interactives assets from raw data',
    )
    actions = parser.add_subparsers(title='Sub Commands', dest='command', metavar='')

    short_flag = 'o'
    full_flag = 'output-directory'
    output_dir = TrendifyDirectory(short_flag, full_flag)


    """ Useful Functions """
    ''' Make '''
    make = actions.add_parser('make', help='Generates products and assets.  Run with -h flag for info on subcommands.')
    make_actions = make.add_subparsers(title='Targets', dest='target', metavar='')
    # Make static assets (after making products and sorting them)
    make_static = make_actions.add_parser('static', help='Makes products, sorts them, and generates static assets')
    InputDirectories.add_argument(make_static)
    UserMethod.add_argument(make_static)
    NProcs.add_argument(make_static)
    output_dir.add_argument(make_static)
    DataProductsFileName.add_argument(make_static)
    # Dashboard (after making products and sorting them)
    make_dashboard = make_actions.add_parser('dashboard', help='Makes products and generates interactive dashboard')
    InputDirectories.add_argument(make_dashboard)
    UserMethod.add_argument(make_dashboard)
    NProcs.add_argument(make_dashboard)
    # output_dir.add_argument(make_static)
    DataProductsFileName.add_argument(make_dashboard)
    # Gallery
    make_gallery = make_actions.add_parser('gallery', help='Generates a gallery of examples')
    make_gallery.add_argument(
        f'-{short_flag}',
        f'--{full_flag}',
        type=Path,
        required=False,
        help=(
            'Sepcify output root directory to which the gallery data, products, and assets will be written. '
            'Defaults to current working directory'
        ),
        default='./gallery/',
    )

    ''' Serve '''
    serve = actions.add_parser('serve', help='Generates products and serves them to dashboard. Run with -h flag for info on subcommands.')
    serve_actions = serve.add_subparsers(title='Targets', dest='target', metavar='')
    # Plotly
    # Serve plotly dashboard (after making products)
    serve_plotly = serve_actions.add_parser('plotly', help='Makes products and serves them to interactive dashboards')
    InputDirectories.add_argument(serve_plotly)
    UserMethod.add_argument(serve_plotly)
    NProcs.add_argument(serve_plotly)
    DataProductsFileName.add_argument(serve_plotly)
    serve_plotly.add_argument('--title', type=str, help='Dashboard title', default='Trendify Autodash')
    serve_plotly.add_argument('--host', type=str, help='What addres to serve the data to', default='0.0.0.0')
    serve_plotly.add_argument('--port', type=int, help='What port to serve the data on', default=8000)

    """ 1-1 API Wrappers """
    ''' Products '''
    ### Products Make ###
    products_make = actions.add_parser('products-make', help='Makes products or assets')
    InputDirectories.add_argument(products_make)
    UserMethod.add_argument(products_make)
    NProcs.add_argument(products_make)
    DataProductsFileName.add_argument(products_make)
    ### Products Sort ###
    products_sort = actions.add_parser('products-sort', help='Sorts data products by tags')
    InputDirectories.add_argument(products_sort)
    output_dir.add_argument(products_sort)
    NProcs.add_argument(products_sort)
    DataProductsFileName.add_argument(products_sort)
    ### Products Serve ###
    products_serve = actions.add_parser('products-serve', help='Serves data products to URL endpoint at 127.0.0.1')
    InputDirectories.add_argument(products_serve)
    DataProductsFileName.add_argument(products_serve)
    products_serve.add_argument('--title', type=str, help='Dashboard title', default='Trendify Autodash')
    products_serve.add_argument('--host', type=str, help='What addres to serve the data to', default='127.0.0.1')
    products_serve.add_argument('--port', type=int, help='What port to serve the data on', default=8000)

    ''' Assets '''
    ### Assets Make Static
    assets_make_static = actions.add_parser('assets-make-static', help='Makes static assets')
    assets_make_static.add_argument('trendify_output_directory')
    NProcs.add_argument(assets_make_static)


    # Test
    if pargs:
        args = parser.parse_args(*pargs)
    else:
        args = parser.parse_args()

    logger.info(f'Running `trendify` with {args = }')

    match args.command:
        case 'products-make':
            API.make_products(
                product_generator=UserMethod.get_from_namespace(args),
                data_dirs=InputDirectories.get_from_namespace(args),
                n_procs=NProcs.get_from_namespace(args),
                data_products_fname=DataProductsFileName.get_from_namespace(args),
            )
        case 'products-sort':
            API.sort_products(
                data_dirs=InputDirectories.get_from_namespace(args),
                output_dir=output_dir.get_from_namespace(args).products_dir,
                n_procs=NProcs.get_from_namespace(args),
                data_products_fname=DataProductsFileName.get_from_namespace(args),
            )
        case 'products-serve':
            API.serve_products_to_plotly_dashboard(
                *tuple(InputDirectories.get_from_namespace(args)),
                title=args.title,
                host=args.host,
                port=args.port,
                data_products_filename=DataProductsFileName.get_from_namespace(args),
            )
        case 'assets-make-static':
            API.make_tables_and_figures(
                products_dir=FileManager(args.trendify_output_directory).products_dir,
                output_dir=FileManager(args.trendify_output_directory).static_assets_dir,
                n_procs=NProcs.get_from_namespace(args),
            )
        case 'make':
            match args.target:
                case 'gallery':
                    from trendify.gallery import make_gallery
                    make_gallery(Path(getattr(args, full_flag.replace('-', '_'), '.')))
                case 'static':
                    um = UserMethod.get_from_namespace(args)
                    ip = InputDirectories.get_from_namespace(args)
                    np = NProcs.get_from_namespace(args)
                    td = output_dir.get_from_namespace(args)
                    fn = DataProductsFileName.get_from_namespace(args)
                    API.make_products(product_generator=um, data_dirs=ip, n_procs=np, data_products_fname=fn)
                    API.sort_products(data_dirs=ip, output_dir=td.products_dir, n_procs=np, data_products_fname=fn)
                    API.make_tables_and_figures(products_dir=td.products_dir, output_dir=td.static_assets_dir, n_procs=np)
                case _:
                    raise NotImplementedError()
        case 'serve':
            match args.target:
                case 'plotly':
                    um = UserMethod.get_from_namespace(args)
                    ip = InputDirectories.get_from_namespace(args)
                    np = NProcs.get_from_namespace(args)
                    # td = output_dir.get_from_namespace(args)
                    fn = DataProductsFileName.get_from_namespace(args)
                    API.make_products(product_generator=um, data_dirs=ip, n_procs=np, data_products_fname=fn)
                    API.serve_products_to_plotly_dashboard(
                        *tuple(ip),
                        title=args.title,
                        host=args.host,
                        port=args.port,
                    )
                case _:
                    raise NotImplementedError()