# encoding: utf-8
from __future__ import print_function, division, absolute_import
import contextlib
import os
from os.path import join, exists
import shutil
import tempfile
from .database import setup_db, setup_fresh_db, check_if_tables_exist, copy_db
from .errors import InvalidOperationError, InvalidLandingZone
from .landing_zone import LandingZone
from .logger import setup_logger_from, drop_logger, check_logging_config, logger
from .utils import abs_folder, enumerate_filename, print_signals
@contextlib.contextmanager
def get_cmdline_logger(verbose):
config_file = join(abs_folder(__file__), "cmdline_logging.yaml")
setup_logger_from(config_file)
if verbose:
logger().setLevel(10)
yield
drop_logger()
def init_config(landing_zone, reset, print_ok, print_err, verbose=False):
"""setup minimal landing zone and create default configuration
"""
from datapool.config_handling import init_config
with get_cmdline_logger(verbose):
if exists(landing_zone):
print_err("+ folder/file {} already exists".format(landing_zone))
return 1
print_ok("- guess settings")
try:
config_folder, messages = init_config(landing_zone, overwrite=reset)
for message in messages:
print_err(" - {}".format(message))
print_ok("- created config files at {}".format(config_folder))
print_ok(
" please edit these files and adapt the data base configuration to your setup")
except Exception as e:
print_err("+ something went wrong: {}".format(e))
return 1
try:
LandingZone(landing_zone).create_empty()
print_ok("+ initialized landing zone at {}".format(landing_zone), fg="green")
except Exception as e:
print_err("+ something went wrong: {}".format(e))
return 1
return 0
def check_config(print_ok, print_err, verbose=False):
"""checks if current configuration is valid, eg if database access is possible, or
if matlab can be started.
"""
from datapool.config_handling import read_config, check_config
config = read_config()
if config is None:
print_err("- no config file found. please run 'pool init-config' first.")
return 1
print_ok("- check settings in config file {}".format(config.__file__))
try:
check_logging_config(config)
except Exception as e:
print_err("- could not setup logger. hint: {}".format(e))
return 1
with get_cmdline_logger(verbose):
ok = True
for ok, message in check_config(config):
if ok:
print_ok(message)
else:
print_err(message)
ok = False
if ok:
print_ok("+ all checks passed", fg="green")
else:
print_err("+ checks failed")
return ok
def init_db(reset, verbose, print_ok, print_err):
"""creates empty tables in configured database, can be used to delete data from an existing
database.
"""
with get_cmdline_logger(verbose):
from datapool.config_handling import read_config
config = read_config()
if config is None:
print_err("+ no config file found. please run 'pool init' first.")
return 1
already_exists = check_if_tables_exist(config.db)
if already_exists:
if reset:
setup_fresh_db(config.db, verbose=verbose)
else:
print_err("+ database is already initialized, use --force TWICE to setup a fresh "
"and empty database. YOU WILL LOOSE ALL EXISTING DATA !!!")
return 1
else:
setup_db(config.db, verbose=verbose)
print_ok("+ intialized db", fg="green")
return 0
def _setup_landing_zone(development_landing_zone, operational_landing_zone, is_first_time, reset,
print_ok, print_err):
print_ok("- setup development landing zone")
if exists(development_landing_zone):
if not reset:
print_err(" - folder {} already exists.".format(development_landing_zone))
return 1
else:
try:
shutil.rmtree(development_landing_zone)
except Exception as e:
print_err(" - could not delete folder {}".format(development_landing_zone))
print_err(" - error message is: {}".format(e.strerror))
return 1
try:
if is_first_time:
print_ok("- operational landing zone is empty. create development landing zone with "
"example files.")
LandingZone.create_from_empty(development_landing_zone)
else:
print_ok("- copy files from operational landing zone.")
LandingZone.create_from(development_landing_zone, operational_landing_zone)
except IOError as e:
print_err("- something went wrong: {}".format(e))
return 1
return 0
def _setup_db(development_landing_zone, config, is_first_time, reset, verbose, print_ok, print_err):
from datapool.config_handling import config_for_develop_db
from datapool.database import setup_db
print_ok("- setup development data base")
config_develop_db, __ = config_for_develop_db(development_landing_zone)
conn_string = config_develop_db.connection_string
try:
tables_exist = check_if_tables_exist(config_develop_db)
except InvalidOperationError:
print_err(" - could not connect to {}. check database settings.".format(conn_string))
return 1
if tables_exist and not reset:
print_err(" - the database {} already holds datapool tables.".format(conn_string))
return 1
if is_first_time:
print_ok("- create tables and indices")
try:
setup_db(config_develop_db)
except Exception as e:
print_err(" - could not setup database {}".format(conn_string))
print_err(" - error message is: {}".format(e))
return 1
else:
print_ok("- clone meta data from operational db to development db")
try:
copy_db(config.db, config_develop_db, delete_existing=True,
copy_signals=False, verbose=verbose)
except Exception as e:
print_err(" - cloning failed. error message is: {}".format(e))
return 1
return 0
def start_develop(development_landing_zone, reset, verbose, print_ok, print_err):
"""creates local landing zone for integrating new devices, conversion scripts etc.
either the configured operational landing zone is cloned or example files are written to
the local landing zone.
Further a local sqlite database is created.
if "reset" is True existing folders and database will be overwritten.
"""
from datapool.config_handling import read_config
config = read_config(DEVELOPZONE=development_landing_zone)
operational_landing_zone = config.landing_zone.folder
if not exists(operational_landing_zone):
print_err("+ configured landing zone {} does not exist".format(operational_landing_zone))
return 1
try:
is_first_time = (os.listdir(operational_landing_zone) == [])
except IOError as e:
print_err("+ can not read {}: {}".format(operational_landing_zone, e))
return 1
with get_cmdline_logger(verbose):
exit_code = _setup_landing_zone(development_landing_zone, operational_landing_zone,
is_first_time, reset, print_ok, print_err)
if not exit_code:
exit_code = _setup_db(development_landing_zone, config, is_first_time, reset, verbose,
print_ok, print_err)
if exit_code:
print_err("+ setup failed")
else:
print_ok("+ setup done", fg="green")
return exit_code
def check_scripts(landing_zone, result_folder, verbose, print_ok, print_err):
print_ok("- check landing zone at {}".format(landing_zone))
if not exists(landing_zone):
print_err(" - folder {} does not exist".format(landing_zone))
return 1
from datapool.data_conversion import check_conversion
from datapool.config_handling import read_config
config = read_config()
if not result_folder:
result_folder = tempfile.mkdtemp()
else:
if os.path.exists(result_folder):
if not os.path.isdir(result_folder):
print_err("+ given path {} exists but is not a folder".format(result_folder))
return 1
else:
os.makedirs(result_folder)
print_ok("- created folder {}".format(result_folder))
with get_cmdline_logger(verbose):
try:
lz = LandingZone(landing_zone)
except InvalidLandingZone as e:
print_err(" - landing zone at {} invalid. reason: {}".format(landing_zone, e))
return 1
found_errors = False
for script, data_path in lz.conversion_scripts_and_data():
print_ok("- check {}".format(script))
for result in check_conversion(script, data_path, config, verbose):
if isinstance(result, Exception):
print_err(" - {}".format(result))
found_errors = True
if not found_errors:
# last result must be the tuple output_file, signals
output_file, signals = result
script_folder_name = os.path.basename(os.path.dirname(script))
csv_path = join(result_folder, script_folder_name) + ".csv"
txt_path = join(result_folder, script_folder_name) + ".txt"
csv_path, txt_path = enumerate_filename(csv_path, txt_path)
shutil.copy(output_file, csv_path)
print_ok("- wrote conversion result to {}".format(csv_path))
with open(txt_path, "w") as fh:
print_signals(signals, file=fh)
print_ok("- wrote conversion result to {}".format(txt_path))
if found_errors:
print_err("+ checks failed. please fix this.")
return 1
print_ok("+ congratulations: checks succeeded.", fg="green")
return 0
def update_operational(development_landing_zone, verbose, overwrite, print_ok, print_err):
from datapool.config_handling import read_config
config = read_config()
operational_landing_zone = config.landing_zone.folder
lz = LandingZone(development_landing_zone)
print_ok("- check before copying files around.")
try:
messages = lz.check_update_operational(operational_landing_zone)
except InvalidLandingZone as e:
print_err(" - {}".format(e))
return 1
if messages:
for message in messages:
print_err(" - problem: {}".format(message))
if not overwrite:
return 1
else:
print_ok(" - will ignore these and overwrite existing files")
files = lz.update_operational(operational_landing_zone)
for file in files:
print_ok("- copied {}".format(file))
print_ok("+ copied {} files to {}".format(len(files), operational_landing_zone), fg="green")
return 0
@contextlib.contextmanager
def _setup_test_db(landing_zone):
from datapool.config_handling import config_for_develop_db
from datapool.database import connect_to_db
db_config, path = config_for_develop_db(landing_zone)
shutil.copy(path, path + ".backup")
engine = connect_to_db(db_config)
yield engine
engine.dispose()
os.unlink(path)
os.rename(path + ".backup", path)
def check_yamls(landing_zone, verbose, print_ok, print_err):
print_ok("- check yamls in landing zone at {}".format(landing_zone))
if not exists(landing_zone):
print_err(" - folder {} does not exist".format(landing_zone))
return 1
with get_cmdline_logger(verbose):
try:
lz = LandingZone(landing_zone)
except InvalidLandingZone as e:
print_err(" - landing zone at {} invalid. reason: {}".format(landing_zone, e))
return 1
with _setup_test_db(landing_zone) as engine:
found_errors = _check_lz(lz, engine, print_ok, print_err, verbose)
if found_errors:
print_err("+ all scripts checked")
return 1
print_ok("+ all scripts checked", fg="green")
return 0
def _check_lz(lz, engine, print_ok, print_err, verbose):
found_errors = False
for rel_path in lz.list_changed_files():
if not rel_path.endswith(".yaml"):
continue
print_ok("- check {}".format(join(lz.root_folder, rel_path)))
for error in _check_yaml(engine, lz.root_folder, rel_path):
found_errors = True
print_err(" - got error: {}".format(error))
return found_errors
def _check_yaml(engine, landing_zone_folder, rel_path):
from datapool.landing_zone_structure import lookup_parser, lookup_checker
parser = lookup_parser(rel_path)
checker = lookup_checker(rel_path)
if parser is None:
yield Exception("no parser for {} implemented. maybe filename of misspelled "
"or file is at wrong place".format(rel_path))
return
if checker is None:
full_path = join(landing_zone_folder, rel_path)
raise RuntimeError("no checker for {} found, but parser exists".format(rel_path))
results = parser(landing_zone_folder, rel_path)
for parse_result in results:
if isinstance(parse_result, Exception):
yield parse_result
else:
check_results = checker(parse_result, engine)
for check_result in check_results:
if isinstance(check_result, Exception):
yield check_result
|