# encoding: utf-8
from __future__ import print_function, division, absolute_import
import glob
import os
from os.path import exists, join
import shutil
from .utils import list_folder_recursive, copy, iter_to_list, here, hexdigest_file
from .errors import InvalidLandingZone
from .landing_zone_structure import scripts_pattern, raw_file_pattern_for_script
def disable_write(path):
os.chmod(path, 0o444)
def enable_write(path):
os.chmod(path, 0o744)
class LandingZone:
def __init__(self, root_folder):
self.root_folder = root_folder
if exists(root_folder):
self._check_landing_zone()
def _check_landing_zone(self):
self._check_start_state()
@property
def path_to_start_state(self):
return join(self.root_folder, ".start_state")
def _resolve_start_state(self):
result = {}
start_state = self.path_to_start_state
try:
fh = open(start_state, "r")
lines = fh.readlines()
except IOError as e:
raise InvalidLandingZone("reading from {} failed. reason: {}".format(start_state, e))
for line in lines:
path, __, checksum = line.rstrip().partition(" ")
result[path] = checksum
return result
def _check_start_state(self):
if not exists(self.path_to_start_state):
raise InvalidLandingZone(
"landing zone at {} does not contain .start_state file".format(self.root_folder))
missing = []
for rel_path, check_sum in self._resolve_start_state().items():
path = os.path.join(self.root_folder, rel_path)
if not exists(path):
missing.append(path)
if missing:
n = len(missing)
if len(missing) > 5:
mm = ", ".join(missing[:5])
msg = "files {}, ... ({} in total) are missing from landing zone".format(mm, n)
else:
mm = ", ".join(missing)
msg = "files {} are missing from landing zone".format(mm, n)
raise InvalidLandingZone(msg)
def create_empty(self):
if not exists(self.root_folder):
os.makedirs(self.root_folder)
@staticmethod
def create_from(new_landing_zone, existing_landing_zone):
assert not exists(new_landing_zone)
lz = LandingZone(new_landing_zone)
shutil.copytree(existing_landing_zone, lz.root_folder)
already_exists = list_folder_recursive(existing_landing_zone)
lz.write_start_state_file(already_exists)
for rel_path in already_exists:
path = join(lz.root_folder, rel_path)
disable_write(path)
return lz
def write_start_state_file(self, rel_pathes):
with open(self.path_to_start_state, "w") as fh:
for rel_path in rel_pathes:
full_path = os.path.join(self.root_folder, rel_path)
print(rel_path, hexdigest_file(full_path), file=fh)
disable_write(self.path_to_start_state)
@staticmethod
def create_from_empty(development_landing_zone):
lz = LandingZone(development_landing_zone)
source = join(here(), "default_initial_landing_zone")
shutil.copytree(source, lz.root_folder)
# operational zone is expected to be empty:
open(lz.path_to_start_state, "w").close()
disable_write(lz.path_to_start_state)
return lz
def p(self, relative_path):
return join(self.root_folder, relative_path)
def add_file(self, path_in_landing_zone, file_path):
full_path = self.p(path_in_landing_zone)
folder = os.path.dirname(full_path)
if not exists(folder):
os.makedirs(folder)
shutil.copy(file_path, folder)
def overwrite_file(self, path_in_landing_zone, file_path):
full_path = self.p(path_in_landing_zone)
shutil.copy(file_path, full_path)
def remove_file(self, path_in_landing_zone):
full_path = self.p(path_in_landing_zone)
os.remove(full_path)
@iter_to_list
def check_update_operational(self, operational_landing_zone):
"""
check if saved state is valid:
check if operational_landing_zone is super set of saved state
"""
operational_files = set(list_folder_recursive(operational_landing_zone))
saved_start_files = set(self._resolve_start_state().keys())
if saved_start_files > operational_files:
raise InvalidLandingZone("development landing zone invalid: {} contains files which "
"are not in {}".format(self.path_to_start_state,
operational_landing_zone))
new_files = self.list_new_files()
for new_file in sorted(new_files):
if exists(join(operational_landing_zone, new_file)):
yield ("development zone contains new file {} which already exists "
"in {}".format(join(self.root_folder, new_file), operational_landing_zone))
def list_new_files(self):
"""lists new files, this are files which are not listed in the start_state_file"""
rel_path_to_hexdigest = self._resolve_start_state()
for rel_path in list_folder_recursive(self.root_folder):
if rel_path not in rel_path_to_hexdigest.keys():
yield rel_path
def list_changed_files(self):
"""lists new and edited files. edit change is determined based on hexdigest"""
yield from self.list_new_files()
rel_path_to_hexdigest = self._resolve_start_state()
for rel_path in list_folder_recursive(self.root_folder):
path = join(self.root_folder, rel_path)
tobe = rel_path_to_hexdigest.get(rel_path)
if tobe is not None:
if hexdigest_file(path) != tobe:
yield rel_path
@iter_to_list
def update_operational(self, operational_landing_zone):
"""
2. detect new files
2. copy them around
"""
saved_start_files = set(self._resolve_start_state().keys())
new_files = self.list_new_files()
for new_file in sorted(new_files): # we sort to make output consistent
copy(join(self.root_folder, new_file), join(operational_landing_zone, new_file))
yield new_file
saved_start_files.add(new_file)
self.update_start_state_file(saved_start_files)
def update_start_state_file(self, rel_pathes):
enable_write(self.path_to_start_state)
self.write_start_state_file(rel_pathes)
disable_write(self.path_to_start_state)
def conversion_scripts_and_data(self):
script_pattern = join(self.root_folder, scripts_pattern)
for script_path in glob.glob(script_pattern):
raw_file_pattern = raw_file_pattern_for_script(script_path)
for raw_file_path in glob.glob(raw_file_pattern):
yield script_path, raw_file_path
|