Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

98

99

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

# encoding: utf-8 

from __future__ import print_function, division, absolute_import 

 

import glob 

import os 

from os.path import exists, join 

import shutil 

 

from .utils import list_folder_recursive, copy, iter_to_list, here, hexdigest_file 

from .errors import InvalidLandingZone 

from .landing_zone_structure import scripts_pattern, raw_file_pattern_for_script 

 

 

def disable_write(path): 

os.chmod(path, 0o444) 

 

 

def enable_write(path): 

os.chmod(path, 0o744) 

 

 

class LandingZone: 

 

def __init__(self, root_folder): 

self.root_folder = root_folder 

if exists(root_folder): 

self._check_landing_zone() 

 

def _check_landing_zone(self): 

self._check_start_state() 

 

@property 

def path_to_start_state(self): 

return join(self.root_folder, ".start_state") 

 

def _resolve_start_state(self): 

result = {} 

start_state = self.path_to_start_state 

try: 

fh = open(start_state, "r") 

lines = fh.readlines() 

except IOError as e: 

raise InvalidLandingZone("reading from {} failed. reason: {}".format(start_state, e)) 

for line in lines: 

path, __, checksum = line.rstrip().partition(" ") 

result[path] = checksum 

return result 

 

def _check_start_state(self): 

if not exists(self.path_to_start_state): 

raise InvalidLandingZone( 

"landing zone at {} does not contain .start_state file".format(self.root_folder)) 

 

missing = [] 

for rel_path, check_sum in self._resolve_start_state().items(): 

path = os.path.join(self.root_folder, rel_path) 

if not exists(path): 

missing.append(path) 

if missing: 

n = len(missing) 

if len(missing) > 5: 

mm = ", ".join(missing[:5]) 

msg = "files {}, ... ({} in total) are missing from landing zone".format(mm, n) 

else: 

mm = ", ".join(missing) 

msg = "files {} are missing from landing zone".format(mm, n) 

raise InvalidLandingZone(msg) 

 

def create_empty(self): 

if not exists(self.root_folder): 

os.makedirs(self.root_folder) 

 

@staticmethod 

def create_from(new_landing_zone, existing_landing_zone): 

assert not exists(new_landing_zone) 

lz = LandingZone(new_landing_zone) 

shutil.copytree(existing_landing_zone, lz.root_folder) 

 

already_exists = list_folder_recursive(existing_landing_zone) 

lz.write_start_state_file(already_exists) 

 

for rel_path in already_exists: 

path = join(lz.root_folder, rel_path) 

disable_write(path) 

 

return lz 

 

def write_start_state_file(self, rel_pathes): 

with open(self.path_to_start_state, "w") as fh: 

for rel_path in rel_pathes: 

full_path = os.path.join(self.root_folder, rel_path) 

print(rel_path, hexdigest_file(full_path), file=fh) 

disable_write(self.path_to_start_state) 

 

@staticmethod 

def create_from_empty(development_landing_zone): 

lz = LandingZone(development_landing_zone) 

source = join(here(), "default_initial_landing_zone") 

shutil.copytree(source, lz.root_folder) 

# operational zone is expected to be empty: 

open(lz.path_to_start_state, "w").close() 

disable_write(lz.path_to_start_state) 

return lz 

 

def p(self, relative_path): 

return join(self.root_folder, relative_path) 

 

def add_file(self, path_in_landing_zone, file_path): 

full_path = self.p(path_in_landing_zone) 

folder = os.path.dirname(full_path) 

if not exists(folder): 

os.makedirs(folder) 

shutil.copy(file_path, folder) 

 

def overwrite_file(self, path_in_landing_zone, file_path): 

full_path = self.p(path_in_landing_zone) 

shutil.copy(file_path, full_path) 

 

def remove_file(self, path_in_landing_zone): 

full_path = self.p(path_in_landing_zone) 

os.remove(full_path) 

 

@iter_to_list 

def check_update_operational(self, operational_landing_zone): 

""" 

check if saved state is valid: 

check if operational_landing_zone is super set of saved state 

""" 

operational_files = set(list_folder_recursive(operational_landing_zone)) 

saved_start_files = set(self._resolve_start_state().keys()) 

if saved_start_files > operational_files: 

raise InvalidLandingZone("development landing zone invalid: {} contains files which " 

"are not in {}".format(self.path_to_start_state, 

operational_landing_zone)) 

new_files = self.list_new_files() 

for new_file in sorted(new_files): 

if exists(join(operational_landing_zone, new_file)): 

yield ("development zone contains new file {} which already exists " 

"in {}".format(join(self.root_folder, new_file), operational_landing_zone)) 

 

def list_new_files(self): 

"""lists new files, this are files which are not listed in the start_state_file""" 

rel_path_to_hexdigest = self._resolve_start_state() 

for rel_path in list_folder_recursive(self.root_folder): 

if rel_path not in rel_path_to_hexdigest.keys(): 

yield rel_path 

 

def list_changed_files(self): 

"""lists new and edited files. edit change is determined based on hexdigest""" 

yield from self.list_new_files() 

rel_path_to_hexdigest = self._resolve_start_state() 

for rel_path in list_folder_recursive(self.root_folder): 

path = join(self.root_folder, rel_path) 

tobe = rel_path_to_hexdigest.get(rel_path) 

if tobe is not None: 

if hexdigest_file(path) != tobe: 

yield rel_path 

 

@iter_to_list 

def update_operational(self, operational_landing_zone): 

""" 

2. detect new files 

2. copy them around 

""" 

saved_start_files = set(self._resolve_start_state().keys()) 

new_files = self.list_new_files() 

 

for new_file in sorted(new_files): # we sort to make output consistent 

copy(join(self.root_folder, new_file), join(operational_landing_zone, new_file)) 

yield new_file 

saved_start_files.add(new_file) 

 

self.update_start_state_file(saved_start_files) 

 

def update_start_state_file(self, rel_pathes): 

enable_write(self.path_to_start_state) 

self.write_start_state_file(rel_pathes) 

disable_write(self.path_to_start_state) 

 

def conversion_scripts_and_data(self): 

script_pattern = join(self.root_folder, scripts_pattern) 

for script_path in glob.glob(script_pattern): 

raw_file_pattern = raw_file_pattern_for_script(script_path) 

for raw_file_path in glob.glob(raw_file_pattern): 

yield script_path, raw_file_path