Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

98

99

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266

267

268

269

270

271

272

273

274

275

276

277

278

279

280

281

282

283

284

285

286

# encoding: utf-8 

from __future__ import print_function, division, absolute_import 

 

from collections import OrderedDict 

from contextlib import closing 

import os 

import shutil 

 

from .config import MagicConfig, write_ini, read_ini 

from .utils import find_executable, iter_to_list 

 

 

def guess_config(known_config): 

"""you can provide known config settings as 

 

known_config = { "db.connection_string:"...", 

"worker.port": 3333 } 

 

which will overrung the settings we guess in this function 

""" 

 

assert isinstance(known_config, dict) 

 

executables = ("matlab", "R", "julia", "python3.5") 

sections = ("matlab", "r", "julia", "python") 

 

messages = [] 

 

# we use OrderedDict to guarantee stable ordering of entries in config 

# file(s) 

config = OrderedDict() 

for executable, section in zip(executables, sections): 

path = find_executable(executable, "") 

if not path: 

messages.append("{!r} not found on $PATH".format(executable)) 

config["{}.executable".format(section)] = path 

 

config.update(known_config) 

 

return default_config(config), messages 

 

 

def default_config(known_config): 

"""you can overwrite with calls like: 

 

default_config(db.connections_string="....", 

worker.port=3333) 

""" 

 

config = MagicConfig() 

 

config.db.connection_string = "postgresql://user:password@localhost:5432/datapool" 

 

config.r.extension = ".r" 

 

config.matlab.extension = ".m" 

 

config.julia.extension = ".jl" 

config.julia.version = "0.5.0" 

 

config.python.extension = ".py" 

 

config.logging.config_file = "./logging_config.yaml" 

config.log_receiver.port = 5559 

 

config.worker.port = 5555 

config.worker.count = 5 

 

for key, value in known_config.items(): 

fields = key.split(".") 

assert len(fields) == 2 

section, field = fields 

setattr(getattr(config, section), field, value) 

 

return config 

 

 

def config_for_develop_db(landing_zone_folder): 

config = MagicConfig() 

path = os.path.join(landing_zone_folder, ".develop.db") 

config.connection_string = "sqlite+pysqlite:///{}".format(path) 

return config, path 

 

 

@iter_to_list 

def check_config(config): 

 

from .database import connect_to_db 

 

@iter_to_list 

def check(section, field, type, maybe_empty=True): 

if section not in config.keys(): 

yield False, "- section {} missing".format(section) 

if field not in config[section].keys(): 

yield False, "- field {} in section {} missing".format(field, section) 

 

value = config[section][field] 

if not maybe_empty and value == "": 

yield False, "- field {} in section {} has no value set".format(field, section) 

if not isinstance(value, type): 

yield (False, 

"- field {} in section {} is not of type {}".format(field, 

section, type.__name__) 

) 

 

errors = check("db", "connection_string", str, False) 

yield from errors 

 

if not errors: 

try: 

yield True, "- try to connect to db" 

connect_to_db(config.db) 

yield True, " - connected to db" 

except Exception as e: 

yield (False, " - could not connect to db {}".format(config.db.connection_string)) 

 

errors = check("r", "extension", str, False) 

yield from errors 

 

errors = check("r", "executable", str) 

yield from errors 

 

if not errors: 

if config.r.executable == "": 

yield True, "- R not configured, skip tests" 

else: 

yield True, "- check R configuration + code execution" 

from .r_runner import RRunner 

m = RRunner(config.r.executable) 

try: 

m.start_interpreter() 

except: 

yield False, " - could not start R from {}".format(config.r.executable) 

 

errors = check("matlab", "extension", str, False) 

yield from errors 

 

errors = check("matlab", "executable", str) 

yield from errors 

if not errors: 

if config.matlab.executable == "": 

yield True, "- matlab not configured, skip tests" 

else: 

yield True, "- check matlab configuration + code execution" 

from .matlab_runner import MatlabRunner 

m = MatlabRunner(config.matlab.executable) 

try: 

m.start_interpreter() 

except: 

yield False, " - could not start matlab from {}".format(config.matlab.executable) 

 

errors = check("julia", "extension", str, False) 

yield from errors 

 

errors = check("julia", "executable", str) 

yield from errors 

 

if not errors: 

if config.julia.executable == "": 

yield True, "- julia not configured, skip tests" 

else: 

yield True, "- check julia configuration + code execution" 

from .julia_runner import JuliaRunner 

m = JuliaRunner(config.julia.executable) 

try: 

m.start_interpreter() 

except: 

yield False, " - could not start julia from {}".format(config.julia.executable) 

else: 

version = m.get_julia_version_string() 

if config.julia.version != version: 

yield False, ("- julia interpreter is of version {}, configured is {}" 

"".format(version, config.julia.version)) 

else: 

yield True, "- check julia version." 

 

errors = check("python", "extension", str, False) 

yield from errors 

 

errors = check("python", "executable", str) 

yield from errors 

 

if not errors: 

if config.python.executable == "": 

yield True, "- python not configured, skip tests" 

else: 

yield True, "- check python configuration + code execution" 

from .python_runner import PythonRunner 

m = PythonRunner(config.python.executable) 

try: 

m.start_interpreter() 

except: 

yield False, " - could not start python from {}".format(config.python.executable) 

 

errors = check("logging", "config_file", str, "") 

yield from errors 

 

# local import avoids circular import here: 

from .logger import resolve_logger_config_path 

if not errors and not os.path.exists(resolve_logger_config_path(config)): 

yield (False, 

"- configured config logging config {} does not exist".format( 

config.logging.config_file) 

) 

yield from errors 

 

errors = check("log_receiver", "port", int) 

yield from errors 

 

if not errors: 

port = config.log_receiver.port 

if _is_port_used(port): 

yield False, "- configured log_receiver port {} is already in use".format(port) 

 

errors = check("worker", "port", int) 

yield from errors 

 

if not errors: 

port = config.worker.port 

if _is_port_used(port): 

yield False, "- configured worker port {} is already in use".format(port) 

 

errors = check("worker", "count", int) 

yield from errors 

if not errors and config.worker.count <= 0: 

yield (False, 

"- configured worker count value {} is not greater than zero".format( 

config.worker.count) 

) 

 

 

def _is_port_used(port): 

import socket 

with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock: 

sock.settimeout(.5) 

return sock.connect_ex(('127.0.0.1', port)) == 0 

 

 

def datapool_folder(): 

etc = os.environ.get("ETC", "/etc") 

data_pool_folder = os.path.join(etc, "datapool") 

return data_pool_folder 

 

 

def datapool_ini_file_path(): 

folder = datapool_folder() 

path = os.path.join(folder, "datapool.ini") 

return path 

 

 

def init_config(landing_zone_folder, overwrite=False): 

 

folder = datapool_folder() 

if os.path.exists(folder): 

if not overwrite: 

raise ValueError("datapool folder {} already exists".format(folder)) 

else: 

os.makedirs(folder) 

 

path = datapool_ini_file_path() 

 

# we use OrderedDict to guarantee stable ordering of entries in config 

# file(s) 

known_settings = OrderedDict([("landing_zone.folder", landing_zone_folder)]) 

config, messages = guess_config(known_settings) 

 

write_ini(config, path) 

 

from .utils import abs_folder 

default_log_config = os.path.join(abs_folder(__file__), "cmdline_logging.yaml") 

shutil.copy(default_log_config, os.path.join(folder, config.logging.config_file)) 

 

return folder, messages 

 

 

def read_config(**variable_settings): 

path = datapool_ini_file_path() 

if os.path.exists(path): 

return read_ini(path, variable_settings) 

else: 

return None 

 

 

def write_config(config): 

assert "__file__" in config.keys() 

write_ini(config, config.__file__)