PyXMake Developer Guide  1.0
PyXMake
Utility.py
1 # -*- coding: utf-8 -*-
2 # %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
3 # % Utility - Classes and Functions %
4 # %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
5 """
6 Classes and functions defined for convenience.
7 
8 @note: PyCODAC module
9 Created on 15.07.2016
10 
11 @version: 1.0
12 ----------------------------------------------------------------------------------------------
13 @requires:
14  -
15 
16 @change:
17  -
18 
19 @author: garb_ma [DLR-FA,STM Braunschweig]
20 ----------------------------------------------------------------------------------------------
21 """
22 
23 ## @package PyXMake.Tools.Utility
24 # Module of basic functions.
25 ## @author
26 # Marc Garbade
27 ## @date
28 # 15.07.2017
29 ## @par Notes/Changes
30 # - Added documentation // mg 29.03.2018
31 
32 from future import standard_library
33 try:
34  standard_library.install_aliases()
35 except:
36  pass
37 
38 try:
39  from builtins import str
40  from builtins import object
41 except ImportError:
42  pass
43 
44 from past.builtins import long
45 from past.builtins import unicode
46 
47 import sys
48 import os
49 import io
50 import six
51 import shutil
52 import subprocess
53 import platform
54 import struct
55 import posixpath, ntpath
56 
57 import numpy as np
58 import random as rd
59 import tempfile
60 import zipfile
61 
62 try:
63  import cPickle as cp # @UnusedImport
64 except:
65  import pickle as cp # @Reimport
66 
67 try:
68  from contextlib import contextmanager # @UnusedImport
69 except ImportError:
70  from contextlib2 import contextmanager # @Reimport
71 
72 from types import MethodType
73 from .. import PyXMakePath
74 
75 ## @class PyXMake.Tools.Utility.ChangedWorkingDirectory
76 # Class to create 2to3 compatible pickling dictionary. Inherited from built-in object.
78  """
79  Context manager for temporarily changing the current working directory.
80 
81  @author: Brian M. Hunt
82  """
83  def __init__(self, newPath):
84  self.newPath = os.path.expanduser(newPath)
85  # Create target directory & all intermediate directories if don't exists
86  if not os.path.exists(self.newPath) and self.newPath != os.getcwd():
87  print("==================================")
88  print("Creating a new scratch folder @:", self.newPath)
89  print("This folder will not be deleted once the job is done!")
90  print("==================================")
91  os.makedirs(self.newPath)
92 
93  def __enter__(self):
94  self.savedPath = os.getcwd()
95  os.chdir(self.newPath)
96 
97  def __exit__(self, etype, value, traceback):
98  os.chdir(self.savedPath)
99 
100 ## @class PyXMake.Tools.Utility.GetDataFromPickle
101 # Class to create 2to3 compatible pickling dictionary. Inherited from built-in object.
102 class GetDataFromPickle(object):
103  """
104  Class to convert an arbitrary pickle file (2.x & 3.x) into a readable
105  dictionary.
106  """
107  def __init__(self, FileName):
108  """
109  Get a dictionary from a *.cpd file.
110 
111  @param: self, FileName
112  @type: self: object
113  @type: FileName: string
114  """
115  ## Dictionary for further processing.
116  self.Data = GetDataFromPickle.getDictfromFile(FileName)
117  os.remove(FileName)
118 
119  @staticmethod
120  def getDictfromFile(FileName):
121  """
122  Open a *.cpd file and extract the dictionary stored within.
123 
124  @param: FileName
125  @type: FileName: string
126  """
127  FileIn = open(FileName, "rb")
128  Dict = cp.load(FileIn)
129  FileIn.close()
130  return Dict
131 
132 ## @class PyXMake.Tools.Utility.UpdateZIP
133 # Class to create 2to3 compatible pickling dictionary. Inherited from built-in object.
134 class UpdateZIP(object):
135  """
136  Context manager for update an existing ZIP folder
137 
138  @author: Marc Garbade
139  """
140  def __init__(self, zipname, zipdata, outpath=os.getcwd(), exclude=[], update=True):
141  self.ZipName = zipname
142  self.ZipData = zipdata
143 
144  # Collect content of the ZIP folder from input
145  self.buffer = open(os.path.join(os.getcwd(), self.ZipName), 'wb+')
146  shutil.copyfileobj(self.ZipData, self.buffer)
147  self.buffer.close();
148 
149  # Initialize local variables
150  self.Output = io.BytesIO();
151  self.OutputPath = outpath
152  self.ExcludeFiles = exclude
153 
154  self.Update = update
155 
156  def __enter__(self):
157  # Extract data in current workspace and to examine its content
158  with zipfile.ZipFile(str(self.ZipName)) as Input:
159  Input.extractall()
160  os.remove(self.ZipName)
161 
162  # Do not copy input files back into the new zip folder.
163  if not self.Update:
164  self.ExcludeFiles.extend([f for f in os.listdir(os.getcwd()) if os.path.isfile(os.path.join(os.getcwd(), f))])
165 
166  def __exit__(self, etype, value, traceback):
167  # Collect all newly created files and store them in a memory ZIP folder.
168  with zipfile.ZipFile(self.Output,"w", zipfile.ZIP_DEFLATED) as Patch:
169  for dirpath, _, filenames in os.walk(os.getcwd()):
170  for f in filenames:
171  filepath = os.path.join(dirpath,f)
172  # Add all result files to the zip folder. Ignore old zip and object files
173  if not f.endswith((".zip", ".obj")) and f not in self.ExcludeFiles:
174  Patch.write(filepath, f)
175 
176  # Write content of memory ZIP folder to disk (for download). Everything else has been removed by now.
177  with open(os.path.join(self.OutputPath,self.ZipName), "wb") as f:
178  f.write(self.Output.getvalue())
179 
180 @contextmanager
181 def TemporaryDirectory(default=None):
182  """
183  Create a temporary dictionary for use with the "with" statement. Its content is deleted after execution.
184 
185  @param: default
186  @type: default: string
187  """
188  @contextmanager
189  def Changed(newdir, cleanup=lambda: True):
190  """
191  Local helper function to clean up the directory
192  """
193  prevdir = os.getcwd()
194  os.chdir(os.path.expanduser(newdir))
195  try:
196  yield
197  finally:
198  os.chdir(prevdir)
199  cleanup()
200  # Create a new temporary folder in default.
201  # Uses platform-dependent defaults when set to None.
202  dirpath = tempfile.mkdtemp(dir=default)
203  def cleanup():
204  shutil.rmtree(dirpath)
205  with Changed(dirpath, cleanup):
206  yield dirpath
207 
208 @contextmanager
209 def ConsoleRedirect(to=os.devnull, stdout=None):
210  """
211  Redirect console output to a given file.
212  """
213  def fileno(file_or_fd):
214  """
215  Small helper function to check the validity of the dump object.
216  """
217  fd = getattr(file_or_fd, 'fileno', lambda: file_or_fd)()
218  if not isinstance(fd, int):
219  raise ValueError("Expected a file (`.fileno()`) or a file descriptor")
220  return fd
221 
222  def flush(stream):
223  """
224  Also flush c stdio buffers on python 3 (if possible)
225  """
226  try:
227  import ctypes
228  from ctypes.util import find_library
229  except ImportError:
230  libc = None
231  else:
232  try:
233  libc = ctypes.cdll.msvcrt # Windows
234  except OSError:
235  libc = ctypes.cdll.LoadLibrary(find_library('c'))
236  try:
237  # Flush output associated with C/C++
238  libc.fflush(ctypes.c_void_p.in_dll(libc, 'stdout'))
239  except (AttributeError, ValueError, IOError):
240  pass # unsupported
241 
242  # Regular flush
243  stream.flush()
244  pass
245 
246  if stdout is None:
247  stdout = sys.stdout
248 
249  stdout_fd = fileno(stdout)
250  # copy stdout_fd before it is overwritten
251  #NOTE: `copied` is inheritable on Windows when duplicating a standard stream
252  with os.fdopen(os.dup(stdout_fd), stdout.mode) as copied:
253  flush(stdout) # flush library buffers that dup2 knows nothing about
254  try:
255  os.dup2(fileno(to), stdout_fd) # $ exec >&to
256  except ValueError: # filename
257  with open(to, 'wb') as to_file:
258  os.dup2(to_file.fileno(), stdout_fd) # $ exec > to
259  try:
260  yield stdout # allow code to be run with the redirected stdout
261  finally:
262  # restore stdout to its previous value
263  #NOTE: dup2 makes stdout_fd inheritable unconditionally
264  flush(stdout)
265  os.dup2(copied.fileno(), stdout_fd) # $ exec >&copied
266 
267 @contextmanager
268 def MergedConsoleRedirect(f): # $ exec 2>&1
269  """
270  Redirect all console outputs to a given stream
271  """
272  with ConsoleRedirect(to=sys.stdout, stdout=sys.stdin) as inp, ConsoleRedirect(to=sys.stdout, stdout=sys.stderr) as err, ConsoleRedirect(to=f,stdout=sys.stdout) as out:
273  yield (inp, err, out)
274 
275 @contextmanager
276 def FileOutput(FileName): # $ exec 2>&1
277  """
278  Redirect outputs to a given file.
279  """
280  if sys.version_info >= (3,4):
281  import contextlib
282  # Python 3.4 and higher
283  with open(FileName, 'w', encoding="utf-8") as f, contextlib.redirect_stdout(f), MergedConsoleRedirect(sys.stdout):
284  yield f
285  else:
286  # Lower version (unstable and deprecated)
287  with open(FileName, 'w', encoding="utf-8") as f, MergedConsoleRedirect(f):
288  yield f
289 
291  """
292  Get the PyXMake path from *__init__.
293  """
294  Path = PyXMakePath
295  return Path
296 
298  """
299  Get the underlying machine platform in lower cases.
300  """
301  return str(platform.system()).lower()
302 
304  """
305  Get the underlying machine architecture. Returns either x86 or x64 which corresponds to
306  32 or 64 bit systems.
307  """
308  if struct.calcsize("P") * 8 == 64:
309  arch = 'x64'
310  else:
311  arch = 'x86'
312  return arch
313 
314 def GetTemporaryFileName(arg=None, filename="Temp", extension=".cpd", **kwargs):
315  """
316  Create a temporary file name with extension *.cpd by default. Optional argument: Seed for random number generation.
317  """
318  if isinstance(arg, (int, long)):
319  _seed = arg
320  rd.seed(_seed)
321  randTempInteger = rd.randint(1,1000)
322  # Added backwards compatibility
323  TempFileName = filename + str(randTempInteger) + kwargs.get("ending",extension)
324  else:
325  rd.seed()
326  randTempInteger = rd.randint(1,1000)
327  # Added backwards compatibility
328  TempFileName = filename + str(randTempInteger) + kwargs.get("ending",extension)
329  return TempFileName
330 
331 def GetIterableAsList(Iterable):
332  """
333  Walk through an iterable input set and store the results in a list.
334  """
335  AsList = []
336  for i in range(0,len(Iterable)):
337  AsList.append(Iterable[i])
338  return AsList
339 
340 def IsNotEmpty(s):
341  """
342  Check whether a string is empty and/or not given. Returns True otherwise.
343  """
344  return bool(s and s.strip())
345 
346 def AsDrive(s, sep=os.path.sep):
347  """
348  Return s as drive to start an absolute path with path.join(...).
349  """
350  if sep != ntpath.sep:
351  # Linux
352  drive = posixpath.join(posixpath.sep,s)
353  else:
354  # Windows
355  drive = os.path.join(s+":",os.path.sep)
356  return drive
357 
358 def Popen(command, verbosity):
359  """
360  Run command line string "command" in a separate subprocess.
361  Show output in current console window in dependence of verbosity level:
362  - 0 --> Quiet
363  - 1 --> Only show errors
364  - 2 --> Show every command line output.
365 
366  @author: garb_ma
367  @param: command, verbosity
368  @type: string, integer
369  """
370  p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
371  stdout, stderr = p.communicate()
372  # Output standard and error messages in dependence of verbosity level.
373  if verbosity >= 2 and IsNotEmpty(stdout):
374  print((stdout.decode('utf-8',errors='ignore').replace('\n', '')))
375  if verbosity >= 1 and IsNotEmpty(stderr):
376  print((stderr.decode('utf-8',errors='ignore').replace('\n', '')))
377  # Return subprocess
378  return p
379 
380 def SSHPopen(ssh_client, command, verbosity):
381  """
382  Run command line string "command" in a separate SSH client process.
383  Show output in current console window in dependence of verbosity level:
384  - 0 --> Quiet
385  - 1 --> Only show errors
386  - 2 --> Show every command line output.
387 
388  @author: garb_ma
389  @param: command, verbosity
390  @type: string, integer
391  """
392  _, stdout, stderr = ssh_client.exec_command(command)
393  sexit = stdout.channel.recv_exit_status()
394 
395  sout = stdout.readlines()
396  serr = stderr.readlines()
397 
398  if verbosity >= 2 and sout:
399  print("".join(sout))
400  if verbosity >= 1 and serr:
401  print("".join(serr))
402 
403  return sexit
404 
405 def ConcatenateFiles(filename, files, source=os.getcwd(), ending=''):
406  """
407  Concatenate all files into one.
408  """
409  FileRead = 0
410 
411  # Concatenate all files into one temporary file
412  with open(filename,'wb') as wfd:
413  for f in [os.path.join(source,cs) if IsNotEmpty(str(os.path.splitext(cs)[1]))
414  else os.path.join(source,cs+ending) for cs in files]:
415  if FileRead > 0:
416  # Add a empty line between two source file includes
417  wfd.write("\n\n".encode())
418  with open(f,'rb') as fd:
419  shutil.copyfileobj(fd, wfd, 1024*1024*10)
420  FileRead += 1
421 
422 def ReplaceTextinFile(filename, outname, replace, inend='', outend='', source=os.getcwd()):
423  """
424  Replace all occurrences of replace in filename.
425  """
426  Inputfile = os.path.join(source,filename+inend)
427  Outputfile = outname+outend
428  with open(Inputfile) as infile, open(Outputfile, 'w') as outfile:
429  for line in infile:
430  for src, target in replace.items():
431  line = line.replace(src, target)
432  outfile.write(line)
433 
434 def DeleteFilesbyEnding(identifier):
435  """
436  Delete all files from workspace
437 
438  @author: Marc Garbade, 26.02.2018
439 
440  @param: identifier: A tuple specifying the files to remove.
441  @type: Tuple
442  """
443  for f in os.listdir(os.getcwd()):
444  if f.endswith(identifier):
445  os.remove(f)
446 
447 def AddFunctionToObject(_func, _obj):
448  """
449  Bind a function to an existing object.
450  """
451  return MethodType(_func, _obj)
452 
454  """
455  Prepare a object for pickling and convert all numpy
456  arrays to python defaults (2to3 compatible).
457  """
458  _dictobj = _obj.__dict__.copy()
459  _dictobj['__np_obj_path'] = []
460  for path, value in ObjectWalk(_dictobj):
461  if isinstance(value, (np.ndarray, np.generic)):
462  parent = _dictobj
463  for step in path[:-1]:
464  parent = parent[step]
465  parent[path[-1]] = value.tolist()
466  _dictobj['__np_obj_path'].append(path[-1])
467  return _dictobj
468 
470  """
471  Restore the original dictionary by converting python defaults to their
472  numpy equivalents if required (2to3 compatible).
473  """
474  _dictobj = _dict.copy()
475  # Recreate serialized arrays accordingly
476  for key, value in _dictobj.items():
477  if key in _dictobj['__np_obj_path']:
478  _dictobj[key] = np.float64(value)
479  if len(np.shape(value)) == 2:
480  _dictobj[key] = np.asmatrix(value)
481  else:
482  pass
483  else:
484  pass
485  return _dictobj
486 
487 def PathLeaf(path):
488  """
489  Return the last item of an arbitrary path (its leaf).
490  """
491  head, tail = ntpath.split(path)
492  return tail or ntpath.basename(head)
493 
494 def ArbitraryFlattening(container):
495  """
496  Restore the original dictionary by converting python defaults to their
497  numpy equivalents if required (2to3 compatible).
498  """
499  for i in container:
500  if isinstance(i, (list,tuple, np.ndarray)):
501  for j in ArbitraryFlattening(i):
502  yield j
503  else:
504  yield i
505 
506 def FileWalk(source, path=os.getcwd()):
507  """
508  Walk recursively through path. Check if all files listed in source are present.
509  If True, return them. If False, return all files present in the given path.
510  """
511  # Initialize all variables
512  files = list([])
513  # These are all uploaded files
514  InputFiles = [f for f in os.listdir(path) if os.path.isfile(os.path.join(path, f))]
515  try:
516  # Check if source code files have been specified. Otherwise, use everything from the ZIP file
517  if all(np.array([len(source),len(InputFiles)]) >= 2):
518  # Ignore file extension here. This will interfere with the checks later on.
519  if set([os.path.splitext(x)[0] for x in source]).issubset(set([os.path.splitext(f)[0] for f in InputFiles])):
520  files.extend(source)
521  else:
522  pass
523  else:
524  files.extend(InputFiles)
525  except TypeError:
526  files.extend(InputFiles)
527  pass
528 
529  return files
530 
531 def PathWalk(path, exclude=False, startswith=None, endswith=None, contains=None):
532  """
533  Walk recursively through path. Exclude both folders and files if requested.
534  """
535  def Skip(x, starts, contains, ends):
536  """
537  Evaluate skip condition
538  """
539  if isinstance(starts, (six.string_types, tuple)):
540  if x.startswith(starts):
541  return True
542  if isinstance(contains, (six.string_types, tuple, list)):
543  tmp = list([]); tmp.append(contains)
544  tmp = tuple(ArbitraryFlattening(tmp))
545  if any(s in x for s in tmp):
546  return True
547  if isinstance(ends, (six.string_types, tuple)):
548  if x.endswith(ends):
549  return True
550  return False
551 
552  for root, dirs, files in os.walk(os.path.normpath(path)):
553  if exclude:
554  files = [f for f in files if not Skip(f,startswith,contains,endswith)]
555  dirs[:] = [d for d in dirs if not Skip(d,startswith,contains,endswith)]
556  yield root, dirs, files
557 
558 def ObjectWalk(obj, path=(), memo=None):
559  """
560  Walk recursively through nested python objects.
561 
562  @author: Yaniv Aknin, 13.12.2011
563 
564  @param: obj, path, memo
565  @type: object, list, boolean
566  """
567  string_types = (str, unicode)
568  iteritems = lambda mapping: getattr(mapping, 'iteritems', mapping.items)()
569  if memo is None:
570  memo = set()
571  iterator = None
572  if isinstance(obj, dict):
573  iterator = iteritems
574  elif isinstance(obj, (list, set)) and not isinstance(obj, string_types):
575  iterator = enumerate
576  if iterator:
577  if id(obj) not in memo:
578  memo.add(id(obj))
579  for path_component, value in iterator(obj):
580  for result in ObjectWalk(value, path + (path_component,), memo):
581  yield result
582  memo.remove(id(obj))
583  else:
584  yield path, obj
585 
586 if __name__ == '__main__':
587  pass
def RecoverDictionaryfromPickling(_dict)
Definition: Utility.py:469
def ConsoleRedirect(to=os.devnull, stdout=None)
Definition: Utility.py:209
def AsDrive(s, sep=os.path.sep)
Definition: Utility.py:346
def SSHPopen(ssh_client, command, verbosity)
Definition: Utility.py:380
def FileOutput(FileName)
Definition: Utility.py:276
Data
Dictionary for further processing.
Definition: Utility.py:116
def AddFunctionToObject(_func, _obj)
Definition: Utility.py:447
Class to create 2to3 compatible pickling dictionary.
Definition: Utility.py:134
def TemporaryDirectory(default=None)
Definition: Utility.py:181
def GetIterableAsList(Iterable)
Definition: Utility.py:331
def DeleteFilesbyEnding(identifier)
Definition: Utility.py:434
Class to create 2to3 compatible pickling dictionary.
Definition: Utility.py:77
def PathLeaf(path)
Definition: Utility.py:487
Class to create 2to3 compatible pickling dictionary.
Definition: Utility.py:102
def GetTemporaryFileName(arg=None, filename="Temp", extension=".cpd", kwargs)
Definition: Utility.py:314
def ObjectWalk(obj, path=(), memo=None)
Definition: Utility.py:558
def ConcatenateFiles(filename, files, source=os.getcwd(), ending='')
Definition: Utility.py:405
def ReplaceTextinFile(filename, outname, replace, inend='', outend='', source=os.getcwd())
Definition: Utility.py:422
def ArbitraryFlattening(container)
Definition: Utility.py:494
def MergedConsoleRedirect(f)
Definition: Utility.py:268
def PrepareObjectforPickling(_obj)
Definition: Utility.py:453
def FileWalk(source, path=os.getcwd())
Definition: Utility.py:506
def Popen(command, verbosity)
Definition: Utility.py:358
def PathWalk(path, exclude=False, startswith=None, endswith=None, contains=None)
Definition: Utility.py:531