PyXMake Developer Guide  1.0
PyXMake
Make.py
1 # -*- coding: utf-8 -*-
2 # %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
3 # % Make Module - Classes and Functions %
4 # %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
5 """
6 Create a make object to define the building environment and to execute the
7 build commands. The make event is subdivided in a pre-, main- and a post-build
8 event.
9 
10 @note: PyXMake module
11 Created on 20.03.2018
12 
13 @version: 1.0
14 ----------------------------------------------------------------------------------------------
15 @requires:
16  -
17 
18 @change:
19  -
20 
21 @author: garb_ma [DLR-FA,STM Braunschweig]
22 ----------------------------------------------------------------------------------------------
23 """
24 
25 ## @package PyXMake.Build.Make
26 # Create a make object to define the building environment.
27 ## @author
28 # Marc Garbade
29 ## @date
30 # 20.03.2018
31 ## @par Notes/Changes
32 # - Added documentation // mg 29.03.2018
33 try:
34  from builtins import object
35 except ImportError:
36  pass
37 
38 import sys, os, platform
39 import paramiko
40 import abc, six
41 import shutil
42 import shlex
43 import posixpath
44 import random, string
45 import subprocess
46 import tempfile
47 import colorsys
48 
49 import numpy as np
50 
51 from shutil import copyfile
52 from types import MethodType
53 from collections import OrderedDict
54 from PIL import ImageColor
55 
56 from ..Tools import Utility
57 
58 ## Absolute system path to PyXMake.
59 PyXMakePath = Utility.GetPyXMakePath()
60 ## Absolute system path to configuration files.
61 Path2Config = os.path.join(PyXMakePath,"Build","config")
62 
63 ## @class PyXMake.Build.Make.Make
64 # Abstract base class for all make objects. Inherited from built-in ABCMeta & object.
65 # Compatible with both Python 2.x and 3.x.
66 @six.add_metaclass(abc.ABCMeta)
67 class Make(object):
68  """
69  Parent class for all make objects.
70  """
71  @abc.abstractmethod
72  def __init__(self, BuildID, Srcs, scratch=os.getcwd(), msvsc='vs2015', stype='Fortran', verbose=0, *args, **kwargs):
73  """
74  Low-level initialization of parent class.
75  """
76  ## Base string of build object.
77  # Defines the base string of the current build object. The final build name
78  # used in the instanced objects is assembled using this immutable base id.
79  self.buildid = BuildID
80  ## Source file or folders
81  self.srcs = []
82  self.srcs.append(Srcs)
83  self.srcs = list(Utility.ArbitraryFlattening(self.srcs))
84  ## Source file type
85  self.stype = stype
86  ## Level of verbosity of the current build object.
87  # Define the verbosity level of the current build object. Defaults to 0 and suppresses
88  # all outputs to the command line. A higher value increases the level of verbosity up to
89  # a maximum level of 2.
90  self.verbose = verbose
91 
92  # Set scratch directory & default input/ output locations
93  with Utility.ChangedWorkingDirectory(scratch):
94  ## Current scratch directory
95  self.scrtdir = os.getcwd()
96  ## Default search directory for source files.
97  self.srcdir = os.getcwd()
98  ## Default search directory for output.
99  self.outdir = os.getcwd()
100 
101  # Read Intel path from Paths.log
102  with open(os.path.join(PyXMakePath,'Paths.log')) as f:
103  content = f.readlines()
104  content = [x.strip() for x in content]
105  ## Path to Intel Fortran Compiler (read from Paths.log).
106  self.intelpath = content[20]
107 
108  # Initialization of tuple containing temporary files
109  ## Tuple of data to be removed after job completion.
110  self.temps = ()
111 
112  # Initialization of lists containing additional sources, modules or libraries
113  ## List of include directories.
114  self.incdirs = []
115  ## List of library directories.
116  self.libdirs = []
117  ## List of actual libraries (by name) used during linking.
118  self.libs = []
119 
120  # Initialization of list containing files to be copied to the output directory.
121  ## List of files to be copied to the output directory after finish.
122  self.copyfiles = []
123 
124  ## Define the architecture for the build directly by using the keyword argument "arch".
125  # Defaults to None, in which case the architecture is determined by using the python executable.
126  self.setarch = True if kwargs.get('arch', None) in ['x86', 'x64'] else False
127 
128  # Set variables in dependence of identified system
129  if 'win' in sys.platform:
130 
131  if (('32bit' in platform.architecture() and not self.setarch) or (self.setarch and kwargs.get('arch', None) == "x86")):
132  # Set variables for x86
133  os.environ['pyx_proc'] = 'x86'
134  os.environ['pyx_intel'] = 'ia32'
135  ## Processor architecture
136  self.architecture = 'x86'
137 
138  elif (('64bit' in platform.architecture() and not self.setarch) or (self.setarch and kwargs.get('arch', None) == "x64")):
139  # Set variables for x64
140  os.environ['pyx_proc'] = 'amd64'
141  os.environ['pyx_intel'] = 'intel64'
142  self.architecture = 'x64'
143 
144  ## Executable batch script (including absolute system path) to set up the Intel Fortran Compiler.
145  self.iniCompiler = r'"'+os.path.join(PyXMakePath,"Build","cmd","windows","iniCompiler.bat")+'"'
146 
147  elif 'linux' in sys.platform:
148  raise NotImplementedError
149 
150  ## Default version of Microsoft visual studio used by the Intel Fortran Compiler. Defaults to 'vs2010'.
151  self.msvsc = msvsc
152  os.environ['pyx_msvsc'] = self.msvsc
153 
154  ## Post build command. Defaults to an empty string.
155  self.postcmd = ""
156 
157  # Always add MKL library
158  ## Absolute system path of MKL library.
159  mkl_include_path = os.path.join(self.intelpath,"mkl","include")
160  mkl_lib_path = os.path.join(self.intelpath,"mkl","lib",os.environ['pyx_intel'])
161  mkl_compiler_path = os.path.join(self.intelpath,"compiler","lib",os.environ['pyx_intel'])
162 
163  self.incdirs.append(mkl_include_path)
164  self.libdirs.extend([mkl_lib_path, mkl_compiler_path])
165 
166  # Update environment build path
167  os.environ["PATH"] = ";".join([os.getenv("PATH",""),os.path.join(self.intelpath,"bin",os.environ['pyx_intel'])])
168 
169  # Add all source files to list of files to remove from scratch folder. No exceptions.
170  self.temps = self.temps + tuple((os.path.join(self.scrtdir,x) for x in self.srcs))
171  pass
172 
173  def __getstate__(self):
174  """
175  Prepare the object for pickling (2to3 compatible)
176  """
177  _dictobj = Utility.PrepareObjectforPickling(self)
178  return _dictobj
179 
180  def __setstate__(self, _dict):
181  """
182  Recover a dictionary from pickling (2to3 compatible)
183  """
184  _dictobj = Utility.RecoverDictionaryfromPickling(_dict)
185  self.__dict__.update(_dictobj)
186  pass
187 
188  def AddIncludePath(self, includes):
189  """
190  Define additional include directories containing modules or source files as comma separated list.
191  """
192  self.incdirs.append(includes)
193  self.incdirs = list(Utility.ArbitraryFlattening(self.incdirs))
194  pass
195 
196  def AddDependencyPath(self, dependencies):
197  """
198  Define additional directories containing 3rd party libraries as comma separated list.
199  """
200  self.libdirs.append(dependencies)
201  self.libdirs = list(Utility.ArbitraryFlattening(self.libdirs))
202  pass
203 
204  def UseLibraries(self, libs):
205  """
206  Define which non-default libraries should be used during linking.
207  """
208  self.libs.append(libs)
209  self.libs = list(Utility.ArbitraryFlattening(self.libs))
210  pass
211 
212  def SourcePath(self, path):
213  """
214  Define a new source directory. Input is read from workspace by default.
215  """
216  self.srcdir = path
217  pass
218 
219  def OutputPath(self, path, files=""):
220  """
221  Define a new output directory. Output is written to the workspace by default.
222  """
223  self.outdir = path
224  ## List of files copied to the output directory.
225  self.copyfiles.append(files)
226  self.copyfiles = list(Utility.ArbitraryFlattening(self.copyfiles))
227  pass
228 
229  def Environment(self, path, script="ifortvars.bat"):
230  """
231  Load an additional environment file prior to execution of all commands.
232  """
233  ## Execute an additional bash script prior to all build commands.
234  os.environ['pyx_environment'] = os.path.join(path,script)
235  if path.rsplit("\\",1)[1] == "bin":
236  self.intelpath = path[::-1].replace("bin"[::-1],"",1)[::-1]
237  else:
238  self.intelpath = path
239 
240  # Update static MKL library include.
241  for inc in self.incdirs:
242  if all(x in inc for x in ["mkl","include"]):
243  self.incdirs.remove(inc)
244 
245  # Update static MKL library.
246  for lib in self.libdirs:
247  if all(x in lib for x in ["mkl", "lib"]) or all(x in lib for x in ["compiler","lib"]):
248  self.libdirs.remove(lib)
249 
250  # Redefine MKL paths
251  mkl_include_path = os.path.join(self.intelpath,"mkl","include")
252  mkl_lib_path = os.path.join(self.intelpath,"mkl","lib",os.environ['pyx_intel'])
253  mkl_compiler_path = os.path.join(self.intelpath,"compiler","lib",os.environ['pyx_intel'])
254  # Add newly created path to library path.
255  self.incdirs.append(mkl_include_path)
256  self.libdirs.extend([mkl_lib_path, mkl_compiler_path])
257  pass
258 
259  def Preprocessing(self, cmdstring='', inend='', outend='', copyfiles=[],
260  replace = {'!DEC$ IF':'#IF','!DEC$ ELSE':'#ELSE','!DEC$ ENDIF':'#ENDIF'}):
261  """
262  Assemble command string for the pre-build event.
263  """
264  # Add file extension to output file name (if not already been done)
265  if not Utility.IsNotEmpty(os.path.splitext(self.buildname)[1]):
266  self.buildname = self.buildname+outend
267 
268  # Go into scratch directory (if defined)
269  with Utility.ChangedWorkingDirectory(self.scrtdir):
270  # Create two temporary file names.
271  collsrcfile = Utility.GetTemporaryFileName(filename="coll" + "_", extension=inend)
272  presrcfile = Utility.GetTemporaryFileName(filename="pre",extension=outend)
273 
274  # Create a list of all directories to be searched
275  search_dir = [self.srcdir]
276  try:
277  search_dir.extend(self.incdirs)
278  except:
279  pass
280 
281  # Copy all relevant files from the source folder into the current scratch folder.
282  if copyfiles != []:
283  for subdir in search_dir:
284  src_files = os.listdir(subdir)
285  file_names = [x for x in src_files if x in copyfiles]
286  for inputfile in file_names:
287  full_file_name = os.path.join(subdir, inputfile)
288  if (os.path.isfile(full_file_name)):
289  shutil.copy(full_file_name, os.getcwd())
290  self.temps = self.temps + (inputfile, )
291  # Rename source code file
292  else:
293  Utility.ConcatenateFiles(self.buildname,self.srcs,self.srcdir, inend)
294 
295  # Do not apply any pre-processing (required for e.g. Beos)
296  if cmdstring != '':
297  # Concatenate all source files into one temporary file
298  Utility.ConcatenateFiles(collsrcfile, self.srcs, self.srcdir, ending=inend)
299 
300  # Replace pre-processing commands
301  Utility.ReplaceTextinFile(collsrcfile, presrcfile, replace, source=self.scrtdir)
302 
303  # Assemble command string
304  cmdstring += ' %s %s' % (presrcfile, self.buildname)
305 
306  # Store command string in object
307  ## Command executed during pre-build event.
308  self.precmd = self.iniCompiler+" "+cmdstring
309 
310  # Add temporary files to tuple scheduled for removal
311  self.temps = self.temps + (collsrcfile, presrcfile, self.buildname, )
312  pass
313 
314  def Build(self, cmdstring):
315  """
316  Assemble command string for the main build event.
317  """
318  # Initialize command string
319  cmd = ""
320 
321  # Add all include paths to the command string
322  includes = ['-I"'+x+'" ' for x in self.incdirs]
323  for x in includes:
324  cmd += x
325 
326  # Add all dependency paths to the command string
327  dependencies = ['-L"'+x+'" ' for x in self.libdirs]
328  for x in dependencies:
329  cmd += x
330 
331  # Add all required libraries to the command string
332  libs = ['-l'+x+' ' for x in self.libs]
333  for x in libs:
334  cmd += x
335  ## Command line arguments passed in by the user.
336  self.compargs = cmdstring
337  ## Command executed during build event.
338  self.makecmd = self.iniCompiler+" "+os.path.join(self.path2exe,self.exe)+" "+ cmd + cmdstring
339  pass
340 
341  def Postprocessing(self, cmdstring=''):
342  """
343  Assemble command string for the post-build event.
344  """
345  ## Command executed during post-build event.
346  self.postcmd = self.iniCompiler+" "+cmdstring
347  pass
348 
349  def create(self):
350  """
351  Execute make command
352  """
353  # Go into scratch directory (if defined)
354  with Utility.ChangedWorkingDirectory(self.scrtdir):
355 
356  # Pre-build event (if required)
357  try:
358  if self.precmd != '':
359  Utility.Popen(self.precmd, self.verbose)
360  except:
361  pass
362 
363  # Build event (if required)
364  try:
365  if self.makecmd != '':
366  # Execute again to account for input added after compiling command
367  self.Build(self.compargs)
368  Utility.Popen(self.makecmd, self.verbose)
369  except:
370  pass
371 
372  # Post-build event (if required)
373  try:
374  if self.postcmd != '':
375  Utility.Popen(self.postcmd, self.verbose)
376  except:
377  pass
378 
379  # Copy files to predefined output location. Add the original to list of redundant files.
380  # Copy only those files which are not temporary and whose name includes the BuildID.
381  # If a list of files if given, use only those files presented in the list. Add to other files to list
382  # of redundant files.
383  try:
384  if self.outdir != os.getcwd():
385  for f in os.listdir(os.getcwd()):
386  if self.architecture in f and f != self.buildname and f not in self.temps:
387  if self.copyfiles[0] == "":
388  copyfile(os.path.join(os.getcwd(),f), os.path.join(self.outdir,f))
389  self.temps = self.temps + (f, )
390  if f in self.copyfiles:
391  copyfile(os.path.join(os.getcwd(),f), os.path.join(self.outdir,f))
392  self.temps = self.temps + (f, )
393  elif f not in self.copyfiles and self.copyfiles[0] != "" and not os.path.isdir(f):
394  self.temps = self.temps + (f, )
395  except:
396  pass
397 
398  # Finish and delete redundant files
399  Utility.DeleteFilesbyEnding(self.temps)
400  pass
401 
402 ## @class PyXMake.Build.Make.Custom
403 # Base class for all custom build events inherited from Make.
404 class Custom(Make):
405  """
406  Inherited class to build projects without any presets.
407  """
408  def __init__(self, *args, **kwargs):
409  """
410  Initialization of Custom class object.
411  """
412  super(Custom, self).__init__(*args, **kwargs)
413  ## String identifier of current instance.
414  self.MakeObjectKind = 'Custom'
415 
416  ## The executable command used in all build events.
417  self.exe = "cmd.exe /c &&"
418 
419  ## Immutable settings for Custom object.
420  # Temporary build name, assembled using BuildID.
421  self.buildname = self.buildid + self.architecture
422 
423  # Set environment variables for ABAQUS builds (Defaults to latest version).
424  os.environ["ABQ_ENV_FILE"] = "abaqus_v6.env"
425  os.environ["pyx_abaqus"] = os.getenv("pyx_abaqus","abaqus")
426 
427  # Add temporary files to tuple scheduled for removal.
428  self.temps = self.temps + (os.getenv("ABQ_ENV_FILE"), )
429  pass
430 
431  def Preprocessing(self, cmdstring='', inend='', outend='', replace = {'!DEC$ IF':'#IF','!DEC$ ELSE':'#ELSE','!DEC$ ENDIF':'#ENDIF'}):
432  """
433  Assemble command string for the pre-build event.
434  """
435  # Add file extension of build name.
436  self.buildname = self.buildname+outend
437 
438  # Go into scratch directory (if defined)
439  with Utility.ChangedWorkingDirectory(self.scrtdir):
440 
441  # Create two temporary file names.
442  collsrcfile = Utility.GetTemporaryFileName(filename="coll" + "_", extension=inend)
443  presrcfile = Utility.GetTemporaryFileName(filename="pre",extension=outend)
444 
445  # Concatenate all source files into one temporary file
446  Utility.ConcatenateFiles(collsrcfile,self.srcs, self.srcdir, inend)
447 
448  # Replace pre-processing commands
449 
450  Utility.ReplaceTextinFile(collsrcfile, presrcfile, replace, source=self.scrtdir)
451 
452  # Assemble command string
453  cmdstring += ' %s %s' % (presrcfile, self.buildname)
454 
455  # Store command string in object
456  ## Command executed during pre-build event.
457  self.precmd = self.iniCompiler+" "+cmdstring
458 
459  ## Command line arguments passed in by the user.
460  self.compargs = ""
461 
462  # Add temporary files to tuple scheduled for removal
463  self.temps = self.temps + (collsrcfile, presrcfile)
464  pass
465 
466  def Build(self, cmdstring):
467  """
468  Assemble command string for the main build event.
469  """
470  # Do this part only once!
471  if self.compargs == cmdstring:
472  # Add all include paths to the include environment variable
473  os.environ["INCLUDE"] = os.pathsep.join(list(self.incdirs)) + os.pathsep + os.pathsep.join((os.environ.get("MSMPI_INC",""), os.environ.get("INCLUDE","")))
474 
475  # Add all additional library paths to the lib environment variable
476  os.environ["LIB"] = os.pathsep.join(self.libdirs) + os.pathsep + os.pathsep.join((os.environ.get("MSMPI_LIB64",""), os.environ.get("LIB","")))
477 
478  # Add scratch and sources to path environment variable
479  os.environ["PATH"] = os.pathsep.join([self.srcdir,self.scrtdir]) + os.pathsep + os.getenv("PATH","")
480 
481  # Add pre-processed source file to environment variable
482  os.environ["pyx_file"] = self.buildname
483 
484  # Add exception to pyx_libs when ABAQUS build command is used with vs2015 and higher.
485  if self.msvsc.lower() in ["vs2015", "vs2017"]:
486  self.libs.extend(["msvcrt", "vcruntime", "ucrt", "legacy_stdio_definitions"])
487 
488  # Add all libraries to a environment variable
489  pyx_libs = ["'"+x+".lib'" for x in self.libs]
490 
491  # Set environment variable
492  os.environ["pyx_libs"] = ",".join(pyx_libs)
493 
494  ## Command line arguments passed in by the user.
495  self.compargs = cmdstring
496  ## Command executed during build event.
497  self.makecmd = self.iniCompiler+" "+self.exe+" "+cmdstring
498 
499  # Add temporary files to tuple scheduled for removal
500  self.temps = self.temps + (self.buildname, )
501  pass
502 
503 ## @class PyXMake.Build.Make.CCxx
504 # Base class for all C/C++ build events inherited from Make.
505 class CCxx(Make):
506  """
507  Inherited class to build projects using Intel C/C++.
508  """
509  def __init__(self, *args, **kwargs):
510  """
511  Initialization of C/C++ class object.
512  """
513  super(CCxx, self).__init__(*args, **kwargs)
514  ## String identifier of current instance.
515  self.MakeObjectKind = 'CCxx'
516 
517  ## The executable command used in the main build event.
518  self.exe = 'cl.exe'
519 
520  ## Static or dynamic link library flag.
521  self.isStatic = True if kwargs.get('lib', 'static') not in ['shared', 'SHARED', 'Shared'] else False
522 
523  ## Define if the input should be compiled exactly as provided.
524  # Defaults to False, meaning that merging & pre-processing utilities will be carried out.
525  self.incremental = kwargs.get('incremental', False)
526 
527  # Immutable settings for C/Cpp object
528  if self.incremental:
529  self.exe += ' -c %s' % (' '.join(self.srcs))
530  else:
531  self.exe += ' -c %s' % (self.buildname)
532 
533  ## Name of library, assembled using BuildID.
534  self.libname = self.buildid + self.architecture
535  ## Temporary build name.
536  self.buildname = self.libname
537 
538  # Initialization of lists containing additional sources, modules or libraries
539  ## List of libraries which should be statically linked in.
540  self.linkedIn = []
541 
542  # Initialization of tuple containing temporary files
543  ## Blank version of tuple to store temporary file names scheduled for removal.
544  self.temps = ()
545 
546  # Remove MKL from default command line
547  ## Blank version of list containing library directories without initially specifying MKL.
548  self.libdirs = []
549 
550  # Always add conversion headers to the default make directory
551  self.incdirs.append(os.path.join(PyXMakePath,"VTL","make"))
552  pass
553 
554  def OutputPath(self, libpath=os.getcwd()):
555  """
556  Define output directories for modules and libraries.
557  """
558  ## Output path for library files.
559  self.outlibs = libpath
560  pass
561 
562  def Build(self, cmdstring):
563  """
564  Assemble command strings for the main build event.
565  """
566  # Initialize command string
567  cmd = ""#' -module:'+self.outmodule+' -I:"'+self.outmodule+'"'
568 
569  # Add all include paths to the command string
570  includes = [' -I"'+x+'" ' for x in self.incdirs]
571  for x in includes:
572  cmd += x
573 
574  # Choose the librarian and the file extension of the library.
575  if not self.isStatic:
576  librarian = 'link -dll -fixed:no -defaultlib:libcmt.lib -nodefaultlib:msvcrt.lib '
577  ext = '.dll'
578  else:
579  librarian = 'lib '
580  ext = '.lib'
581 
582  # Build commands using Intel Fortran (immutable)
583  ## Used defined command line options.
584  self.compargs = cmdstring
585  ## Intel Compiler command.
586  self.makecmd = self.iniCompiler+" "
587  self.makecmd += self.exe + cmd + cmdstring + ' && '
588  ## Intel Linker command.
589  self.linkcmd = librarian +'*.obj -nologo -machine:'+self.architecture+' -out:'+os.path.join(self.outlibs,self.libname+ext)
590 
591  # Add temporary files to tuple scheduled for removal
592  self.temps = self.temps + (self.libname+'.obj', ".obj",)
593  pass
594 
595  def create(self):
596  """
597  Execute make command
598  """
599  cmd = ''
600 
601  # Go into scratch directory (if defined)
602  with Utility.ChangedWorkingDirectory(self.scrtdir):
603 
604  # Pre-build event (if required)
605  try:
606  if self.precmd != '':
607  Utility.Popen(self.precmd, self.verbose)
608  except:
609  pass
610 
611  # Add all dependency paths to the command string
612  dependencies = [' -LIBPATH:"'+x+'" ' for x in self.libdirs]
613  for y in dependencies:
614  cmd += y
615 
616  # Add libraries for linking to the command string
617  linklist = [' '+x+'.lib ' for x in self.libs]
618  for x in linklist:
619  cmd += x
620 
621  # Build event (if required)
622  try:
623  if self.makecmd != '':
624  # Execute again to account for input added after compiling command
625  self.Build(self.compargs)
626  Utility.Popen(self.makecmd+self.linkcmd+cmd, self.verbose)
627  except:
628  pass
629 
630  # Post-build event (if required)
631  try:
632  if self.postcmd != '':
633  Utility.Popen(self.postcmd, self.verbose)
634  except:
635  pass
636 
637  # Finish and delete redundant files
638  if not self.isStatic:
639  os.remove(os.path.join(self.outlibs,self.libname+'.exp'))
640  os.remove(os.path.join(self.outlibs,self.libname+'.lib'))
641  Utility.DeleteFilesbyEnding(self.temps)
642 
643  pass
644 
645 ## @class PyXMake.Build.Make.Fortran
646 # Base class for all Fortran build events. Inherited from Make.
647 class Fortran(Make):
648  """
649  Inherited class to build projects using Intel Fortran.
650  """
651  def __init__(self, *args, **kwargs):
652  """
653  Initialization of Fortran class object.
654  """
655  super(Fortran, self).__init__(*args, **kwargs)
656  ## String identifier of current instance.
657  self.MakeObjectKind = 'Fortran'
658 
659  ## Static or dynamic link library flag.
660  self.isStatic = True if kwargs.get('lib', 'static') not in ['shared', 'SHARED', 'Shared'] else False
661 
662  # Add static MKL libraries when creating a shared resource library (for Java applications).
663  if not self.isStatic:
664  if self.architecture == "x86":
665  mkl_interface_lib = "mkl_intel_c"
666  else:
667  mkl_interface_lib = "mkl_intel_lp64"
668 
669  # Always link statically against MKL library
670  self.libs.append([mkl_interface_lib, "mkl_intel_thread", "mkl_core","libiomp5md"])
671  self.libs = list(Utility.ArbitraryFlattening(self.libs))
672  pass
673 
674  # Immutable settings for Fortran object
675  ## Name of library, assembled using BuildID.
676  self.libname = self.buildid + self.architecture
677  ## Temporary build name.
678  self.buildname = self.libname
679 
680  # Activate / deactivate incremental linking
681  self.incremental = kwargs.get("incremental",False)
682 
683  # Defined here to be checked later.
684  ## Wrapper interface file for 3rd party FORTRAN code. Automatically creates a module of the underlying source material.
686  self.wrapper_source = ""
687  self.wrapper_module = "pyx_module.f90"
688 
689  # Initialization of lists containing additional sources, modules or libraries
690  ## List of libraries which should be statically linked in.
691  self.linkedIn = []
692 
693  # Initialization of tuple containing temporary files
694  ## Blank version of tuple to store temporary file names scheduled for removal.
695  self.temps = ()
696 
697  # Remove MKL from default command line
698  ## Blank version of list containing library directories without initially specifying MKL.
699  self.libdirs = []
700  pass
701 
702  def OutputPath(self, modulepath=None, libpath=os.getcwd()):
703  """
704  Define output directories for modules and libraries.
705  """
706  # Output module files to scratch directory by default.
707  if modulepath is None:
708  modulepath = self.scrtdir
709  ## Output path for module or header files.
710  self.outmodule = modulepath
711  ## Output path for library files.
712  self.outlibs = libpath
713  pass
714 
715  def Preprocessing(self, inend='', outend='', copyfiles=[],
716  replace = {'!DEC$ IF':'#IF','!DEC$ ELSE':'#ELSE','!DEC$ ENDIF':'#ENDIF'},
717  decorator="!DEC$ ATTRIBUTES DLLEXPORT::"):
718  """
719  Assemble command string for the pre-build event.
720  """
721  # Avoid false positives when created a shared resource library.
722  validater ="bind"
723 
724  # Add file extension of build name
725  self.buildname = self.buildname+outend
726 
727  # Go into scratch directory (if defined)
728  with Utility.ChangedWorkingDirectory(self.scrtdir):
729 
730  # Create a list of all directories to be searched
731  search_dir = [self.srcdir]
732  try:
733  search_dir.extend(self.incdirs)
734  except:
735  pass
736 
737  # Copy all relevant files from the source folder into the current scratch folder.
738  if copyfiles != []:
739  for subdir in search_dir:
740  src_files = os.listdir(subdir)
741  file_names = [x for x in src_files if x in copyfiles]
742  #print(file_names)
743  for inputfile in file_names:
744  full_file_name = os.path.join(subdir, inputfile)
745  if (os.path.isfile(full_file_name)):
746  shutil.copy(full_file_name, os.getcwd())
747  self.temps = self.temps + (inputfile, )
748  else:
749  # Create two temporary file names.
750  collsrcfile = Utility.GetTemporaryFileName(filename="coll" + "_", extension=inend)
751  presrcfile = Utility.GetTemporaryFileName(filename="pre",extension=outend)
752 
753  # Concatenate all source files into one temporary file
754  Utility.ConcatenateFiles(collsrcfile, self.srcs, self.srcdir, inend)
755 
756  # Replace pre-processing commands
757  Utility.ReplaceTextinFile(collsrcfile, presrcfile, replace, source=self.scrtdir)
758 
759  # Check if decorators have to be added to the source file.
760  Inputfile = os.path.join(os.getcwd(),presrcfile)
761  Outputfile = os.path.join(os.getcwd(),self.buildname)
762  with open(Inputfile) as infile, open(Outputfile, 'w') as outfile:
763  for line in infile:
764  outfile.write(line)
765  if not self.isStatic:
766  xname, xline = line.partition('="')[0], line.partition('="')[2]
767  if xline != '' and validater in xname.lower():
768  outfile.write(decorator+xline.partition('"')[0]+"\n")
769  self.temps = self.temps + (collsrcfile, presrcfile, )
770 
771  # Add temporary files to tuple scheduled for removal
772  self.temps = self.temps + (self.buildname, )
773  pass
774 
775  def Wrapper(self, module_name, source_name=None):
776  """
777  Assemble command string for the pre-build event.
778  """
779  # Add module wrapper to the default make directory
780  makedir = os.path.join(PyXMakePath,"VTL","make")
781  TmpFile = Utility.GetTemporaryFileName(extension=str(os.path.splitext(self.wrapper_module)[1]))
782  copyfile(os.path.join(makedir,self.wrapper_module), os.path.join(self.scrtdir,TmpFile))
783 
784  if source_name:
785  self.wrapper_source = source_name
786 
787  self.intermediate_wrapper = Utility.GetTemporaryFileName(extension=str(os.path.splitext(TmpFile)[1]))
788 
789  # Go into scratch directory (if defined)
790  with Utility.ChangedWorkingDirectory(self.scrtdir):
791  # Prepare wrapper module for later use
792  Utility.ReplaceTextinFile(TmpFile, self.intermediate_wrapper, {'%pyx_module%':module_name}, source=self.scrtdir)
793 
794  # Add temporary files to tuple scheduled for removal
795  self.temps = self.temps + (TmpFile, self.intermediate_wrapper, self.wrapper_module, )
796  pass
797 
798  def Build(self, cmdstring):
799  """
800  Assemble command strings for the main build event.
801  """
802  sep = " "; multi_objects = []
803 
804  # Initialize command string
805  if not self.incremental:
806  cmd = ' -object:'+self.libname+' -module:'+self.outmodule+' -I:"'+self.outmodule+'"'
807  else:
808  # Add an trailing separator to indicate a folder
809  cmd = ' -object:'+self.scrtdir+os.path.sep+' -module:'+self.outmodule+' -I:"'+self.outmodule+'"'
810 
811  # Add all include paths to the command string
812  includes = [' -I"'+x+'" ' for x in self.incdirs]
813  for x in includes:
814  cmd += x
815 
816  # Choose the librarian and the file extension of the library.
817  if not self.isStatic:
818  librarian = 'link -dll -fixed:no -defaultlib:libcmt.lib -nodefaultlib:msvcrt.lib '
819  ext = '.dll'
820  else:
821  librarian = 'lib '
822  ext = '.lib'
823 
824  # Build commands using Intel Fortran (immutable)
825  ## Used defined command line options.
826  self.compargs = cmdstring
827  ## Intel Compiler command.
828  self.makecmd = self.iniCompiler+" "
829 
830  # Check whether an interface module wrapper was added to the current folder
831  if os.path.isfile(self.intermediate_wrapper):
832  makefile = "-fpp "+ self.wrapper_module
833  if (Utility.IsNotEmpty(self.wrapper_source)):
834  self.buildname = self.wrapper_source
835  elif self.incremental:
836  makefile = sep.join([x for x in self.srcs if os.path.splitext(x)[1].lower() in (".for", ".f95", ".f", ".f90")])
837  multi_objects = [os.path.splitext(x)[0]+".obj" for x in self.srcs if os.path.splitext(x)[1].lower() in (".for", ".f95", ".f", ".f90")]
838  else:
839  makefile = self.buildname
840 
841  self.makecmd += 'ifort -c '+ makefile + cmd + cmdstring + ' && '
842  ## Intel Linker command.
843  if not multi_objects:
844  self.linkcmd = librarian +self.libname+'.obj -nologo -machine:'+self.architecture+' -out:'+os.path.join(self.outlibs,self.libname+ext)
845  else:
846  self.temps = self.temps + (".obj",)
847  self.linkcmd = librarian +sep.join(multi_objects) +' -nologo -machine:'+self.architecture+' -out:'+os.path.join(self.outlibs,self.libname+ext)
848 
849  # Add temporary files to tuple scheduled for removal
850  self.temps = self.temps + (self.libname+'.obj', '.mod')
851  pass
852 
853  def create(self, **kwargs):
854  """
855  Execute make command
856  """
857  cmd = ''
858 
859  # Go into scratch directory (if defined)
860  with Utility.ChangedWorkingDirectory(self.scrtdir):
861 
862  # Pre-build event (if required)
863  try:
864  if self.precmd != '':
865  Utility.Popen(self.precmd, self.verbose)
866  except:
867  pass
868 
869  # Add all dependency paths to the command string
870  dependencies = [' -LIBPATH:"'+x+'" ' for x in self.libdirs]
871  for y in dependencies:
872  cmd += y
873 
874  # Add libraries for linking to the command string
875  linklist = [' '+x+'.lib ' for x in self.libs]
876  for x in linklist:
877  cmd += x
878 
879  # Delete old module files
880  for f in os.listdir(self.outmodule):
881  if f.endswith('.mod') and not kwargs.get("combine",False):
882  os.remove(os.path.join(self.outmodule,f))
883 
884  # Build event (if required)
885  try:
886  if self.makecmd != '':
887  # Execute again to account for input added after compiling command
888  self.Build(self.compargs)
889  if os.path.isfile(self.intermediate_wrapper):
890  Utility.ReplaceTextinFile(self.intermediate_wrapper, self.wrapper_module, {'%pyx_source%':'"'+self.buildname+'"'}, source=self.scrtdir)
891  Utility.Popen(self.makecmd+self.linkcmd+cmd, self.verbose)
892  except:
893  pass
894 
895  # Post-build event (if required)
896  try:
897  if self.postcmd != '':
898  Utility.Popen(self.postcmd, self.verbose)
899  except:
900  pass
901 
902  # Combine event (needed for TOMS). Combine multiple libraries into ONE.
903  if self.isStatic and kwargs.get("combine", False):
904  sep = ' '; librarian = 'lib '; ext = '.lib'
905  mergedid = os.path.basename(self.outmodule)
906  print(mergedid)
907  multi_libs = [os.path.join(self.outlibs,x) for x in [list(Utility.ArbitraryFlattening(x[2])) for x in Utility.PathWalk(self.outlibs)][0] if x.startswith(mergedid)]
908 
909  try:
910  # Remove old combined library from the list.
911  multi_libs.remove(os.path.join(self.outlibs,mergedid+self.architecture+ext))
912  except:
913  pass
914 
915  self.postcmd = self.iniCompiler + sep + librarian +sep.join(multi_libs) +' -nologo -machine:'+self.architecture+' -out:'+os.path.join(self.outlibs,mergedid+self.architecture+ext)
916  Utility.Popen(self.postcmd, self.verbose)
917  for lib in multi_libs:
918  os.remove(os.path.join(self.outlibs,lib))
919 
920  # Finish and delete redundant files
921  if not self.isStatic:
922  os.remove(os.path.join(self.outlibs,self.libname+'.exp'))
923  os.remove(os.path.join(self.outlibs,self.libname+'.lib'))
924  Utility.DeleteFilesbyEnding(self.temps)
925 
926  pass
927 
928 ## @class PyXMake.Build.Make.Py2X
929 # Base class for all Py2X (for now only f2py) build events. Inherited from Make.
930 class Py2X(Make):
931  """
932  Inherited class to build projects using Py2X.
933  """
934  def __init__(self, *args, **kwargs):
935  """
936  Initialization of Py2X class object.
937 
938  @note Currently uses f2py - but should be build with Py2X (DLR) in the future
939  """
940  super(Py2X, self).__init__(*args, **kwargs)
941  ## String identifier of current instance.
942  self.MakeObjectKind = 'Py2X'
943 
944  ## Define whether Intel's MKL should be statically or dynamically linked.
945  # Defaults to True, meaning that Intel's MKL has to be provided by the operating system.
946  self.no_static_mkl = kwargs.get('no_static_mkl', True)
947 
948  ## Define whether the architecture shall be appended to the build name.
949  # Defaults to False, meaning that the architecture is appended.
950  self.no_append_arch = kwargs.get('no_append_arch', False)
951 
952  ## Define if the input should be compiled exactly as provided.
953  # Defaults to False, meaning that merging & pre-processing utilities will be carried out.
954  self.incremental = kwargs.get('incremental', False)
955 
956  # Immutable settings for Py2X object
957  ## Absolute system path to Python executable.
958  self.path2exe = sys.executable.replace("\python.exe","")
959  ## The executable command used in the main build event.
960  self.exe = 'python.exe'
961 
962  # Interoperability change. Always put executables in quotes (how would have guessed...)
963  self.exe = '"{}" '.format(os.path.join(self.path2exe,self.exe))
964 
965  # Get valid f2py call.
966  old_numpy_call = os.path.join(self.path2exe, "Scripts","f2py.py")
967  new_numpy_call = os.path.join(self.path2exe, "Scripts","f2py.exe")
968 
969  if os.path.exists(old_numpy_call):
970  self.exe += '"{}"'.format(old_numpy_call)
971  else:
972  self.exe = '"{}"'.format(new_numpy_call)
973 
974  ## Temporary build name of current job.
975  self.buildname = self.buildid+"_pyd"+self.architecture+".f90"
976 
977  if self.architecture == "x86":
978  mkl_interface_lib = "mkl_intel_c"
979  else:
980  mkl_interface_lib = "mkl_intel_lp64"
981 
982  if not self.no_static_mkl:
983  # Link statically against MKL library. Deactivate this option by default.
984  self.libs.append([mkl_interface_lib, "mkl_intel_thread", "mkl_core", "libiomp5md"])
985  self.libs = list(Utility.ArbitraryFlattening(self.libs))
986  else:
987  # Link dynamically against MKL library.
988  self.libs.append("mkl_rt")
989  self.libs = list(Utility.ArbitraryFlattening(self.libs))
990 
991  if self.no_append_arch:
992  self.architecture = ''
993 
994  # Immutable settings for Py2X object
995  if self.incremental:
996  c_files = [x for x in self.srcs if os.path.splitext(x)[1].lower() in (".for", ".f95", ".f", ".f90")]
997  self.exe += ' -c -m %s %s' % (self.buildid+self.architecture, ' '.join(c_files))
998  else:
999  self.exe += ' -c -m %s %s' % (self.buildid+self.architecture, self.buildname)
1000 
1001  # Copy default mapping file to scratch directory (eq. workspace if not specified)
1002  copyfile(os.path.join(Path2Config,".f2py_f2cmap"), os.path.join(self.scrtdir,".f2py_f2cmap"))
1003 
1004  # Strip path from executable if already present.
1005  if self.exe.startswith(self.path2exe,1):
1006  self.path2exe = ""
1007 
1008  ## Tuple of temporary files deleted after job completion. Has already stored custom variable declaration
1009  # mapping file used by f2py.
1010  self.temps = self.temps + (".f2py_f2cmap", self.buildname)
1011  pass
1012 
1013 ## @class PyXMake.Build.Make.PyInstaller
1014 # Base class for all PyInstaller build events. Inherited from Make.
1016  """
1017  Inherited class to build projects using PyInstaller.
1018  """
1019  def __init__(self, *args, **kwargs):
1020  """
1021  Initialization of PyInstaller class object.
1022 
1023  @note Creates stand-alone application of Python scripts using PyInstaller.
1024  """
1025  super(PyInstaller, self).__init__(*args, **kwargs)
1026  ## String identifier of current instance.
1027  self.MakeObjectKind = "PyInstaller"
1028 
1029  # Set build mode
1030  self.buildtype = kwargs.get("type","onefile")
1031 
1032  # Remove all default libraries, paths and includes from Make class.
1033  self.libs = []
1034  self.libdirs = []
1035  self.incdirs = []
1036 
1037  # Add specification file to temporaries
1038  self.temps += (self.buildid+".spec",)
1039 
1040  pass
1041 
1042  def Encryption(self, encrypt, **kwargs):
1043  """
1044  Encrypt byte-code by using user-supplied or randomly generated key.
1045 
1046  @author: garb_ma
1047  @param encrypt: Boolean
1048  """
1049  if encrypt:
1050  self.key_string = kwargs.get("key_string",''.join(random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(16)))
1051  return
1052 
1053  def Preprocessing(self, cmdstring=''):
1054  """
1055  Assemble command string for the pre-build event.
1056  """
1057  delimn = " "
1058  # Store command string in object
1059  ## Command executed during pre-build event.
1060  self.precmd = delimn.join([self.iniCompiler,cmdstring])
1061  pass
1062 
1063  def Build(self, mode="onefile", **kwargs):
1064  """
1065  Switch build mode. Defaults to one single file.
1066  """
1067  import py_compile
1068  # Get shorthands for compatibility modes
1069  from PyInstaller.compat import is_py27, is_py35, is_py36, is_py37
1070  from setuptools import __version__ as set_version
1071 
1072  # Set build mode
1073  self.buildtype = mode
1074 
1075  # Reset command
1076  self.makecmd = ""
1077  # Create base command and add additional options based on identified inputs.
1078  self.makecmd = ['--name=%s' % self.buildid,'--%s' % self.buildtype, '--clean', "--noconfirm", "--windowed"]
1079 
1080  # Customize the application icon
1081  self.makecmd.append("--icon=%s" % kwargs.get("icon",os.path.join(Path2Config,"stm_logo.ico")))
1082 
1083  # Correction of required missing import for new releases of SetupTools.
1084  if int(str(set_version)[0:2]) >= 45:
1085  self.makecmd.insert(1,'--hidden-import=%s' % "pkg_resources.py2_warn")
1086 
1087  # Use UPX compression to create a smaller application folder/file
1088  if os.path.exists(os.path.join(PyXMakePath,"Build","bin","upx")):
1089  self.makecmd.append("--upx-dir=%s" % os.path.join(PyXMakePath,"Build","bin","upx"))
1090 
1091  # Some binaries are unusable after the obfuscation process. We skip those here.
1092  if is_py27:
1093  excludes = ["python2.dll","python27.dll", "qwindows.dll"]
1094  elif is_py35:
1095  excludes = [x for x in os.listdir(os.path.dirname(sys.executable)) if x.endswith((".dll"))]
1096  elif is_py36:
1097  excludes = [x for x in os.listdir(os.path.dirname(sys.executable)) if x.endswith((".dll"))]
1098  elif is_py37:
1099  excludes = ["python3.dll","python37.dll","ucrtbase.dll", "qwindows.dll"]
1100 
1101  upx_exclude = ["--upx-exclude=%s" % x for x in excludes]
1102  self.makecmd.extend(upx_exclude)
1103 
1104  # Mark additional Python files to be (re-compiled) before copying directly into the executable.
1105  add_compile = list(Utility.ArbitraryFlattening(kwargs.get("add_compile",[])))
1106 
1107  # Add additional include directories
1108  if hasattr(self, "incdirs"):
1109  for data in self.incdirs:
1110  if os.path.exists(data.split(os.path.pathsep)[0]):
1111  new_path = data.split(os.path.pathsep)[0]
1112  # Compile source code files to byte-code
1113  if Utility.PathLeaf(new_path).endswith(".py") and Utility.PathLeaf(new_path) in add_compile:
1114  py_compile.compile(new_path,os.path.join(self.scrtdir,Utility.PathLeaf(new_path)+"c"))
1115  self.temps = self.temps + (Utility.PathLeaf(new_path)+"c",)
1116  new_path = os.path.join(self.scrtdir,Utility.PathLeaf(new_path)+"c")
1117  data = os.path.pathsep.join([new_path,data.split(os.path.pathsep)[1]])
1118  self.makecmd.insert(1,'--add-data=%s' % data)
1119  else:
1120  for path in sys.path:
1121  new_path = os.path.join(path,data.split(os.path.pathsep)[0])
1122  if os.path.exists(new_path) and os.path.isdir(new_path):
1123  # Check if content of folders should be presented in plain text.
1124  if not kwargs.get("add_plain",False):
1125  for root, _, files in Utility.PathWalk(new_path):
1126  # Ignore repository folders (no exceptions!)
1127  if any(x in root for x in [".git", ".svn"]):
1128  continue
1129  for file in files:
1130  # Ignore files matching specified patterns
1131  if not file.endswith((".pdf")) and not any(x in file for x in [".cpython"]):
1132  base_path = data.split(os.path.pathsep)[-1]; sub_path = root.split(base_path)[-1]
1133  sub_struct =os.path.join(base_path,sub_path.lstrip(os.path.sep),".")
1134 
1135  if len(data.split(os.path.pathsep)) != 1:
1136  save_path = os.path.join(data.split(os.path.pathsep)[-1],".")
1137  else:
1138  save_path = sub_struct
1139  # Skip python files if compiled scripts are available
1140  if file.endswith(".py") and os.path.exists(os.path.join(new_path,file+"c")):
1141  continue
1142  elif file.endswith(".py") and file in add_compile:
1143  # Compile all blank source files to byte-code to obfuscate the code.
1144  py_compile.compile(os.path.join(new_path.split(data.split(os.path.pathsep)[0])[0],sub_struct.rstrip("."),file),
1145  os.path.join(new_path.split(data.split(os.path.pathsep)[0])[0],sub_struct.rstrip("."),file+"c"))
1146  file += "c"
1147  # Loop over all files within each directory and store them appropriately
1148  self.makecmd.insert(1,'--add-data=%s' % os.path.pathsep.join([os.path.join(new_path.split(
1149  data.split(os.path.pathsep)[0])[0],sub_struct.rstrip("."),file),save_path]))
1150  else:
1151  # Add folder content as-is to the application
1152  self.makecmd.insert(1,'--add-data=%s' % os.path.pathsep.join([new_path,data.split(os.path.pathsep)[-1]]))
1153  break
1154  # Check if new directory is a file. Preserve original path, otherwise do nothing.
1155  elif os.path.exists(new_path) and os.path.isfile(new_path):
1156  if Utility.PathLeaf(new_path).endswith(".py") and Utility.PathLeaf(new_path) in add_compile:
1157  py_compile.compile(new_path,os.path.join(self.scrtdir,Utility.PathLeaf(new_path)+"c"))
1158  self.temps = self.temps + (Utility.PathLeaf(new_path)+"c",)
1159  new_path = os.path.join(self.scrtdir,Utility.PathLeaf(new_path)+"c")
1160  self.makecmd.insert(1,'--add-data=%s' % os.path.pathsep.join([new_path,os.path.join(os.path.dirname(data.split(os.path.pathsep)[-1]),".")]))
1161  break
1162  else:
1163  continue
1164 
1165  # Set working (scratch) directory if being defined
1166  if hasattr(self, "scrtdir"):
1167  self.makecmd.insert(1,'--specpath=%s' % os.path.join(self.scrtdir))
1168  self.makecmd.insert(1,'--workpath=%s' % os.path.join(self.scrtdir,"build"))
1169 
1170  # Set output path if being defined. Defaults to current working directory
1171  if hasattr(self, "outdir"):
1172  self.makecmd.insert(1,'--distpath=%s' % os.path.join(self.outdir))
1173 
1174  # Add additional search paths
1175  if hasattr(self, "libdirs"):
1176  for path in self.libdirs:
1177  self.makecmd.insert(1,'--paths=%s' % path)
1178 
1179  # Add encryption method
1180  if hasattr(self, "key_string"):
1181  self.makecmd.insert(1,'--key=%s' % self.key_string)
1182 
1183  # Add source path (if given!)
1184  if hasattr(self, "srcdir") and self.srcdir:
1185  self.makecmd.append(os.path.join(self.srcdir,self.srcs[0]))
1186  else:
1187  self.makecmd.append(self.srcs[0])
1188 
1189  # Set verbosity level
1190  if hasattr(self, "verbose"):
1191  if self.verbose <= 0:
1192  level = "ERROR"
1193  elif self.verbose == 1:
1194  level = "WARN"
1195  elif self.verbose >= 2:
1196  level = "INFO"
1197  self.makecmd.insert(1,'--log-level=%s' % level)
1198 
1199  def create(self,**kwargs):
1200  """
1201  Execute make command
1202  """
1203  # Import required package (only when function is actually executed)
1204  import PyInstaller.__main__ as pyi
1205 
1206  # Increase recursion limit (required for OCC)
1207  sys.setrecursionlimit(kwargs.get("recursion_limit",5000))
1208 
1209  # Go into scratch folder
1210  with Utility.ChangedWorkingDirectory(self.scrtdir):
1211 
1212  # Pre-build event (if required)
1213  try:
1214  if self.precmd != '':
1215  Utility.Popen(self.precmd, self.verbose)
1216  except:
1217  pass
1218 
1219  # Create make command
1220  self.Build(mode=self.buildtype, **kwargs)
1221 
1222  # Run build command
1223  pyi.run(self.makecmd)
1224 
1225  # Finish and delete redundant files
1226  Utility.DeleteFilesbyEnding(self.temps)
1227  shutil.rmtree('build', ignore_errors=True)
1228 
1229  # End of class method
1230  return
1231 
1232 ## @class PyXMake.Build.Make.NSIS
1233 # Base class for all NSIS build events. Inherited from Make.
1234 class NSIS(Make):
1235  """
1236  Inherited class to build projects using NSIS.
1237  """
1238  def __init__(self, *args, **kwargs):
1239  """
1240  Initialization of PyInstaller class object.
1241 
1242  @note Creates a portable installer of a source folder using NSIS.
1243  """
1244  super(NSIS, self).__init__(*args, **kwargs)
1245  ## String identifier of current instance.
1246  self.MakeObjectKind = "NSIS"
1247 
1248  # Remove all default libraries, paths and includes from Make class.
1249  self.libs = []
1250  self.libdirs = []
1251  self.incdirs = []
1252 
1253  # Immutable settings for NSIS object
1254  ## Path to NSIS executable.
1255  self.path2exe = os.path.join(PyXMakePath,"Build","bin","nsis","App","NSIS")
1256  ## Executable of NSIS.
1257  self.exe = 'makensis.exe'
1258 
1259  # Initialize build command
1260  self.makecmd = os.path.join(self.path2exe,self.exe)
1261 
1262  # Add specification file to temporaries
1263  self.temps += (self.buildid+".nsh",)
1264  pass
1265 
1266  def FTP(self, user, key, upload_file, host='ftp.dlr.de', path="/public/download/nightly"):
1267  """
1268  Define settings to establish a FTP connection.
1269  """
1270  import ftplib
1271  # Establish FTP connection to file sharing server
1272  ## Remote workspace. This is the upload directory for the given file
1273  self.ftp_client = ftplib.FTP(host, 'ASRI_adm', 'lm9ITHUR')
1274  self.ftp_client.cwd(path)
1275  with open(upload_file, 'rb') as file:
1276  self.ftp_client.storbinary("STOR %s" % Utility.PathLeaf(upload_file), file)
1277  pass
1278 
1279  def create(self, **kwargs):
1280  """
1281  Execute make command
1282  """
1283  space = " "; point = "."; newline = '\n'
1284 
1285  # Create bundle command script
1286  script = ["Unicode true"]
1287  script.append("!define MULTIUSER_EXECUTIONLEVEL user")
1288  script.append("!define ZIP2EXE_NAME `%s`" % self.buildid)
1289  script.append("!define ZIP2EXE_OUTFILE `%s`" % os.path.join(self.outdir,point.join([self.buildid,"exe"])))
1290  script.append("!define ZIP2EXE_COMPRESSOR_LZMA")
1291  script.append("!define ZIP2EXE_INSTALLDIR `$EXEDIR`")
1292  script.append("!include `${NSISDIR}\Contrib\zip2exe\Base.nsh`")
1293  script.append("!include `${NSISDIR}\Contrib\zip2exe\Modern.nsh`")
1294  if kwargs.get("install_path",""):
1295  script.append("InstallDir '%s'" % kwargs.get("install_path","$Desktop"))
1296  script.append("!insertmacro SECTION_BEGIN")
1297  script.append("SetOutPath $INSTDIR")
1298  # Add all source files to the bundle
1299  for src in self.srcs:
1300  script.append("File /r /x *.nsh `%s`" % os.path.join(kwargs.get("assembly_path", self.srcdir),src))
1301  script.append("!insertmacro SECTION_END")
1302  script.append('Icon "%s"' % kwargs.get("icon",os.path.join(Path2Config,"stm_logo.ico")))
1303  script.append("RequestExecutionLevel user")
1304 
1305  # Go into scratch folder
1306  with Utility.ChangedWorkingDirectory(self.scrtdir):
1307  # Run build command
1308  MakeFile = Utility.GetTemporaryFileName(extension=".nsh")
1309 
1310  # Populate script file with customizable features
1311  with open(MakeFile,"w") as f:
1312  f.write(newline.join(script))
1313 
1314  # Execute command
1315  self.makecmd = space.join([self.makecmd, "/V3", os.path.join(self.scrtdir,MakeFile)])
1316  Utility.Popen(self.makecmd, self.verbose)
1317 
1318  # Add temporary file to tuple scheduled for removal
1319  self.temps += (MakeFile,)
1320 
1321  # Finish and delete redundant files
1322  Utility.DeleteFilesbyEnding(self.temps)
1323 
1324  # Upload results to a file sharing server
1325  if kwargs.get("upload",False) and not hasattr(self, "ftp_client"):
1326  user = kwargs.get("user",os.getenv("username")); key = kwargs.get("key","")
1327  self.FTP(user, key, point.join([os.path.join(self.outdir,self.buildid),"exe"]))
1328 
1329  # Success message
1330  print("==================================")
1331  print("Uploaded result to DLR's FTP server")
1332  print("==================================")
1333 
1334  # End of class method
1335  pass
1336 
1337 ## @class PyXMake.Build.Make.Doxygen
1338 # Base class for all Doxygen build events. Inherited from Make.
1339 class Doxygen(Make):
1340  """
1341  Inherited class to automatically build a documentation using Doxygen.
1342  """
1343  def __init__(self, *args,**kwargs):
1344  """
1345  Initialization of doxygen class object.
1346  """
1347  super(Doxygen, self).__init__(*args,**kwargs)
1348  ## String identifier of current instance.
1349  self.MakeObjectKind = 'Doxygen'
1350 
1351  os.environ['dox_pyjava'] = 'NO'
1352  os.environ['dox_fortran'] = 'NO'
1353  os.environ['dox_ccpp'] = 'NO'
1354  os.environ['dox_pdflatex'] = 'NO'
1355 
1356  # Immutable settings for Doxygen object
1357  ## Path to Doxygen executable.
1358  self.path2exe = os.path.join(PyXMakePath,"Build","bin","doxygen")
1359  ## Executable of Doxygen.
1360  self.exe = 'doxygen.exe'
1361 
1362  ## Type of source file. Can be one of Fortran, CCpp or Other.
1363  # Defaults to Fortran if not specified. Starts documentation procedure
1364  # for Java/Python if type is neither Fortran nor CCpp.
1365  if self.stype == 'Fortran':
1366  os.environ['dox_fortran'] = 'YES'
1367  ## Temporary build name of current job.
1368  self.buildname = self.buildid+"_doc.f90"
1369  ## Tuple of temporary files scheduled for removal.
1370  self.temps = self.temps + (self.buildname, )
1371  elif self.stype == 'CCpp':
1372  os.environ['dox_ccpp'] = 'YES'
1373  self.buildname = self.buildid+"_doc.cpp"
1374  self.temps = self.temps + (self.buildname, )
1375  else:
1376  temp = []; delimn = " "
1377  self.buildname = ''
1378  temp.append(self.srcs)
1379  os.environ['dox_pyjava'] = 'YES'
1380  paths = list(Utility.ArbitraryFlattening(temp))
1381 
1382  # Remove empty and/or unnecessary paths from joined input string
1383  for y in paths:
1384  for x in os.listdir(y):
1385  if x.endswith((".java", ".py")):
1386  if not any([z == "__init__.py" for z in os.listdir(y)]) and x.endswith((".py")):
1387  # Automatic documentation of files requires regular packages, not pure folders. Temporarily add an
1388  # __init__.py to every folder containing Python scripts. Removed after completion.
1389  open(os.path.join(y,"__init__.py"), 'a+').close()
1390  # Add absolute path to temporary files scheduled for removal.
1391  self.temps = self.temps + (os.path.join(y,"__init__.py"),)
1392  continue
1393  break
1394  else:
1395  paths.remove(y)
1396  if y in [os.path.dirname(k) for k in list(filter(os.path.isfile, self.temps))]:
1397  paths.append(y)
1398 
1399  self.buildname = delimn.join(paths)
1400 
1401  for x in list(filter(os.path.isfile, self.temps)):
1402  while True:
1403  try:
1404  path = os.path.dirname(x)
1405  if os.path.exists(os.path.join(path,"__init__.py")):
1406  x = path
1407  else:
1408  break
1409  except:
1410  break
1411 
1412  list(filter(os.path.isfile, self.temps))
1413 
1414  # Remove all directories from temporary array. We only seek files.
1415  self.temps = tuple(filter(os.path.isfile, self.temps))
1416 
1417  # Check if non-directories exists
1418  if list(self.temps):
1419  # Schedule them for removal
1420  rev_command = ["del"] + list(self.temps)
1421  self.postcmd = delimn.join(rev_command)
1422 
1423  # Immutable environment variables
1424  os.environ['dox_input'] = self.buildname
1425  os.environ['dox_images'] = Path2Config
1426  os.environ['dox_footer'] = os.path.join(Path2Config,"stm_doc_footer.html")
1427  os.environ['dox_logo'] = os.path.join(Path2Config,"stm_logo.png")
1428  os.environ['dox_pyfilter'] = sys.executable+" "+os.path.join(Path2Config,"stm_pyfilter.py")
1429 
1430  # Set default color scheme.
1431  if not all([os.getenv(x) for x in ("dox_hue","dox_sat","dox_gamma")]):
1432  from PyXMake.Build.config.stm_color import DLRBlue as dox_color
1433  os.environ['dox_hue'], os.environ['dox_sat'], os.environ['dox_gamma'] = [str(int(round(x))) for x in np.multiply([360.,100.,100],
1434  np.array(colorsys.rgb_to_hsv(*(value/255 for value in
1435  ImageColor.getcolor(dox_color,"RGB")))))]
1436 
1437  # Remove MKL from default command line
1438  ## Blank version of list containing library directories without initially specifying MKL.
1439  self.incdirs = []
1440  self.libdirs = []
1441  pass
1442 
1443  def Settings(self, brief, header, outdir='', **kwargs):
1444  """
1445  Define environment variables for the default configuration file.
1446  """
1447  # Overwrite output directory
1448  if outdir != '':
1449  ## Output directory of current job. Defaults to current workspace if not given.
1450  self.outdir = outdir
1451 
1452  # Set environment variables for configuration script
1453  os.environ['dox_brief'] = brief
1454  os.environ['dox_header'] = header
1455  os.environ['dox_outdir'] = self.outdir
1456 
1457  # Set color scheme in HUE (HSV)
1458  if kwargs.get("color",""):
1459  # Only modify color scheme if a hex color was explicitly given
1460  os.environ['dox_hue'], os.environ['dox_sat'], os.environ['dox_gamma'] = [str(x) for x in np.multiply([360.,100.,100],
1461  np.array(colorsys.rgb_to_hsv(*(value/255 for value in
1462  ImageColor.getcolor(kwargs.get("color"),"RGB")))))]
1463  pass
1464 
1465 ## @class PyXMake.Build.Make.Sphinx
1466 # Base class for all Sphinx build events. Inherited from Make.
1467 class Sphinx(Make):
1468  """
1469  Inherited class to automatically build a documentation using Sphinx.
1470  """
1471  def __init__(self, *args,**kwargs):
1472  """
1473  Initialization of sphinx class object.
1474  """
1475  super(Sphinx, self).__init__(*args,**kwargs)
1476  ## String identifier of current instance.
1477  self.MakeObjectKind = 'Sphinx'
1478 
1479  # Validate third party dependencies
1480  from six import exec_
1481  from PyXMake.Build import __install__
1482  exec_(open(__install__.__file__).read(),{"__name__": "__main__", "__file__":__install__.__file__})
1483 
1484  # Build script for Sphinx documentation
1485  os.environ["PATH"] += os.pathsep + os.pathsep.join([os.path.join(os.path.dirname(sys.executable),"Scripts")])
1486 
1487  # Immutable settings for Sphinx object
1488  ## Path to Sphinx executable.
1489  self.path2exe = os.path.join(os.path.dirname(sys.executable),"Scripts")
1490  ## Executable of Sphinx.
1491  self.exe = 'sphinx-build.exe'
1492  ## Ordered dictionary containing all available options
1493  self.BuildOption = OrderedDict()
1494 
1495  # Project name from BuildID
1496  os.environ["sphinx_project"] = str(self.buildid)
1497 
1498  # Name of index file (master document).
1499  os.environ["sphinx_master"] = str(self.srcs[0])
1500 
1501  # Default color scheme
1502  from PyXMake.Build.config.stm_color import DLRBlue as sphinx_color
1503  os.environ["sphinx_color"] = sphinx_color
1504 
1505  # Remove MKL from default command line
1506  ## Blank version of list containing library directories without initially specifying MKL.
1507  self.incdirs = []
1508  self.libdirs = []
1509  pass
1510 
1511  def Settings(self, **kwargs):
1512  """
1513  Define environment variables for the default configuration file.
1514  """
1515  delimn = "_"
1516  # Set environment variables for configuration script
1517  for key, value in kwargs.items():
1518  os.environ[delimn.join(["sphinx",str(key)])] = str(value)
1519  pass
1520 
1521  def create(self, **kwargs):
1522  """
1523  Execute make command
1524  """
1525  from distutils.dir_util import copy_tree
1526 
1527  # You can set these variables from the command line.
1528  self.SPHINXOPTS = os.getenv('sphinx_opts', '')
1529  self.SPHINXBUILD = os.getenv('sphinx_build', 'sphinx-build')
1530  self.PAPER = os.getenv('sphinx_paper', None)
1531 
1532  self.SOURCEDIR = os.getenv('sphinx_sourcedir', self.srcdir)
1533  self.BUILDDIR = os.getenv('sphinx_builddir', Utility.PathLeaf(tempfile.NamedTemporaryFile().name))
1534  self.TEMPDIR = os.getenv('sphinx_templates', "_templates")
1535  self.STATICEDIR = os.getenv('sphinx_static', "_static")
1536 
1537  # Create static and template folder relative to source directory, if required.
1538  os.environ['sphinx_static'] = self.STATICEDIR; os.environ['sphinx_templates'] = self.TEMPDIR
1539 
1540  # Add path dependencies
1541  os.environ["sphinx_include"] = os.getenv("sphinx_include","")
1542  os.environ["sphinx_include"] += os.pathsep + os.pathsep.join(self.incdirs)
1543 
1544  def validate():
1545  """
1546  User-friendly check for sphinx-build
1547  """
1548  with open(os.devnull, 'w') as devnull:
1549  try:
1550  if subprocess.call([self.SPHINXBUILD, '--version'],stdout=devnull, stderr=devnull) == 0:
1551  return
1552  except FileNotFoundError:
1553  pass
1554  print(
1555  "The '{0}' command was not found. Make sure you have Sphinx "
1556  "installed, then set the SPHINXBUILD environment variable "
1557  "to point to the full path of the '{0}' executable. "
1558  "Alternatively you can add the directory with the "
1559  "executable to your PATH. If you don't have Sphinx "
1560  "installed, grab it from http://sphinx-doc.org/)"
1561  .format(self.SPHINXBUILD))
1562  sys.exit(1)
1563  return
1564 
1565  def build(builder, success_msg=None, extra_opts=None, outdir=None,doctrees=True):
1566  """
1567  The default target
1568  """
1569  builddir = os.path.join(self.BUILDDIR or outdir)
1570  command = [self.SPHINXBUILD, '-M', builder, self.SOURCEDIR, builddir]
1571  command.extend(['-c', os.getenv('sphinx_config',self.SOURCEDIR)])
1572  if command[-1] == self.SOURCEDIR:
1573  command = command[:len(command)-2]
1574  if doctrees:
1575  command.extend(['-d', os.path.join(self.BUILDDIR, 'doctrees')])
1576  if extra_opts:
1577  command.extend(extra_opts)
1578  command.extend(shlex.split(self.SPHINXOPTS))
1579  # Execute build command
1580  if Utility.Popen(command,self.verbose) == 0:
1581  print('Build finished. ' + success_msg.format(self.outdir or builddir))
1582 
1583  def buildmethod(function):
1584  """
1585  Decorator function for each build option
1586  """
1587  self.BuildOption[function.__name__] = function
1588  return function
1589 
1590  @buildmethod
1591  def default():
1592  """
1593  The default target
1594  """
1595  return html()
1596 
1597  @buildmethod
1598  def clean():
1599  """
1600  Remove the build directory
1601  """
1602  shutil.rmtree(self.BUILDDIR, ignore_errors=True)
1603 
1604  @buildmethod
1605  def html():
1606  """
1607  Make standalone HTML files
1608  """
1609  return build('html', 'The HTML pages are in {}.')
1610 
1611  @buildmethod
1612  def dirhtml():
1613  """
1614  Make HTML files named index.html in directories
1615  """
1616  return build('dirhtml', 'The HTML pages are in {}')
1617 
1618  @buildmethod
1619  def singlehtml():
1620  """
1621  Make a single large HTML file
1622  """
1623  return build('singlehtml', 'The HTML page is in {}.')
1624 
1625  @buildmethod
1626  def pickle():
1627  """
1628  Make pickle files
1629  """
1630  return build('pickle', 'Now you can process the pickle files.')
1631 
1632  @buildmethod
1633  def json():
1634  """
1635  Make JSON files
1636  """
1637  return build('json', 'Now you can process the JSON files.')
1638 
1639  @buildmethod
1640  def htmlhelp():
1641  """
1642  Make HTML files and a HTML help project
1643  """
1644  return build('htmlhelp', 'Now you can run HTML Help Workshop with the .hhp project file in {}.')
1645 
1646  @buildmethod
1647  def qthelp():
1648  """
1649  Make HTML files and a qthelp project
1650  """
1651  return build('qthelp', 'Now you can run "qcollectiongenerator" with the '
1652  '.qhcp project file in {0}, like this: \n'
1653  '# qcollectiongenerator {0}/RinohType.qhcp\n'
1654  'To view the help file:\n'
1655  '# assistant -collectionFile {0}/RinohType.qhc')
1656 
1657  @buildmethod
1658  def devhelp():
1659  """
1660  Make HTML files and a Devhelp project
1661  """
1662  return build('devhelp', 'To view the help file:\n'
1663  '# mkdir -p $HOME/.local/share/devhelp/RinohType\n'
1664  '# ln -s {} $HOME/.local/share/devhelp/RinohType\n'
1665  '# devhelp')
1666 
1667  @buildmethod
1668  def epub(self):
1669  """
1670  Make an epub
1671  """
1672  return self.build('epub', 'The epub file is in {}.')
1673 
1674  @buildmethod
1675  def rinoh(self):
1676  """
1677  Make a PDF using rinohtype
1678  """
1679  return self.build('rinoh', 'The PDF file is in {}.')
1680 
1681  @buildmethod
1682  def latex():
1683  """
1684  Make LaTeX files, you can set PAPER=a4 or PAPER=letter
1685  """
1686  extra_opts = ['-D', 'latex_paper_size={}'.format(self.PAPER)] if self.PAPER else None
1687  return build('latex', 'The LaTeX files are in {}.\n'
1688  "Run 'make' in that directory to run these through "
1689  "(pdf)latex (use the 'latexpdf' target to do that "
1690  "automatically).", extra_opts)
1691 
1692  @buildmethod
1693  def latexpdf():
1694  """
1695  Make LaTeX files and run them through pdflatex
1696  """
1697  _ = latex()
1698  print('Running LaTeX files through pdflatex...')
1699  builddir = os.path.join(self.BUILDDIR, 'latex')
1700  subprocess.call(['make', '-C', builddir, 'all-pdf'])
1701  print('pdflatex finished; the PDF files are in {}.'.format(builddir))
1702 
1703  @buildmethod
1704  def latexpdfja():
1705  """
1706  Make LaTeX files and run them through platex/dvipdfmx
1707  """
1708  _ = latex()
1709  print('Running LaTeX files through platex and dvipdfmx...')
1710  builddir = os.path.join(self.BUILDDIR, 'latex')
1711  subprocess.call(['make', '-C', builddir, 'all-pdf-ja'])
1712  print('pdflatex finished; the PDF files are in {}.'.format(builddir))
1713 
1714  @buildmethod
1715  def text():
1716  """
1717  Make text files
1718  """
1719  return build('text', 'The text files are in {}.')
1720 
1721  @buildmethod
1722  def man():
1723  """
1724  Make manual pages
1725  """
1726  return build('man', 'The manual pages are in {}.')
1727 
1728  @buildmethod
1729  def texinfo():
1730  """
1731  Make TexInfo files
1732  """
1733  return build('texinfo', 'The Texinfo files are in {}.\n'
1734  "Run 'make' in that directory to run these "
1735  "through makeinfo (use the 'info' target to do "
1736  "that automatically).")
1737 
1738  @buildmethod
1739  def info():
1740  """
1741  Make Texinfo files and run them through makeinfo
1742  """
1743  _ = texinfo()
1744  print('Running Texinfo files through makeinfo...')
1745  builddir = os.path.join(self.BUILDDIR, 'texinfo')
1746  subprocess.call(['make', '-C', builddir, 'info'])
1747  print('makeinfo finished; the Info files are in {}.'.format(builddir))
1748 
1749  @buildmethod
1750  def gettext():
1751  """
1752  Make PO message catalogs
1753  """
1754  return build('gettext', 'The message catalogs are in {}.', outdir='locale',doctrees=False)
1755 
1756  @buildmethod
1757  def changes():
1758  """
1759  Make an overview of all changed/added/deprecated items
1760  """
1761  return build('changes', 'The overview file is in {}.')
1762 
1763  @buildmethod
1764  def xml():
1765  """
1766  Make Docutils-native XML files
1767  """
1768  return build('xml', 'The XML files are in {}.')
1769 
1770  @buildmethod
1771  def pseudoxml():
1772  """
1773  Make pseudoxml-XML files for display purposes
1774  """
1775  return self.build('pseudoxml', 'The pseudo-XML files are in {}.')
1776 
1777  @buildmethod
1778  def linkcheck():
1779  """
1780  Check all external links for integrity
1781  """
1782  return build('linkcheck', 'Look for any errors in the above output or in {}/output.txt.')
1783 
1784  @buildmethod
1785  def doctest():
1786  """
1787  Run all doctests embedded in the documentation (if enabled)
1788  """
1789  return build('doctest', 'Look at the results in {}/output.txt.')
1790 
1791  @buildmethod
1792  def assist():
1793  """
1794  List all targets
1795  """
1796  print("Please use '{} <target>' where <target> is one of" .format(sys.argv[0]))
1797  width = max(len(name) for name in self.BuildOption)
1798  for name, target in self.BuildOption.items():
1799  print(' {name:{width}} {descr}'.format(name=name, width=width, descr=target.__doc__))
1800 
1801  # Validate installation status
1802  validate()
1803  # Get additional command line arguments
1804  args = ['default'] or sys.argv[1:]
1805  for arg in args:
1806  # Create a temporary build directory. We do not need unsuccessful build
1807  with Utility.TemporaryDirectory(self.scrtdir):
1808  # Create a new local directory
1809  temp = Utility.PathLeaf(tempfile.NamedTemporaryFile().name)
1810  # Copy all source files into the temporary directory
1811  shutil.copytree(self.SOURCEDIR, temp, ignore=shutil.ignore_patterns(".git",".svn")); self.SOURCEDIR = temp
1812  # If no configuration file is found in the source folder, use the default template
1813  if not os.path.exists(os.path.join(temp,"conf.py")):
1814  # Rename default configuration file to match naming convention.
1815  copyfile(os.path.join(PyXMakePath,"Build","config","stm_conf.py"), os.path.join(temp,"conf.py"))
1816  # Create an additional static folder if not already existing
1817  if not os.path.exists(os.path.join(temp,"_static")):
1818  os.mkdir(os.path.join(temp,'_static')); copyfile(os.path.join(PyXMakePath,"Build","config","stm_style.css"), os.path.join(temp,"_static","style.css"))
1819  # Create an additional templates folder
1820  if not os.path.exists(os.path.join(temp,"_templates")):
1821  os.mkdir(os.path.join(temp,'_templates')); copyfile(os.path.join(PyXMakePath,"Build","config","stm_layout.html"), os.path.join(temp,"_templates","layout.html"))
1822  # Create all documentations
1823  self.BuildOption[arg]()
1824  # Do not keep temporary tree by default
1825  if not kwargs.get("keep_doctree",False):
1826  try:
1827  shutil.rmtree(os.path.join(self.BUILDDIR,"doctrees"))
1828  except OSError:
1829  pass
1830  # Copy results to output directory
1831  copy_tree(self.BUILDDIR, self.outdir)
1832 
1833  # Return success
1834  return 0
1835 
1836 ## @class PyXMake.Build.Make.SSH
1837 # Base class for all build events requiring a SSH connection. Inherited from Make.
1838 class SSH(Make):
1839  """
1840  Inherited class for all builds using SSH connection.
1841  """
1842  def __init__(self, *args, **kwargs):
1843  """
1844  Initialization of SSH class object.
1845  """
1846  super(SSH, self).__init__(*args, **kwargs)
1847 
1848  # Add Fortran wrapper function to SSH class.
1849  setattr(self, str(Fortran.Wrapper.__name__), MethodType(Fortran.Wrapper, self))
1850 
1851  # Defined here to be checked later.
1852  ## Wrapper interface file for 3rd party FORTRAN code. Automatically creates a module of the underlying source material.
1854  self.wrapper_source = ""
1855  self.wrapper_module = "pyx_module.f90"
1856 
1857  ## String identifier of current instance.
1858  self.MakeObjectKind = 'SSH'
1859 
1860  # Immutable settings for SSH object
1861  ## Name of library, assembled using BuildID.
1862  self.libname = "lib"+self.buildid + self.architecture
1863  ## Temporary build name.
1864  self.buildname = self.buildid+'_ssh'
1865  ## Environment variables to be set prior to the execution of the build command. Intel Fortran 12+
1866  self.export = "export CPATH=$CPATH"
1867  ## Environment variables to be set prior to the execution of the build command. Intel Fortran 11 and lower.
1868  self.__old_export = "export FPATH=$FPATH"
1869  ## Environment variable to be set prior to the execution of the build command.
1870  self.__library_path = "export LIBRARY_PATH=$LIBRARY_PATH"
1871  ## Custom intel path
1872  self.__intel_path = "export pyx_ifort=$(which ifort)"
1873 
1874  ## Define if the input should be compiled exactly as provided.
1875  # Defaults to False, meaning that merging & pre-processing utilities will be carried out.
1876  self.incremental = kwargs.get('incremental', False)
1877 
1878  # Initialization of lists containing additional sources, modules or libraries
1879  ## List of libraries which should be statically linked in.
1880  self.linkedIn = []
1881 
1882  # Initialization of tuple containing temporary files
1883  ## Blank version of tuple to store temporary file names scheduled for removal.
1884  self.temps = ()
1885 
1886  ## Load an additional library prior to execution of all commands. Defaults to an empty string.
1887  self.environment = ""
1888 
1889  # Remove MKL from default command line
1890  ## Blank version of list containing library directories. MKL library has been removed since location
1891  # on the SSH remote computer is not known a priori.
1892  self.libdirs = []
1893  pass
1894 
1895  def OutputPath(self, libpath, modulepath=None):
1896  """
1897  Define output directories for modules and libraries. Output written to the workspace is DELETED.
1898  """
1899  # Output module files to scratch directory by default.
1900  if modulepath is None:
1901  modulepath = libpath
1902  ## Output path for module or header files.
1903  self.outmodule = modulepath + posixpath.sep
1904  ## Output path for library files.
1905  self.outlibs = libpath + posixpath.sep
1906  pass
1907 
1908  def Settings(self, user, key, host='129.247.54.37', port=22):
1909  """
1910  Define settings to establish a SSH connection.
1911  """
1912  # Establish SSH connection to institute cluster // mg 07.08.17
1913  ## Remote workspace. This is the scratch directory for the build event.
1914  # Defaults to /home/user/.
1915  self.workspace = posixpath.join(Utility.AsDrive('home',posixpath.sep),user)+posixpath.sep
1916  ## Instance of SSHClient to establish a SSH connection.
1917  self.ssh_client = paramiko.SSHClient()
1918  self.ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
1919  self.ssh_client.connect(hostname=host, port=port, username=user, key_filename=key)
1920  pass
1921 
1922  def Environment(self, path="", bash="", args=""):
1923  """
1924  Load an additional environment file prior to execution of all commands.
1925  """
1926  ## Execute an additional bash script prior to all build commands.
1927  self.environment += "source " + posixpath.join(path,bash)+" "+args+"; "
1928  pass
1929 
1930  def Postprocessing(self, cmdstring=""):
1931  """
1932  Assemble command string for the post-build event.
1933  """
1934  ## Command executed during post-build event.
1935  self.postcmd = cmdstring
1936  pass
1937 
1938  def Build(self, cmdstring, run= "ifort", path="", lib= "", linkedIn=""):
1939  """
1940  Assemble command strings for the main build event.
1941  """
1942  cmd = ""
1943 
1944  # Which libraries are used for linking (relative path).
1945  self.libs.append(lib)
1946  self.libs = list(Utility.ArbitraryFlattening(self.libs))
1947 
1948  # Which libraries have to be additionally linked in (absolute path).
1949  self.linkedIn.append(linkedIn)
1950  self.linkedIn = list(Utility.ArbitraryFlattening(self.linkedIn))
1951 
1952  # Build commands using Intel Fortran (mutable)
1953  ## (Intel Fortran) Compiler Path
1954  self.path2exe = path; self.exe = run
1955 
1956  # Check whether an interface module wrapper was added to the current folder
1957  if os.path.isfile(self.intermediate_wrapper):
1958  if (Utility.IsNotEmpty(self.wrapper_source)):
1959  self.buildname = self.wrapper_source
1960 
1961  # Immutable settings for SSH object
1962  if self.incremental:
1963  c_files = [x for x in self.srcs if os.path.splitext(x)[1].lower() in (".for", ".f95", ".f", ".f90")]
1964  cmd += ' %s ' % (' '.join(c_files))
1965 
1966  # Always add MKL when used with f2py.
1967  if self.exe in ("f2py"):
1968  cmd += "-m "+str(self.buildid+self.architecture)
1969  # Assist f2py by providing shared MKL libraries directly
1970  self.libs.extend(["mkl_rt","iomp5", "pthread", "m", "dl"])
1971 
1972  # Add libraries for referencing to the command string (they are only used as resources)
1973  for x in [' -l'+x+' ' for x in self.libs]:
1974  cmd += x
1975 
1976  ## Remote (Intel) Compiler command.
1977  self.makecmd = posixpath.join(self.path2exe,self.exe)+" -c "+ cmd + cmdstring
1978 
1979  ## Remote Linker command.
1980  if self.exe not in ("ifort", "gcc", "g++"):
1981  ## Do no use any archiver
1982  self.linkcmd = ""
1983  elif self.exe == "ifort":
1984  ## Remote Intel Linker command.
1985  self.linkcmd = posixpath.join(self.path2exe,"xiar")+" -rc "
1986  else:
1987  ## Simply execute UNIX archiver
1988  self.linkcmd = posixpath.join("","ar")+" -rc "
1989  pass
1990 
1991  def create(self, **kwargs):
1992  """
1993  Define settings to establish SSH connection.
1994  """
1995  cmd = ""
1996 
1997  # Add all include paths to CPATH & FPATH (deprecated!) environment variable
1998  includes = [':"'+x+'"' for x in self.incdirs]
1999  for x in includes:
2000  self.export += x
2001  self.__old_export += x
2002  self.export += " && " + self.__old_export
2003 
2004  # Add all library paths to LIBRARY_PATH environment variable
2005  library = [':"'+x+'" ' for x in self.libdirs]
2006  for x in library:
2007  self.__library_path += x
2008  self.export += " && " + self.__library_path
2009 
2010  # Add libraries for linking to the command string
2011  try:
2012  if self.linkedIn[0] != "":
2013  linklist = ['ar -x "'+x+'" && ' for x in self.linkedIn]
2014  for x in linklist:
2015  cmd += x
2016  # Link list is empty or does not exist
2017  except IndexError:
2018  pass
2019 
2020  # Get the target and the base name of library (created in the process).
2021  target = posixpath.join(self.workspace,self.buildname)
2022  # base = os.path.splitext(target)[0]
2023 
2024  # Go into scratch directory (if defined)
2025  with Utility.ChangedWorkingDirectory(self.scrtdir):
2026 
2027  # Pre-build event (if required)
2028  try:
2029  if self.precmd != '':
2030  Utility.Popen(self.precmd, self.verbose)
2031  except:
2032  pass
2033 
2034  # Establish ssh connection and execute make commands on the linux cluster.
2035  sftp = self.ssh_client.open_sftp()
2036 
2037  try:
2038  sftp.put(self.buildname,target)
2039  if os.path.isfile(self.intermediate_wrapper):
2040  # Use general-purpose wrapper file
2041  Utility.ReplaceTextinFile(self.intermediate_wrapper, self.wrapper_module, {'%pyx_source%':'"'+self.buildname+'"',"#":" "}, source=self.scrtdir)
2042  sftp.put(os.path.join(self.scrtdir,self.wrapper_module),posixpath.join(self.workspace, self.wrapper_module))
2043  target = posixpath.join(self.workspace,self.wrapper_module)
2044  # Preserve the unique name of each object file
2045  self.makecmd += " -o "+ os.path.splitext(self.buildname)[0]+".o"
2046  except:
2047  target = ""
2048  for cs in self.srcs:
2049  sftp.put(os.path.join(self.scrtdir,cs),posixpath.join(self.workspace,cs))
2050 
2051  # Put f2py mapping file into current workspace
2052  if self.exe not in ("ifort", "gcc", "g++"):
2053  sftp.put(os.path.join(Path2Config,".f2py_f2cmap"), posixpath.join(self.workspace,".f2py_f2cmap"))
2054  sftp.close()
2055 
2056  # Create output folder if not existing
2057  _, _, _ = self.ssh_client.exec_command("mkdir -p "+self.outmodule+"; mkdir -p "+self.workspace)
2058 
2059  # Delete old content in output folders
2060  if not kwargs.get("combine", False):
2061  _, _, _ = self.ssh_client.exec_command('rm -f '+self.outmodule+'*.mod; rm -f '+self.workspace+'*.mod')
2062  else:
2063  _, _, _ = self.ssh_client.exec_command('rm -f '+self.workspace+'*.mod')
2064 
2065  self.command = self.environment + self.export + " && cd "+self.workspace+ " && " + self.makecmd
2066  if Utility.IsNotEmpty(target):
2067  self.command+= ' "'+target+'"'
2068  sbuild = Utility.SSHPopen(self.ssh_client, self.command, self.verbose)
2069 
2070  if sbuild==0:
2071  # There is a valid link command. Use it
2072  if Utility.IsNotEmpty(self.linkcmd):
2073  self.command = self.environment + self.export + " && " + cmd + self.linkcmd + self.libname+'.a '+'*.o '
2074  sarch = Utility.SSHPopen(self.ssh_client, self.command, self.verbose)
2075  # Copy all created shared libraries.
2076  else:
2077  sarch = 1; scopy = 0
2078  self.postcmd = 'cp -rf '+self.workspace+'*.so '+self.outlibs
2079 
2080  if sarch==0:
2081  self.command = 'cp -rf '+self.workspace+'*.mod '+self.outmodule+'; cp -rf '+self.workspace+'*.a '+self.outlibs
2082  scopy = Utility.SSHPopen(self.ssh_client, self.command, self.verbose)
2083 
2084  if scopy == 0:
2085  _, _, _ = self.ssh_client.exec_command('rm '+self.workspace+'*.o; rm '+self.workspace+'*.mod; rm '+self.workspace+'*.a; rm '+
2086  self.workspace+'*.f90; rm -rf '+posixpath.join(self.workspace,"intel"))
2087 
2088  if Utility.IsNotEmpty(self.postcmd):
2089  self.export += " && " + self.__intel_path
2090  self.command = self.environment + self.export + " && " + self.postcmd
2091  spost = Utility.SSHPopen(self.ssh_client, self.command, self.verbose)
2092  if spost == 0:
2093  _, _, _ = self.ssh_client.exec_command('rm -f '+self.workspace+'*.o; rm '+self.workspace+'*.mod; rm -f '+self.workspace+'*.a; rm -f '+
2094  self.workspace+'*.f90; rm -f '+self.workspace+'*.f; rm -f '+
2095  posixpath.join(self.workspace,"intel")+'; rm -f '+
2096  posixpath.join(self.workspace,".f2py_f2cmap") +
2097  " rm -f %s " % (' '.join([posixpath.join(self.workspace,cs) for cs in self.srcs])))
2098  pass
2099  pass
2100 
2101  # Combine event (needed for TOMS). Combine multiple libraries into ONE.
2102  if kwargs.get("combine", False):
2103  librarian = 'ar'; ext = '.a'; decomp = " && "+librarian+" -x "
2104  mergedid = "lib"+posixpath.basename(self.outmodule.rstrip("/"))
2105  _ , stdout, _ = self.ssh_client.exec_command('ls '+self.outlibs)
2106  multi_libs = [x for x in [x.rstrip("\n") for x in stdout.readlines() if x.startswith(mergedid)]]
2107 
2108  try:
2109  # Remove old combined library from the list.
2110  multi_libs.remove(mergedid+self.architecture+ext)
2111  except:
2112  pass
2113 
2114  self.postcmd = self.environment + self.export + " && cd "+self.outlibs.rstrip("/")+" && "
2115  self.postcmd += librarian +" -x " + decomp.join(multi_libs) +" && "
2116  self.postcmd += librarian + " -rc " + mergedid+self.architecture+ext+ " *.o"
2117 
2118  Utility.SSHPopen(self.ssh_client, self.postcmd, self.verbose)
2119  for lib in multi_libs:
2120  _, _, _ = self.ssh_client.exec_command('rm -f '+posixpath.join(self.outlibs,lib))
2121  self.ssh_client.exec_command('rm -f '+self.outlibs+'*.o')
2122 
2123  # Go into scratch directory (if defined)
2124  with Utility.ChangedWorkingDirectory(self.scrtdir):
2125 
2126  # Finish and delete redundant files
2127  Utility.DeleteFilesbyEnding(self.temps)
2128 
2129 if __name__ == '__main__':
2130  pass
outmodule
Output path for module or header files.
Definition: Make.py:710
linkcmd
Intel Linker command.
Definition: Make.py:589
def create(self)
Definition: Make.py:349
MakeObjectKind
String identifier of current instance.
Definition: Make.py:1349
BuildOption
Ordered dictionary containing all available options.
Definition: Make.py:1493
srcdir
Default search directory for source files.
Definition: Make.py:97
isStatic
Static or dynamic link library flag.
Definition: Make.py:660
Base class for all PyInstaller build events.
Definition: Make.py:1015
def Settings(self, user, key, host='129.247.54.37', port=22)
Definition: Make.py:1908
Base class for all custom build events inherited from Make.
Definition: Make.py:404
exe
Executable of Doxygen.
Definition: Make.py:1360
verbose
Level of verbosity of the current build object.
Definition: Make.py:90
def __init__(self, args, kwargs)
Definition: Make.py:1019
workspace
Remote workspace.
Definition: Make.py:1915
exe
Executable of NSIS.
Definition: Make.py:1257
linkcmd
Intel Linker command.
Definition: Make.py:844
def __init__(self, args, kwargs)
Definition: Make.py:408
def Postprocessing(self, cmdstring="")
Definition: Make.py:1930
MakeObjectKind
String identifier of current instance.
Definition: Make.py:1477
def create(self, kwargs)
Definition: Make.py:1279
def FTP(self, user, key, upload_file, host='ftp.dlr.de', path="/public/download/nightly")
Definition: Make.py:1266
buildid
Base string of build object.
Definition: Make.py:79
libname
Name of library, assembled using BuildID.
Definition: Make.py:676
def Build(self, cmdstring, run="ifort", path="", lib="", linkedIn="")
Definition: Make.py:1938
def __init__(self, args, kwargs)
Definition: Make.py:934
ssh_client
Instance of SSHClient to establish a SSH connection.
Definition: Make.py:1917
isStatic
Static or dynamic link library flag.
Definition: Make.py:521
path2exe
Absolute system path to Python executable.
Definition: Make.py:958
def __init__(self, args, kwargs)
Definition: Make.py:1238
intelpath
Path to Intel Fortran Compiler (read from Paths.log).
Definition: Make.py:106
libdirs
List of library directories.
Definition: Make.py:116
incremental
Define if the input should be compiled exactly as provided.
Definition: Make.py:1876
outlibs
Output path for library files.
Definition: Make.py:712
def Encryption(self, encrypt, kwargs)
Definition: Make.py:1042
__library_path
Environment variable to be set prior to the execution of the build command.
Definition: Make.py:1870
def OutputPath(self, libpath, modulepath=None)
Definition: Make.py:1895
architecture
Processor architecture.
Definition: Make.py:136
__old_export
Environment variables to be set prior to the execution of the build command.
Definition: Make.py:1868
def __init__(self, args, kwargs)
Definition: Make.py:509
def OutputPath(self, libpath=os.getcwd())
Definition: Make.py:554
def __setstate__(self, _dict)
Definition: Make.py:180
iniCompiler
Executable batch script (including absolute system path) to set up the Intel Fortran Compiler...
Definition: Make.py:145
linkedIn
List of libraries which should be statically linked in.
Definition: Make.py:540
def OutputPath(self, modulepath=None, libpath=os.getcwd())
Definition: Make.py:702
def create(self, kwargs)
Definition: Make.py:1991
Base class for all Doxygen build events.
Definition: Make.py:1339
def Build(self, cmdstring)
Definition: Make.py:466
exe
The executable command used in the main build event.
Definition: Make.py:960
Base class for all build events requiring a SSH connection.
Definition: Make.py:1838
setarch
Define the architecture for the build directly by using the keyword argument "arch".
Definition: Make.py:126
environment
Load an additional library prior to execution of all commands.
Definition: Make.py:1887
def Build(self, mode="onefile", kwargs)
Definition: Make.py:1063
def Preprocessing(self, cmdstring='')
Definition: Make.py:1053
temps
Tuple of data to be removed after job completion.
Definition: Make.py:110
exe
The executable command used in all build events.
Definition: Make.py:417
Abstract base class for all make objects.
Definition: Make.py:67
def Wrapper(self, module_name, source_name=None)
Definition: Make.py:775
linkedIn
List of libraries which should be statically linked in.
Definition: Make.py:691
def __init__(self, args, kwargs)
Definition: Make.py:1842
libname
Name of library, assembled using BuildID.
Definition: Make.py:534
msvsc
Default version of Microsoft visual studio used by the Intel Fortran Compiler.
Definition: Make.py:151
incremental
Define if the input should be compiled exactly as provided.
Definition: Make.py:954
MakeObjectKind
String identifier of current instance.
Definition: Make.py:515
ftp_client
Remote workspace.
Definition: Make.py:1273
path2exe
Path to Sphinx executable.
Definition: Make.py:1489
def Settings(self, brief, header, outdir='', kwargs)
Definition: Make.py:1443
def SourcePath(self, path)
Definition: Make.py:212
MakeObjectKind
String identifier of current instance.
Definition: Make.py:414
precmd
Command executed during pre-build event.
Definition: Make.py:308
no_static_mkl
Define whether Intel&#39;s MKL should be statically or dynamically linked.
Definition: Make.py:946
def __init__(self, BuildID, Srcs, scratch=os.getcwd(), msvsc='vs2015', stype='Fortran', verbose=0, args, kwargs)
Definition: Make.py:72
copyfiles
List of files to be copied to the output directory after finish.
Definition: Make.py:122
def __init__(self, args, kwargs)
Definition: Make.py:1343
outlibs
Output path for library files.
Definition: Make.py:559
linkedIn
List of libraries which should be statically linked in.
Definition: Make.py:1880
Base class for all NSIS build events.
Definition: Make.py:1234
def Build(self, cmdstring)
Definition: Make.py:798
__intel_path
Custom intel path.
Definition: Make.py:1872
compargs
Command line arguments passed in by the user.
Definition: Make.py:336
def create(self)
Definition: Make.py:595
incdirs
List of include directories.
Definition: Make.py:114
path2exe
(Intel Fortran) Compiler Path
Definition: Make.py:1954
def AddDependencyPath(self, dependencies)
Definition: Make.py:196
def create(self, kwargs)
Definition: Make.py:1199
libs
List of actual libraries (by name) used during linking.
Definition: Make.py:118
def AddIncludePath(self, includes)
Definition: Make.py:188
srcs
Source file or folders.
Definition: Make.py:81
def __init__(self, args, kwargs)
Definition: Make.py:1471
Base class for all C/C++ build events inherited from Make.
Definition: Make.py:505
def Environment(self, path="", bash="", args="")
Definition: Make.py:1922
no_append_arch
Define whether the architecture shall be appended to the build name.
Definition: Make.py:950
def UseLibraries(self, libs)
Definition: Make.py:204
Base class for all Sphinx build events.
Definition: Make.py:1467
intermediate_wrapper
Wrapper interface file for 3rd party FORTRAN code.
Definition: Make.py:685
MakeObjectKind
String identifier of current instance.
Definition: Make.py:1858
incremental
Define if the input should be compiled exactly as provided.
Definition: Make.py:525
linkcmd
Remote Linker command.
Definition: Make.py:1982
MakeObjectKind
String identifier of current instance.
Definition: Make.py:942
exe
Executable of Sphinx.
Definition: Make.py:1491
Base class for all Py2X (for now only f2py) build events.
Definition: Make.py:930
def create(self, kwargs)
Definition: Make.py:853
Module containing all relevant modules and scripts associated with the building process.
Definition: __init__.py:1
exe
The executable command used in the main build event.
Definition: Make.py:518
makecmd
Command executed during build event.
Definition: Make.py:338
def __init__(self, args, kwargs)
Definition: Make.py:651
export
Environment variables to be set prior to the execution of the build command.
Definition: Make.py:1866
def Build(self, cmdstring)
Definition: Make.py:562
outdir
Default search directory for output.
Definition: Make.py:99
Base class for all Fortran build events.
Definition: Make.py:647
path2exe
Path to Doxygen executable.
Definition: Make.py:1358
MakeObjectKind
String identifier of current instance.
Definition: Make.py:1027
MakeObjectKind
String identifier of current instance.
Definition: Make.py:1246
def Environment(self, path, script="ifortvars.bat")
Definition: Make.py:229
intermediate_wrapper
Wrapper interface file for 3rd party FORTRAN code.
Definition: Make.py:1853
MakeObjectKind
String identifier of current instance.
Definition: Make.py:657
libname
Name of library, assembled using BuildID.
Definition: Make.py:1862
outmodule
Output path for module or header files.
Definition: Make.py:1903
path2exe
Path to NSIS executable.
Definition: Make.py:1255
def Settings(self, kwargs)
Definition: Make.py:1511
def create(self, kwargs)
Definition: Make.py:1521
scrtdir
Current scratch directory.
Definition: Make.py:95
stype
Source file type.
Definition: Make.py:85
outlibs
Output path for library files.
Definition: Make.py:1905
def Postprocessing(self, cmdstring='')
Definition: Make.py:341
def OutputPath(self, path, files="")
Definition: Make.py:219
def Build(self, cmdstring)
Definition: Make.py:314
postcmd
Post build command.
Definition: Make.py:155
def __getstate__(self)
Definition: Make.py:173