Coverage for /Users/Dave/git_repos/_packages_/python/fundamentals/fundamentals/mysql/directory_script_runner.py : 15%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#!/usr/local/bin/python
2# encoding: utf-8
3"""
4*Given a directory of MySQL scripts, execute the scripts and process the script files according to their success or failure*
6Run the following code once to set a login-path for your mysql server:
8 mysql_config_editor set --login-path=<uniqueLoginName> --host=localhost --user=<myUsername> --password
10This store's your credentials in an encrypted file located at '~/.mylogin.cnf'.
11Use `mysql_config_editor print --all` to see all of the login-paths set.
14Usage:
15 mysqlSucker <pathToDirectory> <loginPath> <databaseName> [-s successRule -f failureRule]
17 pathToDirectory path to the directory containing the sql scripts to run (scripts must have `.sql` extension)
18 loginPath the local-path as set with `mysql_config_editor` (`mysqlSucker -h` for more details)
19 databaseName the name of the database to execute the scripts within
21Options:
22 -h, --help show this help message
23 -s successRule, --success successRule what to do if script succeeds. Default *None* [None|delete|subFolderName]
24 -f failureRule, --failure failureRule what to do if script fails. Default *None* [None|delete|subFolderName]
26:Examples:
28 To simply execute the scripts in a directory you can run:
30 mysqlSucker /path/to/scriptdir myLoginPath myDatabaseName
32 To delete script after thay have executed successfully:
34 mysqlSucker /path/to/scriptdir myLoginPath myDatabaseName -s delete
36 To move successful script to a `passed` sub-directory of `/path/to/scriptdir` and failed scripts to a `failed` sub-directory
38 mysqlSucker /path/to/scriptdir myLoginPath myDatabaseName -s pass -f failed
40:Author:
41 David Young
43:Date Created:
44 September 22, 2016
45"""
46################# GLOBAL IMPORTS ####################
47from builtins import str
48import sys
49import os
50import time
51import collections
52import datetime
53from subprocess import Popen, PIPE, STDOUT
54os.environ['TERM'] = 'vt100'
55from fundamentals import tools
58def directory_script_runner(
59 log,
60 pathToScriptDirectory,
61 databaseName,
62 loginPath,
63 force=True,
64 waitForResult=True,
65 successRule=None,
66 failureRule=None):
67 """A function to run all of the mysql scripts in a given directory (in a modified date order, oldest first) and then act on the script file in accordance with the succcess or failure of its execution
69 As it's insecure to pass in mysql database credentials via the command-line, run the following command from the terminal
71 .. code-block:: bash
73 mysql_config_editor set --login-path=<uniqueLoginName> --host=localhost --user=<myUsername> --password
74 > Enter password:
76 This will store your credentials in an encrypted file located at '~/.mylogin.cnf'. This function takes advantage of the `--login-path` so as not to compromise the user's credentials. Use `mysql_config_editor print --all` to see all of the login-paths set.
78 **Key Arguments:**
79 - ``log`` -- logger
80 - ``pathToScriptDirectory`` -- the path to the directory containing the sql script to be run
81 - ``databaseName`` -- the name of the database
82 - ``force`` -- force the script to run, skipping over lines with errors, Default *True*
83 - ``loginPath`` -- the local-path as set with `mysql_config_editor`
84 - ``waitForResult`` -- wait for the mysql script to finish execution? If 'False' the MySQL script will run in background (do not wait for completion), or if 'delete' the script will run then delete regardless of success status. Default *True*. [True|False|delete]
85 - ``successRule`` -- what to do if script succeeds. Default *None* [None|delete|subFolderName]
86 - ``failureRule`` -- what to do if script fails. Default *None* [None|delete|subFolderName]
88 **Return:**
89 - None
91 **Usage:**
93 To run the scripts in the directroy and not act on the script file use something similar to:
95 .. code-block:: python
97 from fundamentals.mysql import directory_script_runner
98 directory_script_runner(
99 log=log,
100 pathToScriptDirectory="/path/to/mysql_scripts",
101 databaseName="imports",
102 loginPath="myLoginDetails"
103 )
105 To delete successful scripts and archive failed scripts for later inspection:
107 .. code-block:: python
109 from fundamentals.mysql import directory_script_runner
110 directory_script_runner(
111 log=log,
112 pathToScriptDirectory="/path/to/mysql_scripts",
113 databaseName="imports",
114 loginPath="myLoginDetails",
115 successRule="delete",
116 failureRule="failed"
117 )
119 This creates a folder at `/path/to/mysql_scripts/failed` and moves the failed scripts into that folder.
121 Finally to execute the scripts within a directory but not wait for the results to return (much fast but you lose error checking in the MySQL scripts):
123 .. code-block:: python
125 from fundamentals.mysql import directory_script_runner
126 directory_script_runner(
127 log=log,
128 pathToScriptDirectory="/path/to/mysql_scripts",
129 databaseName="imports",
130 loginPath="myLoginDetails",
131 waitForResults=False
132 )
134 Setting ``waitForResults`` = 'delete' will trash the script once it has run (or failed ... be very careful!)
135 """
136 log.debug('starting the ``directory_script_runner`` function')
138 # COMPILE A DICTIONARY OF SCRIPTS / MODIFIED TIMES
139 scriptList = {}
140 for d in os.listdir(pathToScriptDirectory):
141 filePath = os.path.join(pathToScriptDirectory, d)
142 filename = os.path.basename(filePath)
143 extension = filePath.split(".")[-1]
144 if os.path.isfile(filePath) and extension == "sql":
145 modified = datetime.datetime.strptime(time.ctime(
146 os.path.getmtime(filePath)), "%a %b %d %H:%M:%S %Y")
147 scriptList[str(modified) + filename] = filePath
149 # FORCE MYSQL SCRIPT?
150 if force:
151 force = "--force"
152 else:
153 force = ""
155 # ORDER THE DICTIONARY BY MODIFIED TIME - OLDEST FIRST
156 scriptList = collections.OrderedDict(sorted(scriptList.items()))
157 for k, v in list(scriptList.items()):
158 scriptname = os.path.basename(v)
159 if waitForResult == True:
160 cmd = """mysql --login-path=%(loginPath)s %(force)s %(databaseName)s < "%(v)s" """ % locals(
161 )
162 p = Popen(cmd, stdout=PIPE, stderr=PIPE, close_fds=True,
163 env={'PATH': os.getenv('PATH') + ":/usr/local/bin:/usr/bin:/usr/bin:/usr/local/mysql/bin", "MYSQL_TEST_LOGIN_FILE": os.getenv('HOME') + "/.mylogin.cnf"}, shell=True)
164 stdout, stderr = p.communicate()
166 if len(stderr):
167 log.error(
168 "MySQL Script `%(scriptname)s` Failed: '%(stderr)s'" % locals())
169 if failureRule == None or failureRule == False:
170 pass
171 elif failureRule == "delete":
172 os.remove(v)
173 elif "/" not in failureRule:
174 moveTo = pathToScriptDirectory + "/" + failureRule
175 # Recursively create missing directories
176 if not os.path.exists(moveTo):
177 os.makedirs(moveTo)
178 moveTo = moveTo + "/" + scriptname
179 try:
180 log.debug("attempting to rename file %s to %s" %
181 (v, moveTo))
182 os.rename(v, moveTo)
183 except Exception as e:
184 log.error(
185 "could not rename file %s to %s - failed with this error: %s " % (v, moveTo, str(e),))
186 else:
187 if successRule == None or successRule == False:
188 pass
189 elif successRule == "delete":
190 os.remove(v)
191 elif "/" not in successRule:
192 moveTo = pathToScriptDirectory + "/" + successRule
193 # Recursively create missing directories
194 if not os.path.exists(moveTo):
195 os.makedirs(moveTo)
196 moveTo = moveTo + "/" + scriptname
197 try:
198 log.debug("attempting to rename file %s to %s" %
199 (v, moveTo))
200 os.rename(v, moveTo)
201 except Exception as e:
202 log.error(
203 "could not rename file %s to %s - failed with this error: %s " % (v, moveTo, str(e),))
204 else:
205 if waitForResult == "delete":
206 cmd = """mysql --login-path=%(loginPath)s %(force)s %(databaseName)s < "%(v)s" > /dev/null 2>&1 & rm "%(v)s" """ % locals()
207 else:
208 cmd = """mysql --login-path=%(loginPath)s %(force)s %(databaseName)s < "%(v)s" > /dev/null 2>&1 """ % locals()
209 p = Popen(cmd, close_fds=True,
210 env={'PATH': os.getenv('PATH') + ":/usr/local/bin:/usr/bin:", "MYSQL_TEST_LOGIN_FILE": os.getenv('HOME') + "/.mylogin.cnf"}, shell=True, stdin=None, stdout=None, stderr=None)
212 log.debug('completed the ``directory_script_runner`` function')
213 return None
216def main(arguments=None):
217 """
218 The main function used when ``directory_script_runner.py`` is run as a single script from the cl, or when installed as a cl command
219 """
221 # setup the command-line util settings
222 su = tools(
223 arguments=arguments,
224 docString=__doc__,
225 logLevel="WARNING",
226 options_first=False,
227 projectName="fundmentals"
228 )
229 arguments, settings, log, dbConn = su.setup()
231 # UNPACK REMAINING CL ARGUMENTS USING `EXEC` TO SETUP THE VARIABLE NAMES
232 # AUTOMATICALLY
233 for arg, val in list(arguments.items()):
234 if arg[0] == "-":
235 varname = arg.replace("-", "") + "Flag"
236 else:
237 varname = arg.replace("<", "").replace(">", "")
238 if isinstance(val, ("".__class__, u"".__class__)):
239 exec(varname + " = '%s'" % (val,))
240 else:
241 exec(varname + " = %s" % (val,))
242 if arg == "--dbConn":
243 dbConn = val
244 log.debug('%s = %s' % (varname, val,))
246 if successFlag and successFlag.lower() == "none":
247 successFlag = None
248 if failureFlag and failureFlag.lower() == "none":
249 failureFlag = None
251 directory_script_runner(
252 log=log,
253 pathToScriptDirectory=pathToDirectory,
254 databaseName=databaseName,
255 loginPath=loginPath,
256 successRule=successFlag,
257 failureRule=failureFlag
258 )
260 return
262if __name__ == '__main__':
263 main()