Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1#!/usr/local/bin/python 

2# encoding: utf-8 

3""" 

4*Take key-values from a yaml file including a tablename(s) and add them to a mysql database table* 

5 

6Usage: 

7 yaml2db [-d] -s <pathToSettingsFile> <pathToYaml> 

8 yaml2db [-d] --host=<host> --user=<user> --passwd=<passwd> --dbName=<dbName> <pathToYaml> 

9 

10Options: 

11 

12 pathToYaml path to a single yaml file or directory of yaml files 

13 pathToSettingsFile path to a settings file with logging and database information (yaml file) 

14 --host=<host> the database host 

15 --user=<user> database user 

16 --passwd=<passwd> database user password 

17 --dbName=<dbName> name of the database to add the table content to 

18 

19 -d, --delete delete yaml open(s) once added to datbase 

20 -h, --help show this help message 

21 -v, --version show version 

22 -s, --settings the settings file 

23""" 

24from __future__ import print_function 

25 

26from builtins import object 

27import sys 

28import os 

29import yaml 

30import requests 

31import re 

32import glob 

33import docopt 

34from fundamentals import tools, times 

35from fundamentals.mysql import convert_dictionary_to_mysql_table 

36import requests.packages.urllib3 

37requests.packages.urllib3.disable_warnings() 

38 

39 

40def main(arguments=None): 

41 """ 

42 The main function used when ``yaml_to_database.py`` when installed as a cl tool 

43 """ 

44 

45 # setup the command-line util settings 

46 su = tools( 

47 arguments=arguments, 

48 docString=__doc__, 

49 logLevel="WARNING", 

50 options_first=False, 

51 projectName=False 

52 ) 

53 arguments, settings, log, dbConn = su.setup() 

54 

55 # unpack remaining cl arguments using `exec` to setup the variable names 

56 # automatically 

57 for arg, val in list(arguments.items()): 

58 if arg[0] == "-": 

59 varname = arg.replace("-", "") + "Flag" 

60 else: 

61 varname = arg.replace("<", "").replace(">", "") 

62 if isinstance(val, str): 

63 exec(varname + " = '%s'" % (val,)) 

64 else: 

65 exec(varname + " = %s" % (val,)) 

66 if arg == "--dbConn": 

67 dbConn = val 

68 log.debug('%s = %s' % (varname, val,)) 

69 

70 if os.path.isfile(pathToYaml): 

71 from fundamentals.mysql import yaml_to_database 

72 # PARSE YAML FILE CONTENTS AND ADD TO DATABASE 

73 yaml2db = yaml_to_database( 

74 log=log, 

75 settings=settings, 

76 dbConn=dbConn 

77 ) 

78 yaml2db.add_yaml_file_content_to_database( 

79 filepath=pathToYaml, 

80 deleteFile=deleteFlag 

81 ) 

82 basename = os.path.basename(pathToYaml) 

83 print("Content of %(basename)s added to database" % locals()) 

84 

85 else: 

86 from fundamentals.mysql import yaml_to_database 

87 yaml2db = yaml_to_database( 

88 log=log, 

89 settings=settings, 

90 dbConn=dbConn, 

91 pathToInputDir=pathToYaml, 

92 deleteFiles=deleteFlag 

93 ) 

94 yaml2db.ingest() 

95 print("Content of %(pathToYaml)s directory added to database" % locals()) 

96 

97 return 

98 

99 

100class yaml_to_database(object): 

101 """ 

102 *Take key-values from yaml files including a tablename(s) and add them to a mysql database table* 

103 

104 **Key Arguments:** 

105 - ``log`` -- logger 

106 - ``settings`` -- the settings dictionary 

107 - ``pathToInputDir`` -- path to the directory containing the yaml files that will be added to the database table(s). Default *False* 

108 - ``dbConn`` -- connection to database to add the content to 

109 - ``deleteFiles`` - - delete the yamls files once their content has been added to the database. Default * False* 

110 

111 **Usage:** 

112 

113 To setup your logger, settings and database connections, please use the ``fundamentals`` package (`see tutorial here <http://fundamentals.readthedocs.io/en/latest/#tutorial>`_).  

114 

115 To initiate a ``yaml2db`` object, use the following: 

116 

117 .. code-block:: python  

118 

119 from fundamentals.mysql import yaml_to_database 

120 yaml2db = yaml_to_database( 

121 log=log, 

122 settings=settings, 

123 dbConn=dbConn, 

124 pathToInputDir="/path/to/yaml/directory", 

125 deleteFiles=False 

126 )  

127 

128 And here's an example of the content in a yaml file that this ``yaml2db`` object can parse: 

129 

130 .. code-block:: yaml 

131 

132 title: Why you should do most of your text editing in : Sublime Text | Sublime Text Tips 

133 url: http://sublimetexttips.com/why-you-should-do-most-of-your-text-editing-in-sublime-text/?utm_source=drip&utm_medium=email&utm_campaign=editor-proliferation 

134 kind: webpage 

135 subtype: article 

136 table: web_articles,podcasts  

137 """ 

138 # Initialisation 

139 

140 def __init__( 

141 self, 

142 log, 

143 dbConn, 

144 pathToInputDir=False, 

145 settings=False, 

146 deleteFiles=False 

147 

148 ): 

149 self.log = log 

150 log.debug("instansiating a new 'yaml_to_database' object") 

151 self.settings = settings 

152 self.pathToInputDir = pathToInputDir 

153 self.dbConn = dbConn 

154 self.deleteFiles = deleteFiles 

155 # xt-self-arg-tmpx 

156 

157 return None 

158 

159 def ingest(self): 

160 """ 

161 *ingest the contents of the directory of yaml files into a database* 

162 

163 **Return:** 

164 - None 

165 

166 **Usage:** 

167 

168 To import an entire directory of yaml files into a database, use the following: 

169 

170 .. code-block:: python  

171 

172 from fundamentals.mysql import yaml_to_database 

173 yaml2db = yaml_to_database( 

174 log=log, 

175 settings=settings, 

176 dbConn=dbConn, 

177 pathToInputDir="/path/to/yaml/directory", 

178 deleteFiles=False 

179 )  

180 yaml2db.ingest()  

181 """ 

182 self.log.debug('starting the ``ingest`` method') 

183 

184 for d in os.listdir(self.pathToInputDir): 

185 if os.path.isfile(os.path.join(self.pathToInputDir, d)) and "yaml" in d.lower(): 

186 self.add_yaml_file_content_to_database( 

187 filepath=os.path.join(self.pathToInputDir, d), 

188 deleteFile=self.deleteFiles 

189 ) 

190 

191 self.log.debug('completed the ``ingest`` method') 

192 return None 

193 

194 def add_yaml_file_content_to_database( 

195 self, 

196 filepath, 

197 deleteFile=False 

198 ): 

199 """*given a file to a yaml file, add yaml file content to database* 

200 

201 **Key Arguments:** 

202 - ``filepath`` -- the path to the yaml file 

203 - ``deleteFile`` -- delete the yaml file when its content has been added to the database. Default *False* 

204 

205 **Return:** 

206 - None 

207 

208 **Usage:** 

209 

210 To parse and import the contents of a single yaml file into the database, use the following: 

211 

212 .. code-block:: python  

213 

214 from fundamentals.mysql import yaml_to_database 

215 # PARSE YAML FILE CONTENTS AND ADD TO DATABASE 

216 yaml2db = yaml_to_database( 

217 log=log, 

218 settings=settings, 

219 dbConn=dbConn 

220 )  

221 yaml2db.add_yaml_file_content_to_database( 

222 filepath=${1:"/path/to/file.yaml"}, 

223 deleteFile=True 

224 ) 

225 """ 

226 self.log.debug( 

227 'completed the ````add_yaml_file_content_to_database`` method') 

228 

229 import codecs 

230 try: 

231 self.log.debug("attempting to open the file %s" % (filepath,)) 

232 readFile = codecs.open(filepath, encoding='utf-8', mode='r') 

233 thisData = readFile.read() 

234 readFile.close() 

235 except IOError as e: 

236 message = 'could not open the file %s' % (filepath,) 

237 self.log.critical(message) 

238 raise IOError(message) 

239 readFile.close() 

240 

241 matchObject = re.finditer( 

242 r'(^|\n)(?P<key>[^\:]*)\:\s(?P<value>.*?)(\n|$)', 

243 thisData, 

244 flags=re.M | re.S # re.S 

245 ) 

246 

247 yamlContent = {} 

248 for match in matchObject: 

249 if match.group("value")[0] == '"' and match.group("value")[-1] == '"': 

250 v = match.group("value")[1:-1] 

251 elif match.group("value")[0] == "'" and match.group("value")[-1] == "'": 

252 v = match.group("value")[1:-1] 

253 else: 

254 v = match.group("value") 

255 yamlContent[match.group("key")] = v 

256 

257 if "table" not in yamlContent: 

258 self.log.warning( 

259 'A table value is need in the yaml content to indicate which database table to add the content to: %(filepath)s' % locals()) 

260 return None 

261 

262 # NOTE THERE MAY BE MORE THAN ONE DATABASE TABLE 

263 dbTablesTmp = yamlContent["table"].split(",") 

264 del yamlContent["table"] 

265 dbTables = [] 

266 dbTables[:] = [d.strip() for d in dbTablesTmp] 

267 

268 # UNSHORTEN URL 

269 try: 

270 r = requests.head(yamlContent["url"], allow_redirects=True) 

271 yamlContent["url"] = r.url 

272 except: 

273 pass 

274 

275 yamlContent["original_yaml_path"] = filepath 

276 

277 if "url" in yamlContent: 

278 uniqueKeyList = ["url"] 

279 else: 

280 uniqueKeyList = [] 

281 

282 for t in dbTables: 

283 convert_dictionary_to_mysql_table( 

284 dbConn=self.dbConn, 

285 log=self.log, 

286 dictionary=yamlContent, 

287 dbTableName=t, 

288 uniqueKeyList=uniqueKeyList, 

289 dateModified=True, 

290 returnInsertOnly=False, 

291 replace=True 

292 ) 

293 if deleteFile: 

294 os.remove(filepath) 

295 

296 self.log.debug( 

297 'completed the ``add_yaml_file_content_to_database`` method') 

298 return None 

299 

300 # use the tab-trigger below for new method 

301 # xt-class-method 

302 

303 

304if __name__ == '__main__': 

305 main()