Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1#!/usr/local/bin/python 

2# encoding: utf-8 

3""" 

4Take a sqlite database file and copy the tables within it to a MySQL database 

5 

6Usage: 

7 sqlite2mysql -s <pathToSettingsFile> <pathToSqliteDB> [<tablePrefix>] 

8 

9Options: 

10 

11 pathToSqliteDB path to the sqlite database file 

12 tablePrefix a string to prefix the table names when copying to mysql database 

13 pathToSettingsFile path to a settings file with logging and database information (yaml file) 

14 

15 -h, --help show this help message 

16 -v, --version show version 

17 -s, --settings the settings file 

18""" 

19################# GLOBAL IMPORTS #################### 

20from builtins import object 

21import sys 

22import os 

23import sqlite3 as lite 

24os.environ['TERM'] = 'vt100' 

25from fundamentals import tools 

26from fundamentals.mysql import writequery 

27 

28from datetime import datetime, date, time 

29 

30 

31def main(arguments=None): 

32 """ 

33 The main function used when ``yaml_to_database.py`` when installed as a cl tool 

34 """ 

35 

36 # setup the command-line util settings 

37 su = tools( 

38 arguments=arguments, 

39 docString=__doc__, 

40 logLevel="WARNING", 

41 options_first=False, 

42 projectName=False 

43 ) 

44 arguments, settings, log, dbConn = su.setup() 

45 

46 # unpack remaining cl arguments using `exec` to setup the variable names 

47 # automatically 

48 for arg, val in list(arguments.items()): 

49 if arg[0] == "-": 

50 varname = arg.replace("-", "") + "Flag" 

51 else: 

52 varname = arg.replace("<", "").replace(">", "") 

53 if isinstance(val, str): 

54 exec(varname + " = '%s'" % (val,)) 

55 else: 

56 exec(varname + " = %s" % (val,)) 

57 if arg == "--dbConn": 

58 dbConn = val 

59 log.debug('%s = %s' % (varname, val,)) 

60 

61 from fundamentals.mysql import sqlite2mysql 

62 converter = sqlite2mysql( 

63 log=log, 

64 settings=settings, 

65 pathToSqlite=pathToSqliteDB, 

66 tablePrefix=tablePrefix, 

67 dbConn=dbConn 

68 ) 

69 converter.convert_sqlite_to_mysql() 

70 

71 return 

72 

73 

74class sqlite2mysql(object): 

75 """ 

76 *Take a sqlite database file and copy the tables within it to a MySQL database* 

77 

78 **Key Arguments:** 

79 - ``log`` -- logger 

80 - ``settings`` -- the settings dictionary 

81 - ``pathToSqlite`` -- path to the sqlite database to transfer into the MySQL database 

82 - ``tablePrefix`` -- a prefix to add to all the tablename when converting to mysql. Default *""* 

83 - ``dbConn`` -- mysql database connection  

84 

85 **Usage:** 

86 

87 To setup your logger, settings and database connections, please use the ``fundamentals`` package (`see tutorial here <http://fundamentals.readthedocs.io/en/latest/#tutorial>`_).  

88 

89 To convert and import the content of a sqlite database into MySQL run the following: 

90 

91 .. todo:: 

92 

93 - add a tutorial about ``sqlite2mysql`` to documentation 

94 

95 .. code-block:: python  

96 

97 from fundamentals.mysql import sqlite2mysql 

98 converter = sqlite2mysql( 

99 log=log, 

100 settings=settings, 

101 pathToSqlite="/path/to/sqlite.db", 

102 tablePrefix="external" 

103 ) 

104 converter.convert_sqlite_to_mysql() 

105 """ 

106 # Initialisation 

107 

108 def __init__( 

109 self, 

110 log, 

111 pathToSqlite, 

112 tablePrefix="", 

113 settings=False, 

114 dbConn=False 

115 ): 

116 self.log = log 

117 log.debug("instansiating a new 'sqlite2mysql' object") 

118 self.settings = settings 

119 self.pathToSqlite = pathToSqlite 

120 self.tablePrefix = tablePrefix 

121 self.dbConn = dbConn 

122 

123 if not self.tablePrefix: 

124 self.tablePrefix = "" 

125 

126 if len(self.tablePrefix): 

127 self.tablePrefix = self.tablePrefix + "_" 

128 # xt-self-arg-tmpx 

129 

130 return None 

131 

132 def convert_sqlite_to_mysql( 

133 self): 

134 """*copy the contents of the sqlite database into the mysql database* 

135 

136 See class docstring for usage 

137 """ 

138 from fundamentals.renderer import list_of_dictionaries 

139 from fundamentals.mysql import directory_script_runner 

140 self.log.debug('starting the ``convert_sqlite_to_mysql`` method') 

141 

142 con = lite.connect(self.pathToSqlite) 

143 con.row_factory = lite.Row 

144 cur = con.cursor() 

145 

146 # GET ALL TABLE NAMES 

147 cur.execute("SELECT name FROM sqlite_master WHERE type='table';") 

148 tables = cur.fetchall() 

149 

150 createStatements = [] 

151 inserts = [] 

152 for table in tables: 

153 table = table['name'] 

154 if table == "sqlite_sequence": 

155 continue 

156 

157 # CREATE TABLE collection_books (folder_id, fingerprint, primary key(folder_id, fingerprint)); 

158 # GENEREATE THE MYSQL CREATE STATEMENTS FOR EACH TABLE 

159 cur.execute( 

160 "SELECT sql FROM sqlite_master WHERE name = '%(table)s';" % locals()) 

161 createStatement = cur.fetchone() 

162 createStatement = createStatement[0].replace('"', '`') + ";" 

163 if "DEFAULT" not in createStatement: 

164 if "primary key(" in createStatement: 

165 tmp = createStatement.split("primary key(") 

166 tmp[0] = tmp[0].replace( 

167 ",", " varchar(150) DEFAULT NULL,") 

168 createStatement = ("primary key(").join(tmp) 

169 if "primary key," in createStatement: 

170 tmp = createStatement.split("primary key,") 

171 tmp[1] = tmp[1].replace( 

172 ",", " varchar(150) DEFAULT NULL,") 

173 tmp[1] = tmp[1].replace( 

174 ");", " varchar(150) DEFAULT NULL);") 

175 createStatement = ("primary key,").join(tmp) 

176 createStatement = createStatement.replace( 

177 "INTEGER PRIMARY KEY", "INTEGER AUTO_INCREMENT PRIMARY KEY") 

178 createStatement = createStatement.replace( 

179 "AUTOINCREMENT", "AUTO_INCREMENT") 

180 createStatement = createStatement.replace( 

181 "DEFAULT 't'", "DEFAULT '1'") 

182 createStatement = createStatement.replace( 

183 "DEFAULT 'f'", "DEFAULT '0'") 

184 createStatement = createStatement.replace(",'t'", ",'1'") 

185 createStatement = createStatement.replace(",'f'", ",'0'") 

186 if "CREATE TABLE `" in createStatement: 

187 createStatement = createStatement.replace( 

188 "CREATE TABLE `", "CREATE TABLE IF NOT EXISTS `" + self.tablePrefix) 

189 else: 

190 createStatement = createStatement.replace( 

191 "CREATE TABLE ", "CREATE TABLE IF NOT EXISTS " + self.tablePrefix) 

192 if ", primary key(" in createStatement: 

193 createStatement = createStatement.replace(", primary key(", """, 

194`dateCreated` datetime DEFAULT CURRENT_TIMESTAMP, 

195`dateLastModified` datetime DEFAULT CURRENT_TIMESTAMP, 

196`updated` tinyint(4) DEFAULT '0', 

197primary key(""") 

198 else: 

199 createStatement = createStatement.replace(");", """, 

200 `dateCreated` datetime DEFAULT CURRENT_TIMESTAMP, 

201 `dateLastModified` datetime DEFAULT CURRENT_TIMESTAMP, 

202 `updated` tinyint(4) DEFAULT '0'); 

203 """) 

204 createStatement = createStatement.replace( 

205 " text primary key", " varchar(100) primary key") 

206 createStatement = createStatement.replace( 

207 "`EntryText` TEXT NOT NULL,", "`EntryText` TEXT,") 

208 createStatement = createStatement.replace( 

209 "`SelectionText` TEXT NOT NULL", "`SelectionText` TEXT") 

210 createStatement = createStatement.replace( 

211 "`Filename` INTEGER NOT NULL,", "`Filename` TEXT NOT NULL,") 

212 createStatement = createStatement.replace( 

213 "`SessionPartUUID` TEXT NOT NULL UNIQUE,", "`SessionPartUUID` VARCHAR(100) NOT NULL UNIQUE,") 

214 createStatement = createStatement.replace( 

215 "`Name` TEXT PRIMARY KEY NOT NULL", "`Name` VARCHAR(100) PRIMARY KEY NOT NULL") 

216 createStatement = createStatement.replace( 

217 " VARCHAR ", " VARCHAR(100) ") 

218 createStatement = createStatement.replace( 

219 " VARCHAR,", " VARCHAR(100),") 

220 

221 if len(createStatement.lower().split("datecreated")) > 2: 

222 createStatement = createStatement.replace( 

223 "`dateCreated` datetime DEFAULT CURRENT_TIMESTAMP,\n", "") 

224 

225 # GRAB THE DATA TO ADD TO THE MYSQL DATABASE TABLES 

226 cur.execute( 

227 "SELECT * from '%(table)s';" % locals()) 

228 rows = cur.fetchall() 

229 

230 allRows = [] 

231 for row in rows: 

232 allRows.append(dict(row)) 

233 

234 # RECURSIVELY CREATE MISSING DIRECTORIES 

235 if not os.path.exists("/tmp/headjack/"): 

236 os.makedirs("/tmp/headjack/") 

237 

238 writequery( 

239 log=self.log, 

240 sqlQuery=createStatement, 

241 dbConn=self.dbConn, 

242 ) 

243 

244 from fundamentals.mysql import insert_list_of_dictionaries_into_database_tables 

245 # USE dbSettings TO ACTIVATE MULTIPROCESSING 

246 insert_list_of_dictionaries_into_database_tables( 

247 dbConn=self.dbConn, 

248 log=self.log, 

249 dictList=allRows, 

250 dbTableName=self.tablePrefix + table, 

251 uniqueKeyList=[], 

252 dateModified=True, 

253 dateCreated=True, 

254 batchSize=10000, 

255 replace=True, 

256 dbSettings=self.settings["database settings"] 

257 ) 

258 

259 # # DUMP THE DATA INTO A MYSQL DATABASE 

260 # dataSet = list_of_dictionaries( 

261 # log=self.log, 

262 # listOfDictionaries=allRows 

263 # ) 

264 # originalList = dataSet.list 

265 # now = datetime.now() 

266 # now = now.strftime("%Y%m%dt%H%M%S%f.sql") 

267 # mysqlData = dataSet.mysql( 

268 # tableName=self.tablePrefix + table, filepath="/tmp/headjack/" + 

269 # now, createStatement=createStatement) 

270 

271 # directory_script_runner( 

272 # log=self.log, 

273 # pathToScriptDirectory="/tmp/headjack/", 

274 # databaseName=self.settings["database settings"]["db"], 

275 # loginPath=self.settings["database settings"]["loginPath"], 

276 # successRule="delete", 

277 # failureRule="failed" 

278 # ) 

279 

280 con.close() 

281 

282 self.log.debug('completed the ``convert_sqlite_to_mysql`` method') 

283 return None 

284 

285 # use the tab-trigger below for new method 

286 # xt-class-method 

287 

288if __name__ == '__main__': 

289 main()