Coverage for src/edwh_restic_plugin/tasks.py: 0%

82 statements  

« prev     ^ index     » next       coverage.py v7.6.12, created at 2025-02-28 16:28 +0100

1import json 

2import os 

3import subprocess 

4import typing 

5 

6import invoke 

7from edwh.tasks import DOCKER_COMPOSE 

8from invoke import Context, task 

9from termcolor import cprint 

10 

11from .env import DOTENV, read_dotenv, set_env_value 

12from .helpers import _require_restic 

13from .repositories import Repository, registrations 

14from .restictypes import DockerContainer 

15 

16 

17def cli_repo(connection_choice: str = None, restichostname: str = None) -> Repository: 

18 """ 

19 Create a repository object and set up the connection to the backend. 

20 :param connection_choice: choose where you want to store the repo (local, SFTP, B2, swift) 

21 :param restichostname: which hostname to force for restic, or blank for default. 

22 :return: repository object 

23 """ 

24 env = read_dotenv(DOTENV) 

25 if restichostname: 

26 set_env_value(DOTENV, "RESTICHOSTNAME", restichostname) 

27 

28 options = registrations.to_ordered_dict() 

29 

30 connection_lowercase = "" 

31 if connection_choice is None: 

32 # search for the most important backup and use it as default 

33 for option in options: 

34 if f"{option.upper()}_NAME" in env: 

35 connection_lowercase = option.lower() 

36 break 

37 else: 

38 connection_lowercase = connection_choice.lower() 

39 

40 if not (repoclass := registrations.get(connection_lowercase)): 

41 _options = ", ".join(list(options)) 

42 raise ValueError(f"Invalid connection type {connection_choice}. Please use one of {_options}!") 

43 

44 print("Use connection: ", connection_lowercase) 

45 repo = repoclass() 

46 repo.setup() 

47 return repo 

48 

49 

50@task 

51def require_restic(c): 

52 _require_restic(c) 

53 

54 

55@task(aliases=("setup", "init")) 

56def configure(c, connection_choice=None, restichostname=None): 

57 """Setup or update the backup command for your environment. 

58 connection_choice: choose where you want to store the repo (local, SFTP, B2, swift) 

59 restichostname: which hostname to force for restic, or blank for default. 

60 """ 

61 

62 # It has been decided to create a main path called 'backups' for each repository. 

63 # This can be changed or removed if desired. 

64 # A password is only passed with a few functions. 

65 cli_repo(connection_choice, restichostname).configure(c) 

66 

67 

68@task 

69def backup(c, target: str = "", connection_choice: str = None, message: str = None, verbose: bool = True): 

70 """Performs a backup operation using restic on a local or remote/cloud file system. 

71 

72 Args: 

73 c (Context) 

74 target (str): The target of the backup (e.g. 'files', 'stream'; default is all types). 

75 connection_choice (str): The name of the connection to use for the backup. 

76 Defaults to None, which means the default connection will be used. 

77 message (str): A message to attach to the backup snapshot. 

78 Defaults to None, which means no message will be attached. 

79 verbose (bool): If True, outputs more information about the backup process. Defaults to False. 

80 

81 Raises: 

82 Exception: If an error occurs during the backup process. 

83 

84 """ 

85 # After 'backup', a file path can be specified.In this script, a test file is chosen at './test/testbestand'. 

86 # It can be replaced with the desired path over which restic should perform a backup. 

87 # The option --verbose provides more information about the backup that is made.It can be removed if desired. 

88 

89 # By using additions, it is possible to specify what should be included: 

90 # --exclude ,Specified one or more times to exclude one or more items. 

91 # --iexclude, Same as --exclude but ignores the case of paths. 

92 # --exclude-caches, Specified once to exclude folders containing this special file. 

93 # --exclude-file, Specified one or more times to exclude items listed in a given file. 

94 # --iexclude-file, Same as exclude-file but ignores cases like in --iexclude. 

95 # --exclude-if-present 'foo', Specified one or more times to exclude a folder's content if it contains. 

96 # a file called 'foo' (optionally having a given header, no wildcards for the file name supported). 

97 # --exclude-larger-than 'size', Specified once to excludes files larger than the given size. 

98 # Please see 'restic help backup' for more specific information about each exclude option. 

99 

100 cli_repo(connection_choice).backup(c, verbose, target, message) 

101 

102 

103@task 

104def restore(c, connection_choice: str = None, snapshot: str = "latest", target: str = "", verbose: bool = True): 

105 """ 

106 The restore function restores the latest backed-up files by default and puts them in a restore folder. 

107 

108 IMPORTANT: please provide -t for the path where the restore should go. Also remember to include -c for the service 

109 where the backup is stored. 

110 

111 :type c: Context 

112 :param connection_choice: the service where the files are backed up, e.g., 'local' or 'os' (= openstack). 

113 :param snapshot: the ID where the files are backed up, default value is 'latest'. 

114 :param target: The target of the backup (e.g. 'files', 'stream'; default is all types). 

115 :param verbose: display verbose logs (inv restore -v). 

116 :return: None 

117 """ 

118 # For restore, --target is the location where the restore should be placed, --path is the file/path that should be 

119 # retrieved from the repository. 

120 # 'which_restore' is a user input to enable restoring an earlier backup (default = latest). 

121 # Stop the postgres services. 

122 c.run(f"{DOCKER_COMPOSE} stop -t 1 pg-0 pg-1 pgpool", warn=True, hide=True) 

123 

124 # Get the volumes that are being used. 

125 docker_inspect: invoke.Result = c.run("docker inspect pg-0 pg-1", hide=True, warn=True) 

126 if docker_inspect.ok: 

127 # Only if ok, because if pg-0 and pg-1 do not exist, this does not exist either, and nothing needs to be removed 

128 inspected: list[DockerContainer] = json.loads(docker_inspect.stdout) 

129 volumes_to_remove: list[str] = [] 

130 for service in inspected: 

131 volumes_to_remove.extend(mount["Name"] for mount in service["Mounts"] if mount["Type"] == "volume") 

132 # Remove the containers before a volume can be removed. 

133 c.run(f"{DOCKER_COMPOSE} rm -f pg-0 pg-1") 

134 # Remove the volumes. 

135 for volume_name in volumes_to_remove: 

136 c.run(f"docker volume rm {volume_name}") 

137 

138 cli_repo(connection_choice).restore(c, verbose, target, snapshot) 

139 # print("`inv up` to restart the services.") 

140 

141 

142@task(iterable=["tag"]) 

143def snapshots(c, connection_choice: str = None, tag: list[str] = None, n: int = 1, verbose: bool = False): 

144 """ 

145 With this you can see per repo which repo is made when and where, \ 

146 the repo-id can be used at inv restore as an option 

147 

148 :type c: Context 

149 :param connection_choice: service 

150 :param tag: files, stream ect 

151 :param n: amount of snapshot to view, default=1(latest) 

152 :param verbose: show which commands are being executed? 

153 :return: None 

154 """ 

155 # if tags is None set tag to default tags 

156 if tag is None: 

157 tag = ["files", "stream"] 

158 

159 cli_repo(connection_choice).snapshot(c, tags=tag, n=n, verbose=verbose) 

160 

161 

162def interactive(conn: Repository): 

163 subprocess.run(["/bin/bash", "--norc", "--noprofile"], env=os.environ | {"PS1": f"{conn!r} $ "}) 

164 

165 

166@task(pre=[require_restic]) 

167def run(c, connection_choice: str = None, command: typing.Optional[str] = None): 

168 """ 

169 This function prepares for restic and runs the input command until the user types "exit". 

170 

171 :type c: Context 

172 :param connection_choice: The connection name of the repository. 

173 :param command: restic subcommand to use (default: none, prompt interactively) 

174 """ 

175 conn = cli_repo(connection_choice) 

176 conn.prepare_for_restic(c) 

177 

178 if command: 

179 if not command.startswith("restic "): 

180 command = f"restic {command}" 

181 print(c.run(command, hide=True, warn=True, pty=True)) 

182 else: 

183 interactive(conn) 

184 

185 

186@task() 

187def env(c, connection_choice: str = None): 

188 """ 

189 

190 :type c: Context 

191 :param connection_choice: The connection name of the repository. 

192 """ 

193 from copy import deepcopy 

194 

195 old = deepcopy(os.environ) 

196 cli_repo(connection_choice).prepare_for_restic(c) 

197 new = os.environ 

198 for k, v in new.items(): 

199 if k not in old or old[k] != v: 

200 print(f"export {k.upper()}={v}") 

201 

202@task() 

203def forget(c: Context, connection: str = None): 

204 # https://restic.readthedocs.io/en/latest/060_forget.html#removing-snapshots-according-to-a-policy 

205 repo = cli_repo(connection) 

206 

207 print( 

208 repo.forget(c) 

209 )