csvpath.managers.files.files_listener

 1import os
 2import json
 3from datetime import datetime
 4from csvpath.util.exceptions import InputException, FileException
 5from csvpath.util.file_readers import DataFileReader
 6from csvpath.util.file_writers import DataFileWriter
 7from csvpath.util.nos import Nos
 8from csvpath.managers.listener import Listener
 9from csvpath.managers.metadata import Metadata
10
11
12class FilesListener(Listener):  # Registrar,
13    """@private
14    this listener tracks all named-file arrivals"""
15
16    def __init__(self, csvpaths=None):
17        #
18        # FileListener is the primary listener. however,
19        # we want another file listener that tracks all files
20        # staged at the inputs/named_files level.
21        #
22        Listener.__init__(self, csvpaths.config if csvpaths else None)
23        self.csvpaths = csvpaths
24        self.config = None
25        if self.csvpaths:
26            self.config = csvpaths.config
27        self.type_name = "files"
28
29    def manifest_path(self) -> str:
30        root = self.config.get(section="inputs", name="files")
31        mf = os.path.join(root, "manifest.json")
32        if not Nos(mf).exists():
33            with DataFileWriter(path=mf, mode="w") as writer:
34                writer.append("[]")
35        return mf
36
37    def get_manifest(self, mpath) -> list:
38        with DataFileReader(mpath) as reader:
39            return json.load(reader.source)
40
41    def _core_from_metadata(self, mdata: Metadata) -> dict:
42        mani = {}
43        mani["time"] = mdata.time_string
44        mani["uuid"] = mdata.uuid_string
45        mani["file_manifest"] = mdata.manifest_path
46        if mdata.username:
47            mani["username"] = mdata.username
48        if mdata.hostname:
49            mani["hostname"] = mdata.hostname
50        if mdata.ip_address:
51            mani["ip_address"] = mdata.ip_address
52        return mani
53
54    def _prep_update(self, mdata: Metadata) -> dict:
55        mani = self._core_from_metadata(mdata)
56        mani["type"] = mdata.type
57        mani["time"] = mdata.time_string
58        mani["named_file_name"] = mdata.named_file_name
59        mani["origin_path"] = mdata.origin_path
60        mani["fingerprint"] = mdata.fingerprint
61        mani["file_path"] = mdata.file_path
62        mani["file_home"] = mdata.file_home
63        mani["file_name"] = mdata.file_name
64        mani["name_home"] = mdata.name_home
65        if mdata.mark is not None:
66            mani["mark"] = mdata.mark
67        mani["manifest_path"] = self.manifest_path()
68        return mani
69
70    def metadata_update(self, mdata: Metadata) -> None:
71        mani = self._prep_update(mdata)
72        manifest_path = self.manifest_path()
73        #
74        # this will not be Ok for major implementations. there is a
75        # potential for lost updates. however, for small or low risk
76        # implementations it will work. replacing with a concurrancy-safe
77        # listener would be straightforward. a future todo or some kind
78        # of upgrade above the stock library?
79        #
80        jdata = self.get_manifest(manifest_path)
81        jdata.append(mani)
82        with DataFileWriter(path=manifest_path, mode="w") as writer:
83            json.dump(jdata, writer.sink, indent=2)