docs for muutils v0.6.18
View Source on GitHub

muutils.logger.log_util


 1from muutils.jsonlines import jsonl_load_log
 2
 3
 4def get_any_from_stream(stream: list[dict], key: str) -> None:
 5    """get the first value of a key from a stream. errors if not found"""
 6    for msg in stream:
 7        if key in msg:
 8            return msg[key]
 9
10    raise KeyError(f"key '{key}' not found in stream")
11
12
13def gather_log(file: str) -> dict[str, list[dict]]:
14    """gathers and sorts all streams from a log"""
15    data: list[dict] = jsonl_load_log(file)
16    output: dict[str, list[dict]] = dict()
17
18    for item in data:
19        stream: str = item.get("_stream", None)
20        if stream not in output:
21            output[stream] = list()
22        output[stream].append(item)
23
24    return output
25
26
27def gather_stream(
28    file: str,
29    stream: str,
30) -> list[dict]:
31    """gets all entries from a specific stream in a log file"""
32    data: list[dict] = jsonl_load_log(file)
33
34    output: list[dict] = list()
35
36    for item in data:
37        # select for the stream
38        if ("_stream" in item) and (item["_stream"] == stream):
39            output.append(item)
40    return output
41
42
43def gather_val(
44    file: str,
45    stream: str,
46    keys: tuple[str],
47    allow_skip: bool = True,
48) -> list[list]:
49    """gather specific keys from a specific stream in a log file
50
51    example:
52    if "log.jsonl" has contents:
53    ```jsonl
54    {"a": 1, "b": 2, "c": 3, "_stream": "s1"}
55    {"a": 4, "b": 5, "c": 6, "_stream": "s1"}
56    {"a": 7, "b": 8, "c": 9, "_stream": "s2"}
57    ```
58    then `gather_val("log.jsonl", "s1", ("a", "b"))` will return
59    ```python
60    [
61        [1, 2],
62        [4, 5]
63    ]
64    ```
65
66    """
67    data: list[dict] = jsonl_load_log(file)
68
69    output: list[list] = list()
70
71    for item in data:
72        # select for the stream
73        if ("_stream" in item) and (item["_stream"] == stream):
74            # select for the keys
75            if all(k in item for k in keys):
76                output.append(list(item[k] for k in keys))
77            elif not allow_skip:
78                raise ValueError(f"missing keys '{keys = }' in '{item = }'")
79
80    return output

def get_any_from_stream(stream: list[dict], key: str) -> None:
 5def get_any_from_stream(stream: list[dict], key: str) -> None:
 6    """get the first value of a key from a stream. errors if not found"""
 7    for msg in stream:
 8        if key in msg:
 9            return msg[key]
10
11    raise KeyError(f"key '{key}' not found in stream")

get the first value of a key from a stream. errors if not found

def gather_log(file: str) -> dict[str, list[dict]]:
14def gather_log(file: str) -> dict[str, list[dict]]:
15    """gathers and sorts all streams from a log"""
16    data: list[dict] = jsonl_load_log(file)
17    output: dict[str, list[dict]] = dict()
18
19    for item in data:
20        stream: str = item.get("_stream", None)
21        if stream not in output:
22            output[stream] = list()
23        output[stream].append(item)
24
25    return output

gathers and sorts all streams from a log

def gather_stream(file: str, stream: str) -> list[dict]:
28def gather_stream(
29    file: str,
30    stream: str,
31) -> list[dict]:
32    """gets all entries from a specific stream in a log file"""
33    data: list[dict] = jsonl_load_log(file)
34
35    output: list[dict] = list()
36
37    for item in data:
38        # select for the stream
39        if ("_stream" in item) and (item["_stream"] == stream):
40            output.append(item)
41    return output

gets all entries from a specific stream in a log file

def gather_val( file: str, stream: str, keys: tuple[str], allow_skip: bool = True) -> list[list]:
44def gather_val(
45    file: str,
46    stream: str,
47    keys: tuple[str],
48    allow_skip: bool = True,
49) -> list[list]:
50    """gather specific keys from a specific stream in a log file
51
52    example:
53    if "log.jsonl" has contents:
54    ```jsonl
55    {"a": 1, "b": 2, "c": 3, "_stream": "s1"}
56    {"a": 4, "b": 5, "c": 6, "_stream": "s1"}
57    {"a": 7, "b": 8, "c": 9, "_stream": "s2"}
58    ```
59    then `gather_val("log.jsonl", "s1", ("a", "b"))` will return
60    ```python
61    [
62        [1, 2],
63        [4, 5]
64    ]
65    ```
66
67    """
68    data: list[dict] = jsonl_load_log(file)
69
70    output: list[list] = list()
71
72    for item in data:
73        # select for the stream
74        if ("_stream" in item) and (item["_stream"] == stream):
75            # select for the keys
76            if all(k in item for k in keys):
77                output.append(list(item[k] for k in keys))
78            elif not allow_skip:
79                raise ValueError(f"missing keys '{keys = }' in '{item = }'")
80
81    return output

gather specific keys from a specific stream in a log file

example: if "log.jsonl" has contents:

{"a": 1, "b": 2, "c": 3, "_stream": "s1"}
{"a": 4, "b": 5, "c": 6, "_stream": "s1"}
{"a": 7, "b": 8, "c": 9, "_stream": "s2"}

then gather_val("log.jsonl", "s1", ("a", "b")) will return

[
    [1, 2],
    [4, 5]
]