Coverage for maze_dataset\tokenization\__init__.py: 100%
2 statements
« prev ^ index » next coverage.py v7.6.12, created at 2025-02-23 12:49 -0700
« prev ^ index » next coverage.py v7.6.12, created at 2025-02-23 12:49 -0700
1"""turning a maze into text
3- `MazeTokenizerModular` is the new recommended way to do this as of 1.0.0
4- legacy `TokenizationMode` enum and `MazeTokenizer` class for supporting existing code
5- a whole lot of helper classes and functions
7"""
9from maze_dataset.tokenization.maze_tokenizer import (
10 AdjListTokenizers,
11 CoordTokenizers,
12 EdgeGroupings,
13 EdgePermuters,
14 EdgeSubsets,
15 MazeTokenizer,
16 MazeTokenizerModular,
17 PathTokenizers,
18 PromptSequencers,
19 StepSizes,
20 StepTokenizers,
21 TargetTokenizers,
22 TokenizationMode,
23 _TokenizerElement,
24 get_tokens_up_to_path_start,
25)
27__all__ = [
28 # submodules
29 "all_tokenizers",
30 "maze_tokenizer",
31 "save_hashes",
32 # modular maze tokenization components
33 "TokenizationMode",
34 "_TokenizerElement",
35 "MazeTokenizerModular",
36 "PromptSequencers",
37 "CoordTokenizers",
38 "AdjListTokenizers",
39 "EdgeGroupings",
40 "EdgePermuters",
41 "EdgeSubsets",
42 "TargetTokenizers",
43 "StepSizes",
44 "StepTokenizers",
45 "PathTokenizers",
46 # helpers
47 "coord_str_to_tuple",
48 "get_tokens_up_to_path_start",
49 # old tokenizer
50 "MazeTokenizer",
51]