Coverage for maze_dataset\tokenization\__init__.py: 100%

2 statements  

« prev     ^ index     » next       coverage.py v7.6.12, created at 2025-02-23 12:49 -0700

1"""turning a maze into text 

2 

3- `MazeTokenizerModular` is the new recommended way to do this as of 1.0.0 

4- legacy `TokenizationMode` enum and `MazeTokenizer` class for supporting existing code 

5- a whole lot of helper classes and functions 

6 

7""" 

8 

9from maze_dataset.tokenization.maze_tokenizer import ( 

10 AdjListTokenizers, 

11 CoordTokenizers, 

12 EdgeGroupings, 

13 EdgePermuters, 

14 EdgeSubsets, 

15 MazeTokenizer, 

16 MazeTokenizerModular, 

17 PathTokenizers, 

18 PromptSequencers, 

19 StepSizes, 

20 StepTokenizers, 

21 TargetTokenizers, 

22 TokenizationMode, 

23 _TokenizerElement, 

24 get_tokens_up_to_path_start, 

25) 

26 

27__all__ = [ 

28 # submodules 

29 "all_tokenizers", 

30 "maze_tokenizer", 

31 "save_hashes", 

32 # modular maze tokenization components 

33 "TokenizationMode", 

34 "_TokenizerElement", 

35 "MazeTokenizerModular", 

36 "PromptSequencers", 

37 "CoordTokenizers", 

38 "AdjListTokenizers", 

39 "EdgeGroupings", 

40 "EdgePermuters", 

41 "EdgeSubsets", 

42 "TargetTokenizers", 

43 "StepSizes", 

44 "StepTokenizers", 

45 "PathTokenizers", 

46 # helpers 

47 "coord_str_to_tuple", 

48 "get_tokens_up_to_path_start", 

49 # old tokenizer 

50 "MazeTokenizer", 

51]