grscheller.fp.iterables
Library of iterator related functions.
- iterables are not necessarily iterators
- at all times iterator protocol is assumed to be followed, that is
- for all iterators
foo
we assumeiter(foo) is foo
- all iterators are assumed to be iterable
- for all iterators
1# Copyright 2023-2024 Geoffrey R. Scheller 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15""" 16#### Library of iterator related functions. 17 18* iterables are not necessarily iterators 19* at all times iterator protocol is assumed to be followed, that is 20 * for all iterators `foo` we assume `iter(foo) is foo` 21 * all iterators are assumed to be iterable 22 23""" 24 25from __future__ import annotations 26from enum import auto, Enum 27from typing import Callable, cast, Final, Iterator, Iterable 28from typing import overload, Optional, Reversible, TypeVar 29from .nada import Nada, nada 30 31__all__ = [ 'accumulate', 'foldL', 'foldR', 32 'concat', 'merge', 'exhaust', 'FM' ] 33 34class FM(Enum): 35 CONCAT = auto() 36 MERGE = auto() 37 EXHAUST = auto() 38 39D = TypeVar('D') 40L = TypeVar('L') 41R = TypeVar('R') 42S = TypeVar('S') 43 44## Iterate over multiple Iterables 45 46def concat(*iterables: Iterable[D]) -> Iterator[D]: 47 """ 48 #### Sequentially concatenate multiple iterables together. 49 50 * pure Python version of standard library's itertools.chain 51 * iterator yields Sequentially each iterable until all are exhausted 52 * an infinite iterable will prevent subsequent iterables from yielding any values 53 * performant to chain 54 55 """ 56 iterator: Iterator[D] 57 for iterator in map(lambda x: iter(x), iterables): 58 while True: 59 try: 60 value: D = next(iterator) 61 yield value 62 except StopIteration: 63 break 64 65def exhaust(*iterables: Iterable[D]) -> Iterator[D]: 66 """ 67 #### Shuffle together multiple iterables until all are exhausted. 68 69 * iterator yields until all iterables are exhausted 70 71 """ 72 iterList = list(map(lambda x: iter(x), iterables)) 73 if (numIters := len(iterList)) > 0: 74 ii = 0 75 values = [] 76 while True: 77 try: 78 while ii < numIters: 79 values.append(next(iterList[ii])) 80 ii += 1 81 for value in values: 82 yield value 83 ii = 0 84 values.clear() 85 except StopIteration: 86 numIters -= 1 87 if numIters < 1: 88 break 89 del iterList[ii] 90 for value in values: 91 yield value 92 93def merge(*iterables: Iterable[D], yield_partials: bool=False) -> Iterator[D]: 94 """ 95 #### Shuffle together multiple iterables until one is exhausted. 96 97 * iterator yields until one of the iterables is exhausted 98 * if yield_partials is true, yield any unmatched yielded values from the other iterables 99 * this prevents data lose if any of the iterables are iterators with external references 100 101 """ 102 iterList = list(map(lambda x: iter(x), iterables)) 103 if (numIters := len(iterList)) > 0: 104 values = [] 105 while True: 106 try: 107 for ii in range(numIters): 108 values.append(next(iterList[ii])) 109 for value in values: 110 yield value 111 values.clear() 112 except StopIteration: 113 break 114 if yield_partials: 115 for value in values: 116 yield value 117 118## reducing and accumulating 119 120def accumulate(iterable: Iterable[D], f: Callable[[L, D], L], 121 initial: Optional[L]=None) -> Iterator[L]: 122 """ 123 #### Returns an iterator of accumulated values. 124 125 * pure Python version of standard library's itertools.accumulate 126 * function f does not default to addition (for typing flexibility) 127 * begins accumulation with an optional starting value 128 * itertools.accumulate has mypy issues 129 130 """ 131 it = iter(iterable) 132 try: 133 it0 = next(it) 134 except StopIteration: 135 if initial is None: 136 return 137 else: 138 yield initial 139 else: 140 if initial is not None: 141 yield initial 142 acc = f(initial, it0) 143 for ii in it: 144 yield acc 145 acc = f(acc, ii) 146 yield acc 147 else: 148 acc = cast(L, it0) # in this case L = D 149 for ii in it: 150 yield acc 151 acc = f(acc, ii) 152 yield acc 153 154@overload 155def foldL(iterable: Iterable[D], f: Callable[[L, D], L], initial: Optional[L], default: S) -> L|S: 156 ... 157@overload 158def foldL(iterable: Iterable[D], f: Callable[[D, D], D]) -> D|Nada: 159 ... 160@overload 161def foldL(iterable: Iterable[D], f: Callable[[L, D], L], initial: L) -> L: 162 ... 163@overload 164def foldL(iterable: Iterable[D], f: Callable[[L, D], L], initial: Nada) -> Nada: 165 ... 166 167def foldL(iterable: Iterable[D], f: Callable[[L, D], L], 168 initial: Optional[L]=None, default: S|Nada=nada) -> L|S|Nada: 169 """ 170 #### Folds iterable left with optional initial value. 171 172 * note that ~S can be the same type as ~L 173 * note that when an initial value is not given then ~L = ~D 174 * if iterable empty & no initial value given, return default 175 * traditional FP type order given for function f 176 * raises TypeError if the "iterable" is not iterable 177 * never returns if iterable generates an infinite iterator 178 179 """ 180 acc: L 181 if hasattr(iterable, '__iter__'): 182 it = iter(iterable) 183 else: 184 msg = '"Iterable" is not iterable.' 185 raise TypeError(msg) 186 187 if initial is None: 188 try: 189 acc = cast(L, next(it)) # in this case L = D 190 except StopIteration: 191 return cast(S, default) # if default = nothing, then S is Nothing 192 else: 193 acc = initial 194 195 for v in it: 196 acc = f(acc, v) 197 198 return acc 199 200@overload 201def foldR(iterable: Reversible[D], f: Callable[[D, R], R], initial: Optional[R], default: S) -> R|S: 202 ... 203@overload 204def foldR(iterable: Reversible[D], f: Callable[[D, D], D]) -> D|Nothing: 205 ... 206@overload 207def foldR(iterable: Reversible[D], f: Callable[[D, R], R], initial: R) -> R: 208 ... 209@overload 210def foldR(iterable: Reversible[D], f: Callable[[D, R], R], initial: Nada) -> R|Nada: 211 ... 212 213def foldR(iterable: Reversible[D], f: Callable[[D, R], R], 214 initial: Optional[R]=None, default: S|Nada=nada) -> R|S|Nada: 215 """ 216 #### Folds reversible iterable right with an optional initial value. 217 218 * note that ~S can be the same type as ~R 219 * note that when an initial value not given then ~R = ~D 220 * if iterable empty & no initial value given, return default 221 * traditional FP type order given for function f 222 * raises TypeError if iterable is not reversible 223 224 """ 225 acc: R 226 if hasattr(iterable, '__reversed__') or hasattr(iterable, '__len__') and hasattr(iterable, '__getitem__'): 227 it = reversed(iterable) 228 else: 229 msg = 'Iterable is not reversible.' 230 raise TypeError(msg) 231 232 if initial is None: 233 try: 234 acc = cast(R, next(it)) # in this case R = D 235 except StopIteration: 236 return cast(S, default) # if default = nothing, then S is Nothing 237 else: 238 acc = initial 239 240 for v in it: 241 acc = f(v, acc) 242 243 return acc 244 245# @overload 246# def foldLsc(iterable: Iterable[D|S], f: Callable[[D, D|S], D], sentinel: S) -> D|S: ... 247# @overload 248# def foldLsc(iterable: Iterable[D|S], f: Callable[[L, D|S], L], sentinel: S, initial: L) -> L: ... 249# @overload 250# def foldLsc(iterable: Iterable[D|S], f: Callable[[L, D|S], L], sentinel: S, initial: Optional[L]=None) -> L|Nothing: ... 251# @overload 252# def foldLsc(iterable: Iterable[D|S], f: Callable[[L, D|S], L], 253# initial: Optional[L]=None, default: S|Nothing=nothing) -> L|S: ... 254# def foldLsc(iterable: Iterable[D|S], f: Callable[[L, D|S], L], 255# sentinel: S, initial: Optional[L|S]=None) -> L|S: 256# """Folds an iterable from the left with an optional initial value. 257# 258# * if the iterable returns the sentinel value, stop the fold at that point 259# * if f returns the sentinel value, stop the fold at that point 260# * f is never passed the sentinel value 261# * note that _S can be the same type as _D 262# * if iterable empty & no initial value given, return sentinel 263# * note that when initial not given, then _L = _D 264# * traditional FP type order given for function f 265# * raises TypeError if the iterable is not iterable (for the benefit of untyped code) 266# * never returns if iterable generates an infinite iterator & f never returns the sentinel value 267# 268# """ 269# acc: L|S 270# if hasattr(iterable, '__iter__'): 271# it = iter(iterable) 272# else: 273# msg = '"Iterable" is not iterable.' 274# raise TypeError(msg) 275# 276# if initial == sentinel: 277# return sentinel 278# elif initial is None: 279# try: 280# acc = cast(L, next(it)) 281# except StopIteration: 282# return sentinel 283# else: 284# acc = initial 285# 286# for v in it: 287# if v == sentinel: 288# break 289# facc = f(cast(L, acc), v) # if not L = S 290# # then type(acc) is not S 291# if facc == sentinel: 292# break 293# else: 294# acc = facc 295# return acc
def
accumulate( iterable: Iterable[~D], f: Callable[[~L, ~D], ~L], initial: Optional[~L] = None) -> Iterator[~L]:
121def accumulate(iterable: Iterable[D], f: Callable[[L, D], L], 122 initial: Optional[L]=None) -> Iterator[L]: 123 """ 124 #### Returns an iterator of accumulated values. 125 126 * pure Python version of standard library's itertools.accumulate 127 * function f does not default to addition (for typing flexibility) 128 * begins accumulation with an optional starting value 129 * itertools.accumulate has mypy issues 130 131 """ 132 it = iter(iterable) 133 try: 134 it0 = next(it) 135 except StopIteration: 136 if initial is None: 137 return 138 else: 139 yield initial 140 else: 141 if initial is not None: 142 yield initial 143 acc = f(initial, it0) 144 for ii in it: 145 yield acc 146 acc = f(acc, ii) 147 yield acc 148 else: 149 acc = cast(L, it0) # in this case L = D 150 for ii in it: 151 yield acc 152 acc = f(acc, ii) 153 yield acc
Returns an iterator of accumulated values.
- pure Python version of standard library's itertools.accumulate
- function f does not default to addition (for typing flexibility)
- begins accumulation with an optional starting value
- itertools.accumulate has mypy issues
def
foldL( iterable: Iterable[~D], f: Callable[[~L, ~D], ~L], initial: Optional[~L] = None, default: Union[~S, grscheller.fp.nada.Nada] = nada) -> Union[~L, ~S, grscheller.fp.nada.Nada]:
168def foldL(iterable: Iterable[D], f: Callable[[L, D], L], 169 initial: Optional[L]=None, default: S|Nada=nada) -> L|S|Nada: 170 """ 171 #### Folds iterable left with optional initial value. 172 173 * note that ~S can be the same type as ~L 174 * note that when an initial value is not given then ~L = ~D 175 * if iterable empty & no initial value given, return default 176 * traditional FP type order given for function f 177 * raises TypeError if the "iterable" is not iterable 178 * never returns if iterable generates an infinite iterator 179 180 """ 181 acc: L 182 if hasattr(iterable, '__iter__'): 183 it = iter(iterable) 184 else: 185 msg = '"Iterable" is not iterable.' 186 raise TypeError(msg) 187 188 if initial is None: 189 try: 190 acc = cast(L, next(it)) # in this case L = D 191 except StopIteration: 192 return cast(S, default) # if default = nothing, then S is Nothing 193 else: 194 acc = initial 195 196 for v in it: 197 acc = f(acc, v) 198 199 return acc
Folds iterable left with optional initial value.
- note that ~S can be the same type as ~L
- note that when an initial value is not given then ~L = ~D
- if iterable empty & no initial value given, return default
- traditional FP type order given for function f
- raises TypeError if the "iterable" is not iterable
- never returns if iterable generates an infinite iterator
def
foldR( iterable: Reversible[~D], f: Callable[[~D, ~R], ~R], initial: Optional[~R] = None, default: Union[~S, grscheller.fp.nada.Nada] = nada) -> Union[~R, ~S, grscheller.fp.nada.Nada]:
214def foldR(iterable: Reversible[D], f: Callable[[D, R], R], 215 initial: Optional[R]=None, default: S|Nada=nada) -> R|S|Nada: 216 """ 217 #### Folds reversible iterable right with an optional initial value. 218 219 * note that ~S can be the same type as ~R 220 * note that when an initial value not given then ~R = ~D 221 * if iterable empty & no initial value given, return default 222 * traditional FP type order given for function f 223 * raises TypeError if iterable is not reversible 224 225 """ 226 acc: R 227 if hasattr(iterable, '__reversed__') or hasattr(iterable, '__len__') and hasattr(iterable, '__getitem__'): 228 it = reversed(iterable) 229 else: 230 msg = 'Iterable is not reversible.' 231 raise TypeError(msg) 232 233 if initial is None: 234 try: 235 acc = cast(R, next(it)) # in this case R = D 236 except StopIteration: 237 return cast(S, default) # if default = nothing, then S is Nothing 238 else: 239 acc = initial 240 241 for v in it: 242 acc = f(v, acc) 243 244 return acc
Folds reversible iterable right with an optional initial value.
- note that ~S can be the same type as ~R
- note that when an initial value not given then ~R = ~D
- if iterable empty & no initial value given, return default
- traditional FP type order given for function f
- raises TypeError if iterable is not reversible
def
concat(*iterables: Iterable[~D]) -> Iterator[~D]:
47def concat(*iterables: Iterable[D]) -> Iterator[D]: 48 """ 49 #### Sequentially concatenate multiple iterables together. 50 51 * pure Python version of standard library's itertools.chain 52 * iterator yields Sequentially each iterable until all are exhausted 53 * an infinite iterable will prevent subsequent iterables from yielding any values 54 * performant to chain 55 56 """ 57 iterator: Iterator[D] 58 for iterator in map(lambda x: iter(x), iterables): 59 while True: 60 try: 61 value: D = next(iterator) 62 yield value 63 except StopIteration: 64 break
Sequentially concatenate multiple iterables together.
- pure Python version of standard library's itertools.chain
- iterator yields Sequentially each iterable until all are exhausted
- an infinite iterable will prevent subsequent iterables from yielding any values
- performant to chain
def
merge(*iterables: Iterable[~D], yield_partials: bool = False) -> Iterator[~D]:
94def merge(*iterables: Iterable[D], yield_partials: bool=False) -> Iterator[D]: 95 """ 96 #### Shuffle together multiple iterables until one is exhausted. 97 98 * iterator yields until one of the iterables is exhausted 99 * if yield_partials is true, yield any unmatched yielded values from the other iterables 100 * this prevents data lose if any of the iterables are iterators with external references 101 102 """ 103 iterList = list(map(lambda x: iter(x), iterables)) 104 if (numIters := len(iterList)) > 0: 105 values = [] 106 while True: 107 try: 108 for ii in range(numIters): 109 values.append(next(iterList[ii])) 110 for value in values: 111 yield value 112 values.clear() 113 except StopIteration: 114 break 115 if yield_partials: 116 for value in values: 117 yield value
Shuffle together multiple iterables until one is exhausted.
- iterator yields until one of the iterables is exhausted
- if yield_partials is true, yield any unmatched yielded values from the other iterables
- this prevents data lose if any of the iterables are iterators with external references
def
exhaust(*iterables: Iterable[~D]) -> Iterator[~D]:
66def exhaust(*iterables: Iterable[D]) -> Iterator[D]: 67 """ 68 #### Shuffle together multiple iterables until all are exhausted. 69 70 * iterator yields until all iterables are exhausted 71 72 """ 73 iterList = list(map(lambda x: iter(x), iterables)) 74 if (numIters := len(iterList)) > 0: 75 ii = 0 76 values = [] 77 while True: 78 try: 79 while ii < numIters: 80 values.append(next(iterList[ii])) 81 ii += 1 82 for value in values: 83 yield value 84 ii = 0 85 values.clear() 86 except StopIteration: 87 numIters -= 1 88 if numIters < 1: 89 break 90 del iterList[ii] 91 for value in values: 92 yield value
Shuffle together multiple iterables until all are exhausted.
- iterator yields until all iterables are exhausted
class
FM(enum.Enum):
CONCAT =
<FM.CONCAT: 1>
MERGE =
<FM.MERGE: 2>
EXHAUST =
<FM.EXHAUST: 3>
Inherited Members
- enum.Enum
- name
- value