grscheller.fp.iterables
Library of iterator related functions.
- iterables are not necessarily iterators
- at all times iterator protocol is assumed to be followed, that is
- for all iterators
foo
we assumeiter(foo) is foo
- all iterators are assumed to be iterable
- for all iterators
1# Copyright 2023-2024 Geoffrey R. Scheller 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15""" 16#### Library of iterator related functions. 17 18* iterables are not necessarily iterators 19* at all times iterator protocol is assumed to be followed, that is 20 * for all iterators `foo` we assume `iter(foo) is foo` 21 * all iterators are assumed to be iterable 22 23""" 24 25from __future__ import annotations 26from typing import Callable, cast, Final, Iterator, Iterable 27from typing import overload, Optional, Reversible, TypeVar 28from .nada import Nada, nada 29 30__all__ = [ 'concat', 'merge', 'exhaust', 'foldL', 'foldR', 'accumulate' ] 31 32D = TypeVar('D') 33L = TypeVar('L') 34R = TypeVar('R') 35S = TypeVar('S') 36 37## Iterate over multiple Iterables 38 39def concat(*iterables: Iterable[D]) -> Iterator[D]: 40 """ 41 #### Sequentially concatenate multiple iterables together. 42 43 * pure Python version of standard library's itertools.chain 44 * iterator yields Sequentially each iterable until all are exhausted 45 * an infinite iterable will prevent subsequent iterables from yielding any values 46 * performant to chain 47 48 """ 49 iterator: Iterator[D] 50 for iterator in map(lambda x: iter(x), iterables): 51 while True: 52 try: 53 value: D = next(iterator) 54 yield value 55 except StopIteration: 56 break 57 58def exhaust(*iterables: Iterable[D]) -> Iterator[D]: 59 """ 60 #### Shuffle together multiple iterables until all are exhausted. 61 62 * iterator yields until all iterables are exhausted 63 64 """ 65 iterList = list(map(lambda x: iter(x), iterables)) 66 if (numIters := len(iterList)) > 0: 67 ii = 0 68 values = [] 69 while True: 70 try: 71 while ii < numIters: 72 values.append(next(iterList[ii])) 73 ii += 1 74 for value in values: 75 yield value 76 ii = 0 77 values.clear() 78 except StopIteration: 79 numIters -= 1 80 if numIters < 1: 81 break 82 del iterList[ii] 83 for value in values: 84 yield value 85 86def merge(*iterables: Iterable[D], yield_partials: bool=False) -> Iterator[D]: 87 """ 88 #### Shuffle together multiple iterables until one is exhausted. 89 90 * iterator yields until one of the iterables is exhausted 91 * if yield_partials is true, yield any unmatched yielded values from the other iterables 92 * this prevents data lose if any of the iterables are iterators with external references 93 94 """ 95 iterList = list(map(lambda x: iter(x), iterables)) 96 if (numIters := len(iterList)) > 0: 97 values = [] 98 while True: 99 try: 100 for ii in range(numIters): 101 values.append(next(iterList[ii])) 102 for value in values: 103 yield value 104 values.clear() 105 except StopIteration: 106 break 107 if yield_partials: 108 for value in values: 109 yield value 110 111## reducing and accumulating 112 113@overload 114def foldL(iterable: Iterable[D], f: Callable[[L, D], L], initial: Optional[L], default: S) -> L|S: 115 ... 116@overload 117def foldL(iterable: Iterable[D], f: Callable[[D, D], D]) -> D|Nada: 118 ... 119@overload 120def foldL(iterable: Iterable[D], f: Callable[[L, D], L], initial: L) -> L: 121 ... 122@overload 123def foldL(iterable: Iterable[D], f: Callable[[L, D], L], initial: Nada) -> Nada: 124 ... 125 126def foldL(iterable: Iterable[D], f: Callable[[L, D], L], 127 initial: Optional[L]=None, default: S|Nada=nada) -> L|S|Nada: 128 """ 129 #### Folds iterable left with optional initial value. 130 131 * note that ~S can be the same type as ~L 132 * note that when an initial value is not given then ~L = ~D 133 * if iterable empty & no initial value given, return default 134 * traditional FP type order given for function f 135 * raises TypeError if the "iterable" is not iterable 136 * never returns if iterable generates an infinite iterator 137 138 """ 139 acc: L 140 if hasattr(iterable, '__iter__'): 141 it = iter(iterable) 142 else: 143 msg = '"Iterable" is not iterable.' 144 raise TypeError(msg) 145 146 if initial is None: 147 try: 148 acc = cast(L, next(it)) # in this case L = D 149 except StopIteration: 150 return cast(S, default) # if default = nothing, then S is Nothing 151 else: 152 acc = initial 153 154 for v in it: 155 acc = f(acc, v) 156 157 return acc 158 159@overload 160def foldR(iterable: Reversible[D], f: Callable[[D, R], R], initial: Optional[R], default: S) -> R|S: 161 ... 162@overload 163def foldR(iterable: Reversible[D], f: Callable[[D, D], D]) -> D|Nothing: 164 ... 165@overload 166def foldR(iterable: Reversible[D], f: Callable[[D, R], R], initial: R) -> R: 167 ... 168@overload 169def foldR(iterable: Reversible[D], f: Callable[[D, R], R], initial: Nada) -> R|Nada: 170 ... 171 172def foldR(iterable: Reversible[D], f: Callable[[D, R], R], 173 initial: Optional[R]=None, default: S|Nada=nada) -> R|S|Nada: 174 """ 175 #### Folds reversible iterable right with an optional initial value. 176 177 * note that ~S can be the same type as ~R 178 * note that when an initial value not given then ~R = ~D 179 * if iterable empty & no initial value given, return default 180 * traditional FP type order given for function f 181 * raises TypeError if iterable is not reversible 182 183 """ 184 acc: R 185 if hasattr(iterable, '__reversed__') or hasattr(iterable, '__len__') and hasattr(iterable, '__getitem__'): 186 it = reversed(iterable) 187 else: 188 msg = 'Iterable is not reversible.' 189 raise TypeError(msg) 190 191 if initial is None: 192 try: 193 acc = cast(R, next(it)) # in this case R = D 194 except StopIteration: 195 return cast(S, default) # if default = nothing, then S is Nothing 196 else: 197 acc = initial 198 199 for v in it: 200 acc = f(v, acc) 201 202 return acc 203 204# @overload 205# def foldLsc(iterable: Iterable[D|S], f: Callable[[D, D|S], D], sentinel: S) -> D|S: ... 206# @overload 207# def foldLsc(iterable: Iterable[D|S], f: Callable[[L, D|S], L], sentinel: S, initial: L) -> L: ... 208# @overload 209# def foldLsc(iterable: Iterable[D|S], f: Callable[[L, D|S], L], sentinel: S, initial: Optional[L]=None) -> L|Nothing: ... 210# @overload 211# def foldLsc(iterable: Iterable[D|S], f: Callable[[L, D|S], L], 212# initial: Optional[L]=None, default: S|Nothing=nothing) -> L|S: ... 213# def foldLsc(iterable: Iterable[D|S], f: Callable[[L, D|S], L], 214# sentinel: S, initial: Optional[L|S]=None) -> L|S: 215# """Folds an iterable from the left with an optional initial value. 216# 217# * if the iterable returns the sentinel value, stop the fold at that point 218# * if f returns the sentinel value, stop the fold at that point 219# * f is never passed the sentinel value 220# * note that _S can be the same type as _D 221# * if iterable empty & no initial value given, return sentinel 222# * note that when initial not given, then _L = _D 223# * traditional FP type order given for function f 224# * raises TypeError if the iterable is not iterable (for the benefit of untyped code) 225# * never returns if iterable generates an infinite iterator & f never returns the sentinel value 226# 227# """ 228# acc: L|S 229# if hasattr(iterable, '__iter__'): 230# it = iter(iterable) 231# else: 232# msg = '"Iterable" is not iterable.' 233# raise TypeError(msg) 234# 235# if initial == sentinel: 236# return sentinel 237# elif initial is None: 238# try: 239# acc = cast(L, next(it)) 240# except StopIteration: 241# return sentinel 242# else: 243# acc = initial 244# 245# for v in it: 246# if v == sentinel: 247# break 248# facc = f(cast(L, acc), v) # if not L = S 249# # then type(acc) is not S 250# if facc == sentinel: 251# break 252# else: 253# acc = facc 254# return acc 255 256def accumulate(iterable: Iterable[D], f: Callable[[L, D], L], 257 initial: Optional[L]=None) -> Iterator[L]: 258 """ 259 #### Returns an iterator of accumulated values. 260 261 * pure Python version of standard library's itertools.accumulate 262 * function f does not default to addition (for typing flexibility) 263 * begins accumulation with an optional starting value 264 * itertools.accumulate has mypy issues 265 266 """ 267 it = iter(iterable) 268 try: 269 it0 = next(it) 270 except StopIteration: 271 if initial is None: 272 return 273 else: 274 yield initial 275 else: 276 if initial is not None: 277 yield initial 278 acc = f(initial, it0) 279 for ii in it: 280 yield acc 281 acc = f(acc, ii) 282 yield acc 283 else: 284 acc = cast(L, it0) # in this case L = D 285 for ii in it: 286 yield acc 287 acc = f(acc, ii) 288 yield acc
def
concat(*iterables: Iterable[~D]) -> Iterator[~D]:
40def concat(*iterables: Iterable[D]) -> Iterator[D]: 41 """ 42 #### Sequentially concatenate multiple iterables together. 43 44 * pure Python version of standard library's itertools.chain 45 * iterator yields Sequentially each iterable until all are exhausted 46 * an infinite iterable will prevent subsequent iterables from yielding any values 47 * performant to chain 48 49 """ 50 iterator: Iterator[D] 51 for iterator in map(lambda x: iter(x), iterables): 52 while True: 53 try: 54 value: D = next(iterator) 55 yield value 56 except StopIteration: 57 break
Sequentially concatenate multiple iterables together.
- pure Python version of standard library's itertools.chain
- iterator yields Sequentially each iterable until all are exhausted
- an infinite iterable will prevent subsequent iterables from yielding any values
- performant to chain
def
merge(*iterables: Iterable[~D], yield_partials: bool = False) -> Iterator[~D]:
87def merge(*iterables: Iterable[D], yield_partials: bool=False) -> Iterator[D]: 88 """ 89 #### Shuffle together multiple iterables until one is exhausted. 90 91 * iterator yields until one of the iterables is exhausted 92 * if yield_partials is true, yield any unmatched yielded values from the other iterables 93 * this prevents data lose if any of the iterables are iterators with external references 94 95 """ 96 iterList = list(map(lambda x: iter(x), iterables)) 97 if (numIters := len(iterList)) > 0: 98 values = [] 99 while True: 100 try: 101 for ii in range(numIters): 102 values.append(next(iterList[ii])) 103 for value in values: 104 yield value 105 values.clear() 106 except StopIteration: 107 break 108 if yield_partials: 109 for value in values: 110 yield value
Shuffle together multiple iterables until one is exhausted.
- iterator yields until one of the iterables is exhausted
- if yield_partials is true, yield any unmatched yielded values from the other iterables
- this prevents data lose if any of the iterables are iterators with external references
def
exhaust(*iterables: Iterable[~D]) -> Iterator[~D]:
59def exhaust(*iterables: Iterable[D]) -> Iterator[D]: 60 """ 61 #### Shuffle together multiple iterables until all are exhausted. 62 63 * iterator yields until all iterables are exhausted 64 65 """ 66 iterList = list(map(lambda x: iter(x), iterables)) 67 if (numIters := len(iterList)) > 0: 68 ii = 0 69 values = [] 70 while True: 71 try: 72 while ii < numIters: 73 values.append(next(iterList[ii])) 74 ii += 1 75 for value in values: 76 yield value 77 ii = 0 78 values.clear() 79 except StopIteration: 80 numIters -= 1 81 if numIters < 1: 82 break 83 del iterList[ii] 84 for value in values: 85 yield value
Shuffle together multiple iterables until all are exhausted.
- iterator yields until all iterables are exhausted
def
foldL( iterable: Iterable[~D], f: Callable[[~L, ~D], ~L], initial: Optional[~L] = None, default: Union[~S, grscheller.fp.nada.Nada] = nada) -> Union[~L, ~S, grscheller.fp.nada.Nada]:
127def foldL(iterable: Iterable[D], f: Callable[[L, D], L], 128 initial: Optional[L]=None, default: S|Nada=nada) -> L|S|Nada: 129 """ 130 #### Folds iterable left with optional initial value. 131 132 * note that ~S can be the same type as ~L 133 * note that when an initial value is not given then ~L = ~D 134 * if iterable empty & no initial value given, return default 135 * traditional FP type order given for function f 136 * raises TypeError if the "iterable" is not iterable 137 * never returns if iterable generates an infinite iterator 138 139 """ 140 acc: L 141 if hasattr(iterable, '__iter__'): 142 it = iter(iterable) 143 else: 144 msg = '"Iterable" is not iterable.' 145 raise TypeError(msg) 146 147 if initial is None: 148 try: 149 acc = cast(L, next(it)) # in this case L = D 150 except StopIteration: 151 return cast(S, default) # if default = nothing, then S is Nothing 152 else: 153 acc = initial 154 155 for v in it: 156 acc = f(acc, v) 157 158 return acc
Folds iterable left with optional initial value.
- note that ~S can be the same type as ~L
- note that when an initial value is not given then ~L = ~D
- if iterable empty & no initial value given, return default
- traditional FP type order given for function f
- raises TypeError if the "iterable" is not iterable
- never returns if iterable generates an infinite iterator
def
foldR( iterable: Reversible[~D], f: Callable[[~D, ~R], ~R], initial: Optional[~R] = None, default: Union[~S, grscheller.fp.nada.Nada] = nada) -> Union[~R, ~S, grscheller.fp.nada.Nada]:
173def foldR(iterable: Reversible[D], f: Callable[[D, R], R], 174 initial: Optional[R]=None, default: S|Nada=nada) -> R|S|Nada: 175 """ 176 #### Folds reversible iterable right with an optional initial value. 177 178 * note that ~S can be the same type as ~R 179 * note that when an initial value not given then ~R = ~D 180 * if iterable empty & no initial value given, return default 181 * traditional FP type order given for function f 182 * raises TypeError if iterable is not reversible 183 184 """ 185 acc: R 186 if hasattr(iterable, '__reversed__') or hasattr(iterable, '__len__') and hasattr(iterable, '__getitem__'): 187 it = reversed(iterable) 188 else: 189 msg = 'Iterable is not reversible.' 190 raise TypeError(msg) 191 192 if initial is None: 193 try: 194 acc = cast(R, next(it)) # in this case R = D 195 except StopIteration: 196 return cast(S, default) # if default = nothing, then S is Nothing 197 else: 198 acc = initial 199 200 for v in it: 201 acc = f(v, acc) 202 203 return acc
Folds reversible iterable right with an optional initial value.
- note that ~S can be the same type as ~R
- note that when an initial value not given then ~R = ~D
- if iterable empty & no initial value given, return default
- traditional FP type order given for function f
- raises TypeError if iterable is not reversible
def
accumulate( iterable: Iterable[~D], f: Callable[[~L, ~D], ~L], initial: Optional[~L] = None) -> Iterator[~L]:
257def accumulate(iterable: Iterable[D], f: Callable[[L, D], L], 258 initial: Optional[L]=None) -> Iterator[L]: 259 """ 260 #### Returns an iterator of accumulated values. 261 262 * pure Python version of standard library's itertools.accumulate 263 * function f does not default to addition (for typing flexibility) 264 * begins accumulation with an optional starting value 265 * itertools.accumulate has mypy issues 266 267 """ 268 it = iter(iterable) 269 try: 270 it0 = next(it) 271 except StopIteration: 272 if initial is None: 273 return 274 else: 275 yield initial 276 else: 277 if initial is not None: 278 yield initial 279 acc = f(initial, it0) 280 for ii in it: 281 yield acc 282 acc = f(acc, ii) 283 yield acc 284 else: 285 acc = cast(L, it0) # in this case L = D 286 for ii in it: 287 yield acc 288 acc = f(acc, ii) 289 yield acc
Returns an iterator of accumulated values.
- pure Python version of standard library's itertools.accumulate
- function f does not default to addition (for typing flexibility)
- begins accumulation with an optional starting value
- itertools.accumulate has mypy issues