grscheller.fp.iterables
Library of iterator related functions.
- iterables are not necessarily iterators
- at all times iterator protocol is assumed to be followed, that is
- all iterators are assumed to be iterable
- for all iterators
foo
we assumeiter(foo) is foo
1# Copyright 2023-2024 Geoffrey R. Scheller 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15""" 16### Library of iterator related functions. 17 18* iterables are not necessarily iterators 19* at all times iterator protocol is assumed to be followed, that is 20 * all iterators are assumed to be iterable 21 * for all iterators `foo` we assume `iter(foo) is foo` 22 23""" 24from __future__ import annotations 25from enum import auto, Enum 26from typing import Callable, cast, Final, Iterator, Iterable 27from typing import overload, Optional, Reversible, TypeVar 28from .woException import MB 29 30__all__ = [ 'drop', 'dropWhile', 'take', 'takeWhile', 31 'concat', 'merge', 'exhaust', 'FM', 32 'accumulate', 'foldL', 'foldR', 'foldLsc', 'foldRsc' ] 33 34class FM(Enum): 35 CONCAT = auto() 36 MERGE = auto() 37 EXHAUST = auto() 38 39D = TypeVar('D') # D for Data 40L = TypeVar('L') # L for Left 41R = TypeVar('R') # R for Right 42S = TypeVar('S') # S for State 43 44## Iterate over multiple Iterables 45 46def concat(*iterables: Iterable[D]) -> Iterator[D]: 47 """Sequentially concatenate multiple iterables together. 48 49 * pure Python version of standard library's itertools.chain 50 * iterator sequentially yields each iterable until all are exhausted 51 * an infinite iterable will prevent subsequent iterables from yielding any values 52 * performant to chain 53 54 """ 55 for iterator in map(lambda x: iter(x), iterables): 56 while True: 57 try: 58 value = next(iterator) 59 yield value 60 except StopIteration: 61 break 62 63def exhaust(*iterables: Iterable[D]) -> Iterator[D]: 64 """Shuffle together multiple iterables until all are exhausted. 65 66 * iterator yields until all iterables are exhausted 67 68 """ 69 iterList = list(map(lambda x: iter(x), iterables)) 70 if (numIters := len(iterList)) > 0: 71 ii = 0 72 values = [] 73 while True: 74 try: 75 while ii < numIters: 76 values.append(next(iterList[ii])) 77 ii += 1 78 for value in values: 79 yield value 80 ii = 0 81 values.clear() 82 except StopIteration: 83 numIters -= 1 84 if numIters < 1: 85 break 86 del iterList[ii] 87 for value in values: 88 yield value 89 90def merge(*iterables: Iterable[D], yield_partials: bool=False) -> Iterator[D]: 91 """Shuffle together multiple iterables until one is exhausted. 92 93 * iterator yields until one of the iterables is exhausted 94 * if yield_partials is true, yield any unmatched yielded values from other iterables 95 * prevents data lose if any of the iterables are iterators with external references 96 97 """ 98 iterList = list(map(lambda x: iter(x), iterables)) 99 values = [] 100 if (numIters := len(iterList)) > 0: 101 while True: 102 try: 103 for ii in range(numIters): 104 values.append(next(iterList[ii])) 105 for value in values: 106 yield value 107 values.clear() 108 except StopIteration: 109 break 110 if yield_partials: 111 for value in values: 112 yield value 113 114## dropping and taking 115 116def drop(iterable: Iterable[D], n: int) -> Iterator[D]: 117 """Drop the next `n` values from `iterable`.""" 118 it = iter(iterable) 119 for _ in range(n): 120 try: 121 value = next(it) 122 except StopIteration: 123 break 124 return it 125 126def dropWhile(iterable: Iterable[D], pred: Callable[[D], bool]) -> Iterator[D]: 127 """Drop initial values from `iterable` while predicate is true.""" 128 it = iter(iterable) 129 try: 130 value = next(it) 131 except: 132 return it 133 134 while True: 135 try: 136 if not pred(value): 137 break 138 value = next(it) 139 except StopIteration: 140 break 141 return concat((value,), it) 142 143def take(iterable: Iterable[D], n: int) -> Iterator[D]: 144 """Take up to `n` values from `iterable`.""" 145 it = iter(iterable) 146 for _ in range(n): 147 try: 148 value = next(it) 149 yield value 150 except StopIteration: 151 break 152 153def takeWhile(iterable: Iterable[D], pred: Callable[[D], bool]) -> Iterator[D]: 154 """Yield values from `iterable` while predicate is true. 155 156 * potential value loss if iterable is iterator with external references 157 158 """ 159 it = iter(iterable) 160 while True: 161 try: 162 value = next(it) 163 if pred(value): 164 yield value 165 else: 166 break 167 except StopIteration: 168 break 169 170## reducing and accumulating 171 172def accumulate(iterable: Iterable[D], f: Callable[[L, D], L], 173 initial: Optional[L]=None) -> Iterator[L]: 174 """ 175 Returns an iterator of accumulated values. 176 177 * pure Python version of standard library's itertools.accumulate 178 * function f does not default to addition (for typing flexibility) 179 * begins accumulation with an optional starting value 180 * itertools.accumulate had mypy issues 181 182 """ 183 it = iter(iterable) 184 try: 185 it0 = next(it) 186 except StopIteration: 187 if initial is None: 188 return 189 else: 190 yield initial 191 else: 192 if initial is not None: 193 yield initial 194 acc = f(initial, it0) 195 for ii in it: 196 yield acc 197 acc = f(acc, ii) 198 yield acc 199 else: 200 acc = cast(L, it0) # in this case L = D 201 for ii in it: 202 yield acc 203 acc = f(acc, ii) 204 yield acc 205 206def foldL(iterable: Iterable[D], 207 f: Callable[[L, D], L], 208 initial: Optional[L]=None) -> MB[L]: 209 """ 210 Folds an iterable left with optional initial value. 211 212 * traditional FP type order given for function f 213 * when an initial value is not given then ~L = ~D 214 * if iterable empty & no initial value given, return empty MB() 215 * never returns if iterable generates an infinite iterator 216 217 """ 218 acc: L 219 it = iter(iterable) 220 221 if initial is None: 222 try: 223 acc = cast(L, next(it)) # in this case L = D 224 except StopIteration: 225 return MB() 226 else: 227 acc = initial 228 229 for v in it: 230 acc = f(acc, v) 231 232 return MB(acc) 233 234def foldR(iterable: Reversible[D], 235 f: Callable[[D, R], R], 236 initial: Optional[R]=None) -> MB[R]: 237 """ 238 Folds a reversible iterable right with an optional initial value. 239 240 * iterable needs to be reversible 241 * traditional FP type order given for function f 242 * when initial value is not given then ~R = ~D 243 * if iterable empty & no initial value given, return return empty MB() 244 245 """ 246 acc: R 247 it = reversed(iterable) 248 249 if initial is None: 250 try: 251 acc = cast(R, next(it)) # in this case R = D 252 except StopIteration: 253 return MB() 254 else: 255 acc = initial 256 257 for v in it: 258 acc = f(v, acc) 259 260 return MB(acc) 261 262def foldLsc(iterable: Iterable[D], 263 f: Callable[[L, D], L], 264 initial: Optional[L]=None, 265 stopfold: Callable[[D, S], MB[S]]=lambda d, s: MB(s), 266 istate: Optional[S]=None) -> MB[L]: 267 """ 268 Short circuit version of foldL. 269 270 * Callable `stopfold` purpose is to prematurely stop fold before end 271 * useful for infinite iterables 272 273 """ 274 state = cast(MB[S], MB(istate)) 275 276 it = iter(iterable) 277 278 if initial is None: 279 try: 280 acc = cast(L, next(it)) # in this case L = D 281 except StopIteration: 282 return MB() 283 else: 284 acc = initial 285 286 for d in it: 287 if (state := stopfold(d, state.get())): 288 acc = f(acc, d) 289 else: 290 break 291 292 return MB(acc) 293 294def foldRsc(iterable: Iterable[D], 295 f: Callable[[D, R], R], 296 initial: Optional[R]=None, 297 startfold: Callable[[D, S], MB[S]]=lambda d, s: MB(s), 298 istate: Optional[S]=None) -> MB[R]: 299 """ 300 Short circuit version of foldR. 301 302 * Callable `startfold` purpose is to start fold before end 303 * does NOT start fold at end and prematurely stop 304 * useful for infinite and non-reversible iterables 305 306 """ 307 state = cast(MB[S], MB(istate)) 308 309 it = iter(iterable) 310 311 acc: R 312 313 ds: list[D] = [] 314 for d in it: 315 if (state := startfold(d, state.get())): 316 ds.append(d) 317 else: 318 break 319 320 if initial is None: 321 if len(ds) == 0: 322 return MB() 323 else: 324 acc = cast(R, ds.pop()) # in this case R = D 325 else: 326 acc = initial 327 328 while ds: 329 acc = f(ds.pop(), acc) 330 331 return MB(acc)
def
drop(iterable: Iterable[~D], n: int) -> Iterator[~D]:
117def drop(iterable: Iterable[D], n: int) -> Iterator[D]: 118 """Drop the next `n` values from `iterable`.""" 119 it = iter(iterable) 120 for _ in range(n): 121 try: 122 value = next(it) 123 except StopIteration: 124 break 125 return it
Drop the next n
values from iterable
.
def
dropWhile(iterable: Iterable[~D], pred: Callable[[~D], bool]) -> Iterator[~D]:
127def dropWhile(iterable: Iterable[D], pred: Callable[[D], bool]) -> Iterator[D]: 128 """Drop initial values from `iterable` while predicate is true.""" 129 it = iter(iterable) 130 try: 131 value = next(it) 132 except: 133 return it 134 135 while True: 136 try: 137 if not pred(value): 138 break 139 value = next(it) 140 except StopIteration: 141 break 142 return concat((value,), it)
Drop initial values from iterable
while predicate is true.
def
take(iterable: Iterable[~D], n: int) -> Iterator[~D]:
144def take(iterable: Iterable[D], n: int) -> Iterator[D]: 145 """Take up to `n` values from `iterable`.""" 146 it = iter(iterable) 147 for _ in range(n): 148 try: 149 value = next(it) 150 yield value 151 except StopIteration: 152 break
Take up to n
values from iterable
.
def
takeWhile(iterable: Iterable[~D], pred: Callable[[~D], bool]) -> Iterator[~D]:
154def takeWhile(iterable: Iterable[D], pred: Callable[[D], bool]) -> Iterator[D]: 155 """Yield values from `iterable` while predicate is true. 156 157 * potential value loss if iterable is iterator with external references 158 159 """ 160 it = iter(iterable) 161 while True: 162 try: 163 value = next(it) 164 if pred(value): 165 yield value 166 else: 167 break 168 except StopIteration: 169 break
Yield values from iterable
while predicate is true.
- potential value loss if iterable is iterator with external references
def
concat(*iterables: Iterable[~D]) -> Iterator[~D]:
47def concat(*iterables: Iterable[D]) -> Iterator[D]: 48 """Sequentially concatenate multiple iterables together. 49 50 * pure Python version of standard library's itertools.chain 51 * iterator sequentially yields each iterable until all are exhausted 52 * an infinite iterable will prevent subsequent iterables from yielding any values 53 * performant to chain 54 55 """ 56 for iterator in map(lambda x: iter(x), iterables): 57 while True: 58 try: 59 value = next(iterator) 60 yield value 61 except StopIteration: 62 break
Sequentially concatenate multiple iterables together.
- pure Python version of standard library's itertools.chain
- iterator sequentially yields each iterable until all are exhausted
- an infinite iterable will prevent subsequent iterables from yielding any values
- performant to chain
def
merge(*iterables: Iterable[~D], yield_partials: bool = False) -> Iterator[~D]:
91def merge(*iterables: Iterable[D], yield_partials: bool=False) -> Iterator[D]: 92 """Shuffle together multiple iterables until one is exhausted. 93 94 * iterator yields until one of the iterables is exhausted 95 * if yield_partials is true, yield any unmatched yielded values from other iterables 96 * prevents data lose if any of the iterables are iterators with external references 97 98 """ 99 iterList = list(map(lambda x: iter(x), iterables)) 100 values = [] 101 if (numIters := len(iterList)) > 0: 102 while True: 103 try: 104 for ii in range(numIters): 105 values.append(next(iterList[ii])) 106 for value in values: 107 yield value 108 values.clear() 109 except StopIteration: 110 break 111 if yield_partials: 112 for value in values: 113 yield value
Shuffle together multiple iterables until one is exhausted.
- iterator yields until one of the iterables is exhausted
- if yield_partials is true, yield any unmatched yielded values from other iterables
- prevents data lose if any of the iterables are iterators with external references
def
exhaust(*iterables: Iterable[~D]) -> Iterator[~D]:
64def exhaust(*iterables: Iterable[D]) -> Iterator[D]: 65 """Shuffle together multiple iterables until all are exhausted. 66 67 * iterator yields until all iterables are exhausted 68 69 """ 70 iterList = list(map(lambda x: iter(x), iterables)) 71 if (numIters := len(iterList)) > 0: 72 ii = 0 73 values = [] 74 while True: 75 try: 76 while ii < numIters: 77 values.append(next(iterList[ii])) 78 ii += 1 79 for value in values: 80 yield value 81 ii = 0 82 values.clear() 83 except StopIteration: 84 numIters -= 1 85 if numIters < 1: 86 break 87 del iterList[ii] 88 for value in values: 89 yield value
Shuffle together multiple iterables until all are exhausted.
- iterator yields until all iterables are exhausted
class
FM(enum.Enum):
CONCAT =
<FM.CONCAT: 1>
MERGE =
<FM.MERGE: 2>
EXHAUST =
<FM.EXHAUST: 3>
Inherited Members
- enum.Enum
- name
- value
def
accumulate( iterable: Iterable[~D], f: Callable[[~L, ~D], ~L], initial: Optional[~L] = None) -> Iterator[~L]:
173def accumulate(iterable: Iterable[D], f: Callable[[L, D], L], 174 initial: Optional[L]=None) -> Iterator[L]: 175 """ 176 Returns an iterator of accumulated values. 177 178 * pure Python version of standard library's itertools.accumulate 179 * function f does not default to addition (for typing flexibility) 180 * begins accumulation with an optional starting value 181 * itertools.accumulate had mypy issues 182 183 """ 184 it = iter(iterable) 185 try: 186 it0 = next(it) 187 except StopIteration: 188 if initial is None: 189 return 190 else: 191 yield initial 192 else: 193 if initial is not None: 194 yield initial 195 acc = f(initial, it0) 196 for ii in it: 197 yield acc 198 acc = f(acc, ii) 199 yield acc 200 else: 201 acc = cast(L, it0) # in this case L = D 202 for ii in it: 203 yield acc 204 acc = f(acc, ii) 205 yield acc
Returns an iterator of accumulated values.
- pure Python version of standard library's itertools.accumulate
- function f does not default to addition (for typing flexibility)
- begins accumulation with an optional starting value
- itertools.accumulate had mypy issues
def
foldL( iterable: Iterable[~D], f: Callable[[~L, ~D], ~L], initial: Optional[~L] = None) -> grscheller.fp.woException.MB[~L]:
207def foldL(iterable: Iterable[D], 208 f: Callable[[L, D], L], 209 initial: Optional[L]=None) -> MB[L]: 210 """ 211 Folds an iterable left with optional initial value. 212 213 * traditional FP type order given for function f 214 * when an initial value is not given then ~L = ~D 215 * if iterable empty & no initial value given, return empty MB() 216 * never returns if iterable generates an infinite iterator 217 218 """ 219 acc: L 220 it = iter(iterable) 221 222 if initial is None: 223 try: 224 acc = cast(L, next(it)) # in this case L = D 225 except StopIteration: 226 return MB() 227 else: 228 acc = initial 229 230 for v in it: 231 acc = f(acc, v) 232 233 return MB(acc)
Folds an iterable left with optional initial value.
- traditional FP type order given for function f
- when an initial value is not given then ~L = ~D
- if iterable empty & no initial value given, return empty MB()
- never returns if iterable generates an infinite iterator
def
foldR( iterable: Reversible[~D], f: Callable[[~D, ~R], ~R], initial: Optional[~R] = None) -> grscheller.fp.woException.MB[~R]:
235def foldR(iterable: Reversible[D], 236 f: Callable[[D, R], R], 237 initial: Optional[R]=None) -> MB[R]: 238 """ 239 Folds a reversible iterable right with an optional initial value. 240 241 * iterable needs to be reversible 242 * traditional FP type order given for function f 243 * when initial value is not given then ~R = ~D 244 * if iterable empty & no initial value given, return return empty MB() 245 246 """ 247 acc: R 248 it = reversed(iterable) 249 250 if initial is None: 251 try: 252 acc = cast(R, next(it)) # in this case R = D 253 except StopIteration: 254 return MB() 255 else: 256 acc = initial 257 258 for v in it: 259 acc = f(v, acc) 260 261 return MB(acc)
Folds a reversible iterable right with an optional initial value.
- iterable needs to be reversible
- traditional FP type order given for function f
- when initial value is not given then ~R = ~D
- if iterable empty & no initial value given, return return empty MB()
def
foldLsc( iterable: Iterable[~D], f: Callable[[~L, ~D], ~L], initial: Optional[~L] = None, stopfold: Callable[[~D, ~S], grscheller.fp.woException.MB[~S]] = <function <lambda>>, istate: Optional[~S] = None) -> grscheller.fp.woException.MB[~L]:
263def foldLsc(iterable: Iterable[D], 264 f: Callable[[L, D], L], 265 initial: Optional[L]=None, 266 stopfold: Callable[[D, S], MB[S]]=lambda d, s: MB(s), 267 istate: Optional[S]=None) -> MB[L]: 268 """ 269 Short circuit version of foldL. 270 271 * Callable `stopfold` purpose is to prematurely stop fold before end 272 * useful for infinite iterables 273 274 """ 275 state = cast(MB[S], MB(istate)) 276 277 it = iter(iterable) 278 279 if initial is None: 280 try: 281 acc = cast(L, next(it)) # in this case L = D 282 except StopIteration: 283 return MB() 284 else: 285 acc = initial 286 287 for d in it: 288 if (state := stopfold(d, state.get())): 289 acc = f(acc, d) 290 else: 291 break 292 293 return MB(acc)
Short circuit version of foldL.
- Callable
stopfold
purpose is to prematurely stop fold before end- useful for infinite iterables
def
foldRsc( iterable: Iterable[~D], f: Callable[[~D, ~R], ~R], initial: Optional[~R] = None, startfold: Callable[[~D, ~S], grscheller.fp.woException.MB[~S]] = <function <lambda>>, istate: Optional[~S] = None) -> grscheller.fp.woException.MB[~R]:
295def foldRsc(iterable: Iterable[D], 296 f: Callable[[D, R], R], 297 initial: Optional[R]=None, 298 startfold: Callable[[D, S], MB[S]]=lambda d, s: MB(s), 299 istate: Optional[S]=None) -> MB[R]: 300 """ 301 Short circuit version of foldR. 302 303 * Callable `startfold` purpose is to start fold before end 304 * does NOT start fold at end and prematurely stop 305 * useful for infinite and non-reversible iterables 306 307 """ 308 state = cast(MB[S], MB(istate)) 309 310 it = iter(iterable) 311 312 acc: R 313 314 ds: list[D] = [] 315 for d in it: 316 if (state := startfold(d, state.get())): 317 ds.append(d) 318 else: 319 break 320 321 if initial is None: 322 if len(ds) == 0: 323 return MB() 324 else: 325 acc = cast(R, ds.pop()) # in this case R = D 326 else: 327 acc = initial 328 329 while ds: 330 acc = f(ds.pop(), acc) 331 332 return MB(acc)
Short circuit version of foldR.
- Callable
startfold
purpose is to start fold before end- does NOT start fold at end and prematurely stop
- useful for infinite and non-reversible iterables