grscheller.fp.iterables
Library of iterator related functions.
- iterables are not necessarily iterators
- at all times iterator protocol is assumed to be followed, that is
- all iterators are assumed to be iterable
- for all iterators
foo
we assumeiter(foo) is foo
1# Copyright 2023-2024 Geoffrey R. Scheller 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15""" 16### Library of iterator related functions. 17 18* iterables are not necessarily iterators 19* at all times iterator protocol is assumed to be followed, that is 20 * all iterators are assumed to be iterable 21 * for all iterators `foo` we assume `iter(foo) is foo` 22 23""" 24from __future__ import annotations 25from enum import auto, Enum 26from typing import Callable, cast, Iterator, Iterable, Optional, Reversible 27from .err_handling import MB 28 29__all__ = [ 'drop', 'dropWhile', 'take', 'takeWhile', 30 'concat', 'merge', 'exhaust', 'FM', 31 'accumulate', 'foldL', 'foldR', 'foldLsc', 'foldRsc' ] 32 33class FM(Enum): 34 CONCAT = auto() 35 MERGE = auto() 36 EXHAUST = auto() 37 38## Iterate over multiple Iterables 39 40def concat[D](*iterables: Iterable[D]) -> Iterator[D]: 41 """Sequentially concatenate multiple iterables together. 42 43 * pure Python version of standard library's itertools.chain 44 * iterator sequentially yields each iterable until all are exhausted 45 * an infinite iterable will prevent subsequent iterables from yielding any values 46 * performant to chain 47 48 """ 49 for iterator in map(lambda x: iter(x), iterables): 50 while True: 51 try: 52 value = next(iterator) 53 yield value 54 except StopIteration: 55 break 56 57def exhaust[D](*iterables: Iterable[D]) -> Iterator[D]: 58 """Shuffle together multiple iterables until all are exhausted. 59 60 * iterator yields until all iterables are exhausted 61 62 """ 63 iterList = list(map(lambda x: iter(x), iterables)) 64 if (numIters := len(iterList)) > 0: 65 ii = 0 66 values = [] 67 while True: 68 try: 69 while ii < numIters: 70 values.append(next(iterList[ii])) 71 ii += 1 72 for value in values: 73 yield value 74 ii = 0 75 values.clear() 76 except StopIteration: 77 numIters -= 1 78 if numIters < 1: 79 break 80 del iterList[ii] 81 for value in values: 82 yield value 83 84def merge[D](*iterables: Iterable[D], yield_partials: bool=False) -> Iterator[D]: 85 """Shuffle together multiple iterables until one is exhausted. 86 87 * iterator yields until one of the iterables is exhausted 88 * if yield_partials is true, yield any unmatched yielded values from other iterables 89 * prevents data lose if any of the iterables are iterators with external references 90 91 """ 92 iterList = list(map(lambda x: iter(x), iterables)) 93 values = [] 94 if (numIters := len(iterList)) > 0: 95 while True: 96 try: 97 for ii in range(numIters): 98 values.append(next(iterList[ii])) 99 for value in values: 100 yield value 101 values.clear() 102 except StopIteration: 103 break 104 if yield_partials: 105 for value in values: 106 yield value 107 108## dropping and taking 109 110def drop[D](iterable: Iterable[D], n: int) -> Iterator[D]: 111 """Drop the next `n` values from `iterable`.""" 112 it = iter(iterable) 113 for _ in range(n): 114 try: 115 value = next(it) 116 except StopIteration: 117 break 118 return it 119 120def dropWhile[D](iterable: Iterable[D], pred: Callable[[D], bool]) -> Iterator[D]: 121 """Drop initial values from `iterable` while predicate is true.""" 122 it = iter(iterable) 123 try: 124 value = next(it) 125 except: 126 return it 127 128 while True: 129 try: 130 if not pred(value): 131 break 132 value = next(it) 133 except StopIteration: 134 break 135 return concat((value,), it) 136 137def take[D](iterable: Iterable[D], n: int) -> Iterator[D]: 138 """Take up to `n` values from `iterable`.""" 139 it = iter(iterable) 140 for _ in range(n): 141 try: 142 value = next(it) 143 yield value 144 except StopIteration: 145 break 146 147def takeWhile[D](iterable: Iterable[D], pred: Callable[[D], bool]) -> Iterator[D]: 148 """Yield values from `iterable` while predicate is true. 149 150 * potential value loss if iterable is iterator with external references 151 152 """ 153 it = iter(iterable) 154 while True: 155 try: 156 value = next(it) 157 if pred(value): 158 yield value 159 else: 160 break 161 except StopIteration: 162 break 163 164## reducing and accumulating 165 166def accumulate[D,L](iterable: Iterable[D], f: Callable[[L, D], L], 167 initial: Optional[L]=None) -> Iterator[L]: 168 """ 169 Returns an iterator of accumulated values. 170 171 * pure Python version of standard library's itertools.accumulate 172 * function f does not default to addition (for typing flexibility) 173 * begins accumulation with an optional starting value 174 * itertools.accumulate had mypy issues 175 176 """ 177 it = iter(iterable) 178 try: 179 it0 = next(it) 180 except StopIteration: 181 if initial is None: 182 return 183 else: 184 yield initial 185 else: 186 if initial is not None: 187 yield initial 188 acc = f(initial, it0) 189 for ii in it: 190 yield acc 191 acc = f(acc, ii) 192 yield acc 193 else: 194 acc = cast(L, it0) # in this case L = D 195 for ii in it: 196 yield acc 197 acc = f(acc, ii) 198 yield acc 199 200def foldL[D,L](iterable: Iterable[D], 201 f: Callable[[L, D], L], 202 initial: Optional[L]=None) -> MB[L]: 203 """ 204 Folds an iterable left with optional initial value. 205 206 * traditional FP type order given for function f 207 * when an initial value is not given then ~L = ~D 208 * if iterable empty & no initial value given, return empty MB() 209 * never returns if iterable generates an infinite iterator 210 211 """ 212 acc: L 213 it = iter(iterable) 214 215 if initial is None: 216 try: 217 acc = cast(L, next(it)) # in this case L = D 218 except StopIteration: 219 return MB() 220 else: 221 acc = initial 222 223 for v in it: 224 acc = f(acc, v) 225 226 return MB(acc) 227 228def foldR[D,R](iterable: Reversible[D], 229 f: Callable[[D, R], R], 230 initial: Optional[R]=None) -> MB[R]: 231 """ 232 Folds a reversible iterable right with an optional initial value. 233 234 * iterable needs to be reversible 235 * traditional FP type order given for function f 236 * when initial value is not given then ~R = ~D 237 * if iterable empty & no initial value given, return return empty MB() 238 239 """ 240 acc: R 241 it = reversed(iterable) 242 243 if initial is None: 244 try: 245 acc = cast(R, next(it)) # in this case R = D 246 except StopIteration: 247 return MB() 248 else: 249 acc = initial 250 251 for v in it: 252 acc = f(v, acc) 253 254 return MB(acc) 255 256def foldLsc[D,L,S](iterable: Iterable[D], 257 f: Callable[[L, D], L], 258 initial: Optional[L]=None, 259 stopfold: Callable[[D, S], MB[S]]=lambda d, s: MB(s), 260 istate: Optional[S]=None) -> MB[L]: 261 """ 262 Short circuit version of foldL. 263 264 * Callable `stopfold` purpose is to prematurely stop fold before end 265 * useful for infinite iterables 266 267 """ 268 state = cast(MB[S], MB(istate)) 269 270 it = iter(iterable) 271 272 if initial is None: 273 try: 274 acc = cast(L, next(it)) # in this case L = D 275 except StopIteration: 276 return MB() 277 else: 278 acc = initial 279 280 for d in it: 281 if (state := stopfold(d, state.get())): 282 acc = f(acc, d) 283 else: 284 break 285 286 return MB(acc) 287 288def foldRsc[D,R,S](iterable: Iterable[D], 289 f: Callable[[D, R], R], 290 initial: Optional[R]=None, 291 startfold: Callable[[D, S], MB[S]]=lambda d, s: MB(s), 292 istate: Optional[S]=None) -> MB[R]: 293 """ 294 Short circuit version of foldR. 295 296 * Callable `startfold` purpose is to start fold before end 297 * does NOT start fold at end and prematurely stop 298 * useful for infinite and non-reversible iterables 299 300 """ 301 state = cast(MB[S], MB(istate)) 302 303 it = iter(iterable) 304 305 acc: R 306 307 ds: list[D] = [] 308 for d in it: 309 if (state := startfold(d, state.get())): 310 ds.append(d) 311 else: 312 break 313 314 if initial is None: 315 if len(ds) == 0: 316 return MB() 317 else: 318 acc = cast(R, ds.pop()) # in this case R = D 319 else: 320 acc = initial 321 322 while ds: 323 acc = f(ds.pop(), acc) 324 325 return MB(acc)
def
drop(iterable: 'Iterable[D]', n: int) -> 'Iterator[D]':
111def drop[D](iterable: Iterable[D], n: int) -> Iterator[D]: 112 """Drop the next `n` values from `iterable`.""" 113 it = iter(iterable) 114 for _ in range(n): 115 try: 116 value = next(it) 117 except StopIteration: 118 break 119 return it
Drop the next n
values from iterable
.
def
dropWhile(iterable: 'Iterable[D]', pred: 'Callable[[D], bool]') -> 'Iterator[D]':
121def dropWhile[D](iterable: Iterable[D], pred: Callable[[D], bool]) -> Iterator[D]: 122 """Drop initial values from `iterable` while predicate is true.""" 123 it = iter(iterable) 124 try: 125 value = next(it) 126 except: 127 return it 128 129 while True: 130 try: 131 if not pred(value): 132 break 133 value = next(it) 134 except StopIteration: 135 break 136 return concat((value,), it)
Drop initial values from iterable
while predicate is true.
def
take(iterable: 'Iterable[D]', n: int) -> 'Iterator[D]':
138def take[D](iterable: Iterable[D], n: int) -> Iterator[D]: 139 """Take up to `n` values from `iterable`.""" 140 it = iter(iterable) 141 for _ in range(n): 142 try: 143 value = next(it) 144 yield value 145 except StopIteration: 146 break
Take up to n
values from iterable
.
def
takeWhile(iterable: 'Iterable[D]', pred: 'Callable[[D], bool]') -> 'Iterator[D]':
148def takeWhile[D](iterable: Iterable[D], pred: Callable[[D], bool]) -> Iterator[D]: 149 """Yield values from `iterable` while predicate is true. 150 151 * potential value loss if iterable is iterator with external references 152 153 """ 154 it = iter(iterable) 155 while True: 156 try: 157 value = next(it) 158 if pred(value): 159 yield value 160 else: 161 break 162 except StopIteration: 163 break
Yield values from iterable
while predicate is true.
- potential value loss if iterable is iterator with external references
def
concat(*iterables: 'Iterable[D]') -> 'Iterator[D]':
41def concat[D](*iterables: Iterable[D]) -> Iterator[D]: 42 """Sequentially concatenate multiple iterables together. 43 44 * pure Python version of standard library's itertools.chain 45 * iterator sequentially yields each iterable until all are exhausted 46 * an infinite iterable will prevent subsequent iterables from yielding any values 47 * performant to chain 48 49 """ 50 for iterator in map(lambda x: iter(x), iterables): 51 while True: 52 try: 53 value = next(iterator) 54 yield value 55 except StopIteration: 56 break
Sequentially concatenate multiple iterables together.
- pure Python version of standard library's itertools.chain
- iterator sequentially yields each iterable until all are exhausted
- an infinite iterable will prevent subsequent iterables from yielding any values
- performant to chain
def
merge(*iterables: 'Iterable[D]', yield_partials: bool = False) -> 'Iterator[D]':
85def merge[D](*iterables: Iterable[D], yield_partials: bool=False) -> Iterator[D]: 86 """Shuffle together multiple iterables until one is exhausted. 87 88 * iterator yields until one of the iterables is exhausted 89 * if yield_partials is true, yield any unmatched yielded values from other iterables 90 * prevents data lose if any of the iterables are iterators with external references 91 92 """ 93 iterList = list(map(lambda x: iter(x), iterables)) 94 values = [] 95 if (numIters := len(iterList)) > 0: 96 while True: 97 try: 98 for ii in range(numIters): 99 values.append(next(iterList[ii])) 100 for value in values: 101 yield value 102 values.clear() 103 except StopIteration: 104 break 105 if yield_partials: 106 for value in values: 107 yield value
Shuffle together multiple iterables until one is exhausted.
- iterator yields until one of the iterables is exhausted
- if yield_partials is true, yield any unmatched yielded values from other iterables
- prevents data lose if any of the iterables are iterators with external references
def
exhaust(*iterables: 'Iterable[D]') -> 'Iterator[D]':
58def exhaust[D](*iterables: Iterable[D]) -> Iterator[D]: 59 """Shuffle together multiple iterables until all are exhausted. 60 61 * iterator yields until all iterables are exhausted 62 63 """ 64 iterList = list(map(lambda x: iter(x), iterables)) 65 if (numIters := len(iterList)) > 0: 66 ii = 0 67 values = [] 68 while True: 69 try: 70 while ii < numIters: 71 values.append(next(iterList[ii])) 72 ii += 1 73 for value in values: 74 yield value 75 ii = 0 76 values.clear() 77 except StopIteration: 78 numIters -= 1 79 if numIters < 1: 80 break 81 del iterList[ii] 82 for value in values: 83 yield value
Shuffle together multiple iterables until all are exhausted.
- iterator yields until all iterables are exhausted
class
FM(enum.Enum):
CONCAT =
<FM.CONCAT: 1>
MERGE =
<FM.MERGE: 2>
EXHAUST =
<FM.EXHAUST: 3>
def
accumulate( iterable: 'Iterable[D]', f: 'Callable[[L, D], L]', initial: 'Optional[L]' = None) -> 'Iterator[L]':
167def accumulate[D,L](iterable: Iterable[D], f: Callable[[L, D], L], 168 initial: Optional[L]=None) -> Iterator[L]: 169 """ 170 Returns an iterator of accumulated values. 171 172 * pure Python version of standard library's itertools.accumulate 173 * function f does not default to addition (for typing flexibility) 174 * begins accumulation with an optional starting value 175 * itertools.accumulate had mypy issues 176 177 """ 178 it = iter(iterable) 179 try: 180 it0 = next(it) 181 except StopIteration: 182 if initial is None: 183 return 184 else: 185 yield initial 186 else: 187 if initial is not None: 188 yield initial 189 acc = f(initial, it0) 190 for ii in it: 191 yield acc 192 acc = f(acc, ii) 193 yield acc 194 else: 195 acc = cast(L, it0) # in this case L = D 196 for ii in it: 197 yield acc 198 acc = f(acc, ii) 199 yield acc
Returns an iterator of accumulated values.
- pure Python version of standard library's itertools.accumulate
- function f does not default to addition (for typing flexibility)
- begins accumulation with an optional starting value
- itertools.accumulate had mypy issues
def
foldL( iterable: 'Iterable[D]', f: 'Callable[[L, D], L]', initial: 'Optional[L]' = None) -> 'MB[L]':
201def foldL[D,L](iterable: Iterable[D], 202 f: Callable[[L, D], L], 203 initial: Optional[L]=None) -> MB[L]: 204 """ 205 Folds an iterable left with optional initial value. 206 207 * traditional FP type order given for function f 208 * when an initial value is not given then ~L = ~D 209 * if iterable empty & no initial value given, return empty MB() 210 * never returns if iterable generates an infinite iterator 211 212 """ 213 acc: L 214 it = iter(iterable) 215 216 if initial is None: 217 try: 218 acc = cast(L, next(it)) # in this case L = D 219 except StopIteration: 220 return MB() 221 else: 222 acc = initial 223 224 for v in it: 225 acc = f(acc, v) 226 227 return MB(acc)
Folds an iterable left with optional initial value.
- traditional FP type order given for function f
- when an initial value is not given then ~L = ~D
- if iterable empty & no initial value given, return empty MB()
- never returns if iterable generates an infinite iterator
def
foldR( iterable: 'Reversible[D]', f: 'Callable[[D, R], R]', initial: 'Optional[R]' = None) -> 'MB[R]':
229def foldR[D,R](iterable: Reversible[D], 230 f: Callable[[D, R], R], 231 initial: Optional[R]=None) -> MB[R]: 232 """ 233 Folds a reversible iterable right with an optional initial value. 234 235 * iterable needs to be reversible 236 * traditional FP type order given for function f 237 * when initial value is not given then ~R = ~D 238 * if iterable empty & no initial value given, return return empty MB() 239 240 """ 241 acc: R 242 it = reversed(iterable) 243 244 if initial is None: 245 try: 246 acc = cast(R, next(it)) # in this case R = D 247 except StopIteration: 248 return MB() 249 else: 250 acc = initial 251 252 for v in it: 253 acc = f(v, acc) 254 255 return MB(acc)
Folds a reversible iterable right with an optional initial value.
- iterable needs to be reversible
- traditional FP type order given for function f
- when initial value is not given then ~R = ~D
- if iterable empty & no initial value given, return return empty MB()
def
foldLsc( iterable: 'Iterable[D]', f: 'Callable[[L, D], L]', initial: 'Optional[L]' = None, stopfold: 'Callable[[D, S], MB[S]]' = <function <lambda>>, istate: 'Optional[S]' = None) -> 'MB[L]':
257def foldLsc[D,L,S](iterable: Iterable[D], 258 f: Callable[[L, D], L], 259 initial: Optional[L]=None, 260 stopfold: Callable[[D, S], MB[S]]=lambda d, s: MB(s), 261 istate: Optional[S]=None) -> MB[L]: 262 """ 263 Short circuit version of foldL. 264 265 * Callable `stopfold` purpose is to prematurely stop fold before end 266 * useful for infinite iterables 267 268 """ 269 state = cast(MB[S], MB(istate)) 270 271 it = iter(iterable) 272 273 if initial is None: 274 try: 275 acc = cast(L, next(it)) # in this case L = D 276 except StopIteration: 277 return MB() 278 else: 279 acc = initial 280 281 for d in it: 282 if (state := stopfold(d, state.get())): 283 acc = f(acc, d) 284 else: 285 break 286 287 return MB(acc)
Short circuit version of foldL.
- Callable
stopfold
purpose is to prematurely stop fold before end- useful for infinite iterables
def
foldRsc( iterable: 'Iterable[D]', f: 'Callable[[D, R], R]', initial: 'Optional[R]' = None, startfold: 'Callable[[D, S], MB[S]]' = <function <lambda>>, istate: 'Optional[S]' = None) -> 'MB[R]':
289def foldRsc[D,R,S](iterable: Iterable[D], 290 f: Callable[[D, R], R], 291 initial: Optional[R]=None, 292 startfold: Callable[[D, S], MB[S]]=lambda d, s: MB(s), 293 istate: Optional[S]=None) -> MB[R]: 294 """ 295 Short circuit version of foldR. 296 297 * Callable `startfold` purpose is to start fold before end 298 * does NOT start fold at end and prematurely stop 299 * useful for infinite and non-reversible iterables 300 301 """ 302 state = cast(MB[S], MB(istate)) 303 304 it = iter(iterable) 305 306 acc: R 307 308 ds: list[D] = [] 309 for d in it: 310 if (state := startfold(d, state.get())): 311 ds.append(d) 312 else: 313 break 314 315 if initial is None: 316 if len(ds) == 0: 317 return MB() 318 else: 319 acc = cast(R, ds.pop()) # in this case R = D 320 else: 321 acc = initial 322 323 while ds: 324 acc = f(ds.pop(), acc) 325 326 return MB(acc)
Short circuit version of foldR.
- Callable
startfold
purpose is to start fold before end- does NOT start fold at end and prematurely stop
- useful for infinite and non-reversible iterables