grscheller.fp.iterables

  • iterables are not necessarily iterators
  • at all times iterator protocol is assumed to be followed, that is
    • all iterators are assumed to be iterable
    • for all iterators foo we assume iter(foo) is foo
  1# Copyright 2023-2024 Geoffrey R. Scheller
  2#
  3# Licensed under the Apache License, Version 2.0 (the "License");
  4# you may not use this file except in compliance with the License.
  5# You may obtain a copy of the License at
  6#
  7#     http://www.apache.org/licenses/LICENSE-2.0
  8#
  9# Unless required by applicable law or agreed to in writing, software
 10# distributed under the License is distributed on an "AS IS" BASIS,
 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12# See the License for the specific language governing permissions and
 13# limitations under the License.
 14
 15"""
 16### Library of iterator related functions.
 17
 18* iterables are not necessarily iterators
 19* at all times iterator protocol is assumed to be followed, that is
 20  * all iterators are assumed to be iterable
 21  * for all iterators `foo` we assume `iter(foo) is foo`
 22
 23"""
 24from __future__ import annotations
 25from enum import auto, Enum
 26from typing import Callable, cast, Iterator, Iterable, Optional, Reversible
 27from .err_handling import MB
 28
 29__all__ = [ 'drop', 'dropWhile', 'take', 'takeWhile',
 30            'concat', 'merge', 'exhaust', 'FM',
 31            'accumulate', 'foldL', 'foldR', 'foldLsc', 'foldRsc' ]
 32
 33class FM(Enum):
 34    CONCAT = auto()
 35    MERGE = auto()
 36    EXHAUST = auto()
 37
 38## Iterate over multiple Iterables
 39
 40def concat[D](*iterables: Iterable[D]) -> Iterator[D]:
 41    """Sequentially concatenate multiple iterables together.
 42
 43    * pure Python version of standard library's itertools.chain
 44    * iterator sequentially yields each iterable until all are exhausted
 45    * an infinite iterable will prevent subsequent iterables from yielding any values
 46    * performant to chain
 47
 48    """
 49    for iterator in map(lambda x: iter(x), iterables):
 50        while True:
 51            try:
 52                value = next(iterator)
 53                yield value
 54            except StopIteration:
 55                break
 56
 57def exhaust[D](*iterables: Iterable[D]) -> Iterator[D]:
 58    """Shuffle together multiple iterables until all are exhausted.
 59
 60    * iterator yields until all iterables are exhausted
 61
 62    """
 63    iterList = list(map(lambda x: iter(x), iterables))
 64    if (numIters := len(iterList)) > 0:
 65        ii = 0
 66        values = []
 67        while True:
 68            try:
 69                while ii < numIters:
 70                    values.append(next(iterList[ii]))
 71                    ii += 1
 72                for value in values:
 73                    yield value
 74                ii = 0
 75                values.clear()
 76            except StopIteration:
 77                numIters -= 1
 78                if numIters < 1:
 79                    break
 80                del iterList[ii]
 81        for value in values:
 82            yield value
 83
 84def merge[D](*iterables: Iterable[D], yield_partials: bool=False) -> Iterator[D]:
 85    """Shuffle together multiple iterables until one is exhausted.
 86
 87    * iterator yields until one of the iterables is exhausted
 88    * if yield_partials is true, yield any unmatched yielded values from other iterables
 89      * prevents data lose if any of the iterables are iterators with external references
 90
 91    """
 92    iterList = list(map(lambda x: iter(x), iterables))
 93    values = []
 94    if (numIters := len(iterList)) > 0:
 95        while True:
 96            try:
 97                for ii in range(numIters):
 98                    values.append(next(iterList[ii]))
 99                for value in values:
100                    yield value
101                values.clear()
102            except StopIteration:
103                break
104        if yield_partials:
105            for value in values:
106                yield value
107
108## dropping and taking
109
110def drop[D](iterable: Iterable[D], n: int) -> Iterator[D]:
111    """Drop the next `n` values from `iterable`."""
112    it = iter(iterable)
113    for _ in range(n):
114        try:
115            value = next(it)
116        except StopIteration:
117            break
118    return it
119
120def dropWhile[D](iterable: Iterable[D], pred: Callable[[D], bool]) -> Iterator[D]:
121    """Drop initial values from `iterable` while predicate is true."""
122    it = iter(iterable)
123    try:
124        value = next(it)
125    except:
126        return it
127
128    while True:
129        try:
130            if not pred(value):
131                break
132            value = next(it)
133        except StopIteration:
134            break
135    return concat((value,), it)
136
137def take[D](iterable: Iterable[D], n: int) -> Iterator[D]:
138    """Take up to `n` values from `iterable`."""
139    it = iter(iterable)
140    for _ in range(n):
141        try:
142            value = next(it)
143            yield value
144        except StopIteration:
145            break
146
147def takeWhile[D](iterable: Iterable[D], pred: Callable[[D], bool]) -> Iterator[D]:
148    """Yield values from `iterable` while predicate is true.
149
150       * potential value loss if iterable is iterator with external references
151
152    """
153    it = iter(iterable)
154    while True:
155        try:
156            value = next(it)
157            if pred(value):
158                yield value
159            else:
160                break
161        except StopIteration:
162            break
163
164## reducing and accumulating
165
166def accumulate[D,L](iterable: Iterable[D], f: Callable[[L, D], L],
167                  initial: Optional[L]=None) -> Iterator[L]:
168    """
169    Returns an iterator of accumulated values.
170
171    * pure Python version of standard library's itertools.accumulate
172    * function f does not default to addition (for typing flexibility)
173    * begins accumulation with an optional starting value
174    * itertools.accumulate had mypy issues
175
176    """
177    it = iter(iterable)
178    try:
179        it0 = next(it)
180    except StopIteration:
181        if initial is None:
182            return
183        else:
184            yield initial
185    else:
186        if initial is not None:
187            yield initial
188            acc = f(initial, it0)
189            for ii in it:
190                yield acc
191                acc = f(acc, ii)
192            yield acc
193        else:
194            acc = cast(L, it0)  # in this case L = D
195            for ii in it:
196                yield acc
197                acc = f(acc, ii)
198            yield acc
199
200def foldL[D,L](iterable: Iterable[D],
201          f: Callable[[L, D], L],
202          initial: Optional[L]=None) -> MB[L]:
203    """
204    Folds an iterable left with optional initial value.
205
206    * traditional FP type order given for function f
207    * when an initial value is not given then ~L = ~D
208    * if iterable empty & no initial value given, return empty MB()
209    * never returns if iterable generates an infinite iterator
210
211    """
212    acc: L
213    it = iter(iterable)
214
215    if initial is None:
216        try:
217            acc = cast(L, next(it))  # in this case L = D
218        except StopIteration:
219            return MB()
220    else:
221        acc = initial
222
223    for v in it:
224        acc = f(acc, v)
225
226    return MB(acc)
227
228def foldR[D,R](iterable: Reversible[D],
229          f: Callable[[D, R], R],
230          initial: Optional[R]=None) -> MB[R]:
231    """
232    Folds a reversible iterable right with an optional initial value.
233
234    * iterable needs to be reversible
235    * traditional FP type order given for function f
236    * when initial value is not given then ~R = ~D
237    * if iterable empty & no initial value given, return return empty MB()
238
239    """
240    acc: R
241    it = reversed(iterable)
242
243    if initial is None:
244        try:
245            acc = cast(R, next(it))  # in this case R = D
246        except StopIteration:
247            return MB()
248    else:
249        acc = initial
250
251    for v in it:
252        acc = f(v, acc)
253
254    return MB(acc)
255
256def foldLsc[D,L,S](iterable: Iterable[D],
257            f: Callable[[L, D], L],
258            initial: Optional[L]=None,
259            stopfold: Callable[[D, S], MB[S]]=lambda d, s: MB(s),
260            istate: Optional[S]=None) -> MB[L]:
261    """
262    Short circuit version of foldL.
263
264    * Callable `stopfold` purpose is to prematurely stop fold before end
265      * useful for infinite iterables
266
267    """
268    state = cast(MB[S], MB(istate))
269
270    it = iter(iterable)
271
272    if initial is None:
273        try:
274            acc = cast(L, next(it))  # in this case L = D
275        except StopIteration:
276            return MB()
277    else:
278        acc = initial
279
280    for d in it:
281        if (state := stopfold(d, state.get())):
282            acc = f(acc, d)
283        else:
284            break
285
286    return MB(acc)
287
288def foldRsc[D,R,S](iterable: Iterable[D],
289            f: Callable[[D, R], R],
290            initial: Optional[R]=None,
291            startfold: Callable[[D, S], MB[S]]=lambda d, s: MB(s),
292            istate: Optional[S]=None) -> MB[R]:
293    """
294    Short circuit version of foldR.
295
296    * Callable `startfold` purpose is to start fold before end
297      * does NOT start fold at end and prematurely stop
298      * useful for infinite and non-reversible iterables
299
300    """
301    state = cast(MB[S], MB(istate))
302
303    it = iter(iterable)
304
305    acc: R
306
307    ds: list[D] = []
308    for d in it:
309        if (state := startfold(d, state.get())):
310            ds.append(d)
311        else:
312            break
313
314    if initial is None:
315        if len(ds) == 0:
316            return MB()
317        else:
318            acc = cast(R, ds.pop())  # in this case R = D
319    else:
320        acc = initial
321
322    while ds:
323        acc = f(ds.pop(), acc)
324
325    return MB(acc)
def drop(iterable: 'Iterable[D]', n: int) -> 'Iterator[D]':
111def drop[D](iterable: Iterable[D], n: int) -> Iterator[D]:
112    """Drop the next `n` values from `iterable`."""
113    it = iter(iterable)
114    for _ in range(n):
115        try:
116            value = next(it)
117        except StopIteration:
118            break
119    return it

Drop the next n values from iterable.

def dropWhile(iterable: 'Iterable[D]', pred: 'Callable[[D], bool]') -> 'Iterator[D]':
121def dropWhile[D](iterable: Iterable[D], pred: Callable[[D], bool]) -> Iterator[D]:
122    """Drop initial values from `iterable` while predicate is true."""
123    it = iter(iterable)
124    try:
125        value = next(it)
126    except:
127        return it
128
129    while True:
130        try:
131            if not pred(value):
132                break
133            value = next(it)
134        except StopIteration:
135            break
136    return concat((value,), it)

Drop initial values from iterable while predicate is true.

def take(iterable: 'Iterable[D]', n: int) -> 'Iterator[D]':
138def take[D](iterable: Iterable[D], n: int) -> Iterator[D]:
139    """Take up to `n` values from `iterable`."""
140    it = iter(iterable)
141    for _ in range(n):
142        try:
143            value = next(it)
144            yield value
145        except StopIteration:
146            break

Take up to n values from iterable.

def takeWhile(iterable: 'Iterable[D]', pred: 'Callable[[D], bool]') -> 'Iterator[D]':
148def takeWhile[D](iterable: Iterable[D], pred: Callable[[D], bool]) -> Iterator[D]:
149    """Yield values from `iterable` while predicate is true.
150
151       * potential value loss if iterable is iterator with external references
152
153    """
154    it = iter(iterable)
155    while True:
156        try:
157            value = next(it)
158            if pred(value):
159                yield value
160            else:
161                break
162        except StopIteration:
163            break

Yield values from iterable while predicate is true.

  • potential value loss if iterable is iterator with external references
def concat(*iterables: 'Iterable[D]') -> 'Iterator[D]':
41def concat[D](*iterables: Iterable[D]) -> Iterator[D]:
42    """Sequentially concatenate multiple iterables together.
43
44    * pure Python version of standard library's itertools.chain
45    * iterator sequentially yields each iterable until all are exhausted
46    * an infinite iterable will prevent subsequent iterables from yielding any values
47    * performant to chain
48
49    """
50    for iterator in map(lambda x: iter(x), iterables):
51        while True:
52            try:
53                value = next(iterator)
54                yield value
55            except StopIteration:
56                break

Sequentially concatenate multiple iterables together.

  • pure Python version of standard library's itertools.chain
  • iterator sequentially yields each iterable until all are exhausted
  • an infinite iterable will prevent subsequent iterables from yielding any values
  • performant to chain
def merge(*iterables: 'Iterable[D]', yield_partials: bool = False) -> 'Iterator[D]':
 85def merge[D](*iterables: Iterable[D], yield_partials: bool=False) -> Iterator[D]:
 86    """Shuffle together multiple iterables until one is exhausted.
 87
 88    * iterator yields until one of the iterables is exhausted
 89    * if yield_partials is true, yield any unmatched yielded values from other iterables
 90      * prevents data lose if any of the iterables are iterators with external references
 91
 92    """
 93    iterList = list(map(lambda x: iter(x), iterables))
 94    values = []
 95    if (numIters := len(iterList)) > 0:
 96        while True:
 97            try:
 98                for ii in range(numIters):
 99                    values.append(next(iterList[ii]))
100                for value in values:
101                    yield value
102                values.clear()
103            except StopIteration:
104                break
105        if yield_partials:
106            for value in values:
107                yield value

Shuffle together multiple iterables until one is exhausted.

  • iterator yields until one of the iterables is exhausted
  • if yield_partials is true, yield any unmatched yielded values from other iterables
    • prevents data lose if any of the iterables are iterators with external references
def exhaust(*iterables: 'Iterable[D]') -> 'Iterator[D]':
58def exhaust[D](*iterables: Iterable[D]) -> Iterator[D]:
59    """Shuffle together multiple iterables until all are exhausted.
60
61    * iterator yields until all iterables are exhausted
62
63    """
64    iterList = list(map(lambda x: iter(x), iterables))
65    if (numIters := len(iterList)) > 0:
66        ii = 0
67        values = []
68        while True:
69            try:
70                while ii < numIters:
71                    values.append(next(iterList[ii]))
72                    ii += 1
73                for value in values:
74                    yield value
75                ii = 0
76                values.clear()
77            except StopIteration:
78                numIters -= 1
79                if numIters < 1:
80                    break
81                del iterList[ii]
82        for value in values:
83            yield value

Shuffle together multiple iterables until all are exhausted.

  • iterator yields until all iterables are exhausted
class FM(enum.Enum):
34class FM(Enum):
35    CONCAT = auto()
36    MERGE = auto()
37    EXHAUST = auto()
CONCAT = <FM.CONCAT: 1>
MERGE = <FM.MERGE: 2>
EXHAUST = <FM.EXHAUST: 3>
def accumulate( iterable: 'Iterable[D]', f: 'Callable[[L, D], L]', initial: 'Optional[L]' = None) -> 'Iterator[L]':
167def accumulate[D,L](iterable: Iterable[D], f: Callable[[L, D], L],
168                  initial: Optional[L]=None) -> Iterator[L]:
169    """
170    Returns an iterator of accumulated values.
171
172    * pure Python version of standard library's itertools.accumulate
173    * function f does not default to addition (for typing flexibility)
174    * begins accumulation with an optional starting value
175    * itertools.accumulate had mypy issues
176
177    """
178    it = iter(iterable)
179    try:
180        it0 = next(it)
181    except StopIteration:
182        if initial is None:
183            return
184        else:
185            yield initial
186    else:
187        if initial is not None:
188            yield initial
189            acc = f(initial, it0)
190            for ii in it:
191                yield acc
192                acc = f(acc, ii)
193            yield acc
194        else:
195            acc = cast(L, it0)  # in this case L = D
196            for ii in it:
197                yield acc
198                acc = f(acc, ii)
199            yield acc

Returns an iterator of accumulated values.

  • pure Python version of standard library's itertools.accumulate
  • function f does not default to addition (for typing flexibility)
  • begins accumulation with an optional starting value
  • itertools.accumulate had mypy issues
def foldL( iterable: 'Iterable[D]', f: 'Callable[[L, D], L]', initial: 'Optional[L]' = None) -> 'MB[L]':
201def foldL[D,L](iterable: Iterable[D],
202          f: Callable[[L, D], L],
203          initial: Optional[L]=None) -> MB[L]:
204    """
205    Folds an iterable left with optional initial value.
206
207    * traditional FP type order given for function f
208    * when an initial value is not given then ~L = ~D
209    * if iterable empty & no initial value given, return empty MB()
210    * never returns if iterable generates an infinite iterator
211
212    """
213    acc: L
214    it = iter(iterable)
215
216    if initial is None:
217        try:
218            acc = cast(L, next(it))  # in this case L = D
219        except StopIteration:
220            return MB()
221    else:
222        acc = initial
223
224    for v in it:
225        acc = f(acc, v)
226
227    return MB(acc)

Folds an iterable left with optional initial value.

  • traditional FP type order given for function f
  • when an initial value is not given then ~L = ~D
  • if iterable empty & no initial value given, return empty MB()
  • never returns if iterable generates an infinite iterator
def foldR( iterable: 'Reversible[D]', f: 'Callable[[D, R], R]', initial: 'Optional[R]' = None) -> 'MB[R]':
229def foldR[D,R](iterable: Reversible[D],
230          f: Callable[[D, R], R],
231          initial: Optional[R]=None) -> MB[R]:
232    """
233    Folds a reversible iterable right with an optional initial value.
234
235    * iterable needs to be reversible
236    * traditional FP type order given for function f
237    * when initial value is not given then ~R = ~D
238    * if iterable empty & no initial value given, return return empty MB()
239
240    """
241    acc: R
242    it = reversed(iterable)
243
244    if initial is None:
245        try:
246            acc = cast(R, next(it))  # in this case R = D
247        except StopIteration:
248            return MB()
249    else:
250        acc = initial
251
252    for v in it:
253        acc = f(v, acc)
254
255    return MB(acc)

Folds a reversible iterable right with an optional initial value.

  • iterable needs to be reversible
  • traditional FP type order given for function f
  • when initial value is not given then ~R = ~D
  • if iterable empty & no initial value given, return return empty MB()
def foldLsc( iterable: 'Iterable[D]', f: 'Callable[[L, D], L]', initial: 'Optional[L]' = None, stopfold: 'Callable[[D, S], MB[S]]' = <function <lambda>>, istate: 'Optional[S]' = None) -> 'MB[L]':
257def foldLsc[D,L,S](iterable: Iterable[D],
258            f: Callable[[L, D], L],
259            initial: Optional[L]=None,
260            stopfold: Callable[[D, S], MB[S]]=lambda d, s: MB(s),
261            istate: Optional[S]=None) -> MB[L]:
262    """
263    Short circuit version of foldL.
264
265    * Callable `stopfold` purpose is to prematurely stop fold before end
266      * useful for infinite iterables
267
268    """
269    state = cast(MB[S], MB(istate))
270
271    it = iter(iterable)
272
273    if initial is None:
274        try:
275            acc = cast(L, next(it))  # in this case L = D
276        except StopIteration:
277            return MB()
278    else:
279        acc = initial
280
281    for d in it:
282        if (state := stopfold(d, state.get())):
283            acc = f(acc, d)
284        else:
285            break
286
287    return MB(acc)

Short circuit version of foldL.

  • Callable stopfold purpose is to prematurely stop fold before end
    • useful for infinite iterables
def foldRsc( iterable: 'Iterable[D]', f: 'Callable[[D, R], R]', initial: 'Optional[R]' = None, startfold: 'Callable[[D, S], MB[S]]' = <function <lambda>>, istate: 'Optional[S]' = None) -> 'MB[R]':
289def foldRsc[D,R,S](iterable: Iterable[D],
290            f: Callable[[D, R], R],
291            initial: Optional[R]=None,
292            startfold: Callable[[D, S], MB[S]]=lambda d, s: MB(s),
293            istate: Optional[S]=None) -> MB[R]:
294    """
295    Short circuit version of foldR.
296
297    * Callable `startfold` purpose is to start fold before end
298      * does NOT start fold at end and prematurely stop
299      * useful for infinite and non-reversible iterables
300
301    """
302    state = cast(MB[S], MB(istate))
303
304    it = iter(iterable)
305
306    acc: R
307
308    ds: list[D] = []
309    for d in it:
310        if (state := startfold(d, state.get())):
311            ds.append(d)
312        else:
313            break
314
315    if initial is None:
316        if len(ds) == 0:
317            return MB()
318        else:
319            acc = cast(R, ds.pop())  # in this case R = D
320    else:
321        acc = initial
322
323    while ds:
324        acc = f(ds.pop(), acc)
325
326    return MB(acc)

Short circuit version of foldR.

  • Callable startfold purpose is to start fold before end
    • does NOT start fold at end and prematurely stop
    • useful for infinite and non-reversible iterables