grscheller.fp.iterables

  • iterables are not necessarily iterators
  • at all times iterator protocol is assumed to be followed, that is
    • all iterators are assumed to be iterable
    • for all iterators foo we assume iter(foo) is foo
  1# Copyright 2023-2024 Geoffrey R. Scheller
  2#
  3# Licensed under the Apache License, Version 2.0 (the "License");
  4# you may not use this file except in compliance with the License.
  5# You may obtain a copy of the License at
  6#
  7#     http://www.apache.org/licenses/LICENSE-2.0
  8#
  9# Unless required by applicable law or agreed to in writing, software
 10# distributed under the License is distributed on an "AS IS" BASIS,
 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12# See the License for the specific language governing permissions and
 13# limitations under the License.
 14
 15"""
 16### Library of iterator related functions.
 17
 18* iterables are not necessarily iterators
 19* at all times iterator protocol is assumed to be followed, that is
 20  * all iterators are assumed to be iterable
 21  * for all iterators `foo` we assume `iter(foo) is foo`
 22
 23"""
 24from __future__ import annotations
 25from enum import auto, Enum
 26from typing import Callable, cast, Final, Iterator, Iterable
 27from typing import overload, Optional, Reversible, TypeVar
 28from .woException import MB
 29
 30__all__ = [ 'drop', 'dropWhile', 'take', 'takeWhile',
 31            'concat', 'merge', 'exhaust', 'FM',
 32            'accumulate', 'foldL', 'foldR', 'foldLsc', 'foldRsc' ]
 33
 34class FM(Enum):
 35    CONCAT = auto()
 36    MERGE = auto()
 37    EXHAUST = auto()
 38
 39D = TypeVar('D')      # D for Data
 40L = TypeVar('L')      # L for Left
 41R = TypeVar('R')      # R for Right
 42S = TypeVar('S')      # S for State
 43
 44## Iterate over multiple Iterables
 45
 46def concat(*iterables: Iterable[D]) -> Iterator[D]:
 47    """Sequentially concatenate multiple iterables together.
 48
 49    * pure Python version of standard library's itertools.chain
 50    * iterator sequentially yields each iterable until all are exhausted
 51    * an infinite iterable will prevent subsequent iterables from yielding any values
 52    * performant to chain
 53
 54    """
 55    for iterator in map(lambda x: iter(x), iterables):
 56        while True:
 57            try:
 58                value = next(iterator)
 59                yield value
 60            except StopIteration:
 61                break
 62
 63def exhaust(*iterables: Iterable[D]) -> Iterator[D]:
 64    """Shuffle together multiple iterables until all are exhausted.
 65
 66    * iterator yields until all iterables are exhausted
 67
 68    """
 69    iterList = list(map(lambda x: iter(x), iterables))
 70    if (numIters := len(iterList)) > 0:
 71        ii = 0
 72        values = []
 73        while True:
 74            try:
 75                while ii < numIters:
 76                    values.append(next(iterList[ii]))
 77                    ii += 1
 78                for value in values:
 79                    yield value
 80                ii = 0
 81                values.clear()
 82            except StopIteration:
 83                numIters -= 1
 84                if numIters < 1:
 85                    break
 86                del iterList[ii]
 87        for value in values:
 88            yield value
 89
 90def merge(*iterables: Iterable[D], yield_partials: bool=False) -> Iterator[D]:
 91    """Shuffle together multiple iterables until one is exhausted.
 92
 93    * iterator yields until one of the iterables is exhausted
 94    * if yield_partials is true, yield any unmatched yielded values from other iterables
 95      * prevents data lose if any of the iterables are iterators with external references
 96
 97    """
 98    iterList = list(map(lambda x: iter(x), iterables))
 99    values = []
100    if (numIters := len(iterList)) > 0:
101        while True:
102            try:
103                for ii in range(numIters):
104                    values.append(next(iterList[ii]))
105                for value in values:
106                    yield value
107                values.clear()
108            except StopIteration:
109                break
110        if yield_partials:
111            for value in values:
112                yield value
113
114## dropping and taking
115
116def drop(iterable: Iterable[D], n: int) -> Iterator[D]:
117    """Drop the next `n` values from `iterable`."""
118    it = iter(iterable)
119    for _ in range(n):
120        try:
121            value = next(it)
122        except StopIteration:
123            break
124    return it
125
126def dropWhile(iterable: Iterable[D], pred: Callable[[D], bool]) -> Iterator[D]:
127    """Drop initial values from `iterable` while predicate is true."""
128    it = iter(iterable)
129    try:
130        value = next(it)
131    except:
132        return it
133
134    while True:
135        try:
136            if not pred(value):
137                break
138            value = next(it)
139        except StopIteration:
140            break
141    return concat((value,), it)
142
143def take(iterable: Iterable[D], n: int) -> Iterator[D]:
144    """Take up to `n` values from `iterable`."""
145    it = iter(iterable)
146    for _ in range(n):
147        try:
148            value = next(it)
149            yield value
150        except StopIteration:
151            break
152
153def takeWhile(iterable: Iterable[D], pred: Callable[[D], bool]) -> Iterator[D]:
154    """Yield values from `iterable` while predicate is true.
155
156       * potential value loss if iterable is iterator with external references
157
158    """
159    it = iter(iterable)
160    while True:
161        try:
162            value = next(it)
163            if pred(value):
164                yield value
165            else:
166                break
167        except StopIteration:
168            break
169
170## reducing and accumulating
171
172def accumulate(iterable: Iterable[D], f: Callable[[L, D], L],
173               initial: Optional[L]=None) -> Iterator[L]:
174    """
175    Returns an iterator of accumulated values.
176
177    * pure Python version of standard library's itertools.accumulate
178    * function f does not default to addition (for typing flexibility)
179    * begins accumulation with an optional starting value
180    * itertools.accumulate had mypy issues
181
182    """
183    it = iter(iterable)
184    try:
185        it0 = next(it)
186    except StopIteration:
187        if initial is None:
188            return
189        else:
190            yield initial
191    else:
192        if initial is not None:
193            yield initial
194            acc = f(initial, it0)
195            for ii in it:
196                yield acc
197                acc = f(acc, ii)
198            yield acc
199        else:
200            acc = cast(L, it0)  # in this case L = D
201            for ii in it:
202                yield acc
203                acc = f(acc, ii)
204            yield acc
205
206def foldL(iterable: Iterable[D],
207          f: Callable[[L, D], L],
208          initial: Optional[L]=None) -> MB[L]:
209    """
210    Folds an iterable left with optional initial value.
211
212    * traditional FP type order given for function f
213    * when an initial value is not given then ~L = ~D
214    * if iterable empty & no initial value given, return empty MB()
215    * never returns if iterable generates an infinite iterator
216
217    """
218    acc: L
219    it = iter(iterable)
220
221    if initial is None:
222        try:
223            acc = cast(L, next(it))  # in this case L = D
224        except StopIteration:
225            return MB()
226    else:
227        acc = initial
228
229    for v in it:
230        acc = f(acc, v)
231
232    return MB(acc)
233
234def foldR(iterable: Reversible[D],
235          f: Callable[[D, R], R],
236          initial: Optional[R]=None) -> MB[R]:
237    """
238    Folds a reversible iterable right with an optional initial value.
239
240    * iterable needs to be reversible
241    * traditional FP type order given for function f
242    * when initial value is not given then ~R = ~D
243    * if iterable empty & no initial value given, return return empty MB()
244
245    """
246    acc: R
247    it = reversed(iterable)
248
249    if initial is None:
250        try:
251            acc = cast(R, next(it))  # in this case R = D
252        except StopIteration:
253            return MB()
254    else:
255        acc = initial
256
257    for v in it:
258        acc = f(v, acc)
259
260    return MB(acc)
261
262def foldLsc(iterable: Iterable[D],
263            f: Callable[[L, D], L],
264            initial: Optional[L]=None,
265            stopfold: Callable[[D, S], MB[S]]=lambda d, s: MB(s),
266            istate: Optional[S]=None) -> MB[L]:
267    """
268    Short circuit version of foldL.
269
270    * Callable `stopfold` purpose is to prematurely stop fold before end
271      * useful for infinite iterables
272
273    """
274    state = cast(MB[S], MB(istate))
275
276    it = iter(iterable)
277
278    if initial is None:
279        try:
280            acc = cast(L, next(it))  # in this case L = D
281        except StopIteration:
282            return MB()
283    else:
284        acc = initial
285
286    for d in it:
287        if (state := stopfold(d, state.get())):
288            acc = f(acc, d)
289        else:
290            break
291
292    return MB(acc)
293
294def foldRsc(iterable: Iterable[D],
295            f: Callable[[D, R], R],
296            initial: Optional[R]=None,
297            startfold: Callable[[D, S], MB[S]]=lambda d, s: MB(s),
298            istate: Optional[S]=None) -> MB[R]:
299    """
300    Short circuit version of foldR.
301
302    * Callable `startfold` purpose is to start fold before end
303      * does NOT start fold at end and prematurely stop
304      * useful for infinite and non-reversible iterables
305
306    """
307    state = cast(MB[S], MB(istate))
308
309    it = iter(iterable)
310
311    acc: R
312
313    ds: list[D] = []
314    for d in it:
315        if (state := startfold(d, state.get())):
316            ds.append(d)
317        else:
318            break
319
320    if initial is None:
321        if len(ds) == 0:
322            return MB()
323        else:
324            acc = cast(R, ds.pop())  # in this case R = D
325    else:
326        acc = initial
327
328    while ds:
329        acc = f(ds.pop(), acc)
330
331    return MB(acc)
def drop(iterable: Iterable[~D], n: int) -> Iterator[~D]:
117def drop(iterable: Iterable[D], n: int) -> Iterator[D]:
118    """Drop the next `n` values from `iterable`."""
119    it = iter(iterable)
120    for _ in range(n):
121        try:
122            value = next(it)
123        except StopIteration:
124            break
125    return it

Drop the next n values from iterable.

def dropWhile(iterable: Iterable[~D], pred: Callable[[~D], bool]) -> Iterator[~D]:
127def dropWhile(iterable: Iterable[D], pred: Callable[[D], bool]) -> Iterator[D]:
128    """Drop initial values from `iterable` while predicate is true."""
129    it = iter(iterable)
130    try:
131        value = next(it)
132    except:
133        return it
134
135    while True:
136        try:
137            if not pred(value):
138                break
139            value = next(it)
140        except StopIteration:
141            break
142    return concat((value,), it)

Drop initial values from iterable while predicate is true.

def take(iterable: Iterable[~D], n: int) -> Iterator[~D]:
144def take(iterable: Iterable[D], n: int) -> Iterator[D]:
145    """Take up to `n` values from `iterable`."""
146    it = iter(iterable)
147    for _ in range(n):
148        try:
149            value = next(it)
150            yield value
151        except StopIteration:
152            break

Take up to n values from iterable.

def takeWhile(iterable: Iterable[~D], pred: Callable[[~D], bool]) -> Iterator[~D]:
154def takeWhile(iterable: Iterable[D], pred: Callable[[D], bool]) -> Iterator[D]:
155    """Yield values from `iterable` while predicate is true.
156
157       * potential value loss if iterable is iterator with external references
158
159    """
160    it = iter(iterable)
161    while True:
162        try:
163            value = next(it)
164            if pred(value):
165                yield value
166            else:
167                break
168        except StopIteration:
169            break

Yield values from iterable while predicate is true.

  • potential value loss if iterable is iterator with external references
def concat(*iterables: Iterable[~D]) -> Iterator[~D]:
47def concat(*iterables: Iterable[D]) -> Iterator[D]:
48    """Sequentially concatenate multiple iterables together.
49
50    * pure Python version of standard library's itertools.chain
51    * iterator sequentially yields each iterable until all are exhausted
52    * an infinite iterable will prevent subsequent iterables from yielding any values
53    * performant to chain
54
55    """
56    for iterator in map(lambda x: iter(x), iterables):
57        while True:
58            try:
59                value = next(iterator)
60                yield value
61            except StopIteration:
62                break

Sequentially concatenate multiple iterables together.

  • pure Python version of standard library's itertools.chain
  • iterator sequentially yields each iterable until all are exhausted
  • an infinite iterable will prevent subsequent iterables from yielding any values
  • performant to chain
def merge(*iterables: Iterable[~D], yield_partials: bool = False) -> Iterator[~D]:
 91def merge(*iterables: Iterable[D], yield_partials: bool=False) -> Iterator[D]:
 92    """Shuffle together multiple iterables until one is exhausted.
 93
 94    * iterator yields until one of the iterables is exhausted
 95    * if yield_partials is true, yield any unmatched yielded values from other iterables
 96      * prevents data lose if any of the iterables are iterators with external references
 97
 98    """
 99    iterList = list(map(lambda x: iter(x), iterables))
100    values = []
101    if (numIters := len(iterList)) > 0:
102        while True:
103            try:
104                for ii in range(numIters):
105                    values.append(next(iterList[ii]))
106                for value in values:
107                    yield value
108                values.clear()
109            except StopIteration:
110                break
111        if yield_partials:
112            for value in values:
113                yield value

Shuffle together multiple iterables until one is exhausted.

  • iterator yields until one of the iterables is exhausted
  • if yield_partials is true, yield any unmatched yielded values from other iterables
    • prevents data lose if any of the iterables are iterators with external references
def exhaust(*iterables: Iterable[~D]) -> Iterator[~D]:
64def exhaust(*iterables: Iterable[D]) -> Iterator[D]:
65    """Shuffle together multiple iterables until all are exhausted.
66
67    * iterator yields until all iterables are exhausted
68
69    """
70    iterList = list(map(lambda x: iter(x), iterables))
71    if (numIters := len(iterList)) > 0:
72        ii = 0
73        values = []
74        while True:
75            try:
76                while ii < numIters:
77                    values.append(next(iterList[ii]))
78                    ii += 1
79                for value in values:
80                    yield value
81                ii = 0
82                values.clear()
83            except StopIteration:
84                numIters -= 1
85                if numIters < 1:
86                    break
87                del iterList[ii]
88        for value in values:
89            yield value

Shuffle together multiple iterables until all are exhausted.

  • iterator yields until all iterables are exhausted
class FM(enum.Enum):
35class FM(Enum):
36    CONCAT = auto()
37    MERGE = auto()
38    EXHAUST = auto()
CONCAT = <FM.CONCAT: 1>
MERGE = <FM.MERGE: 2>
EXHAUST = <FM.EXHAUST: 3>
Inherited Members
enum.Enum
name
value
def accumulate( iterable: Iterable[~D], f: Callable[[~L, ~D], ~L], initial: Optional[~L] = None) -> Iterator[~L]:
173def accumulate(iterable: Iterable[D], f: Callable[[L, D], L],
174               initial: Optional[L]=None) -> Iterator[L]:
175    """
176    Returns an iterator of accumulated values.
177
178    * pure Python version of standard library's itertools.accumulate
179    * function f does not default to addition (for typing flexibility)
180    * begins accumulation with an optional starting value
181    * itertools.accumulate had mypy issues
182
183    """
184    it = iter(iterable)
185    try:
186        it0 = next(it)
187    except StopIteration:
188        if initial is None:
189            return
190        else:
191            yield initial
192    else:
193        if initial is not None:
194            yield initial
195            acc = f(initial, it0)
196            for ii in it:
197                yield acc
198                acc = f(acc, ii)
199            yield acc
200        else:
201            acc = cast(L, it0)  # in this case L = D
202            for ii in it:
203                yield acc
204                acc = f(acc, ii)
205            yield acc

Returns an iterator of accumulated values.

  • pure Python version of standard library's itertools.accumulate
  • function f does not default to addition (for typing flexibility)
  • begins accumulation with an optional starting value
  • itertools.accumulate had mypy issues
def foldL( iterable: Iterable[~D], f: Callable[[~L, ~D], ~L], initial: Optional[~L] = None) -> grscheller.fp.woException.MB[~L]:
207def foldL(iterable: Iterable[D],
208          f: Callable[[L, D], L],
209          initial: Optional[L]=None) -> MB[L]:
210    """
211    Folds an iterable left with optional initial value.
212
213    * traditional FP type order given for function f
214    * when an initial value is not given then ~L = ~D
215    * if iterable empty & no initial value given, return empty MB()
216    * never returns if iterable generates an infinite iterator
217
218    """
219    acc: L
220    it = iter(iterable)
221
222    if initial is None:
223        try:
224            acc = cast(L, next(it))  # in this case L = D
225        except StopIteration:
226            return MB()
227    else:
228        acc = initial
229
230    for v in it:
231        acc = f(acc, v)
232
233    return MB(acc)

Folds an iterable left with optional initial value.

  • traditional FP type order given for function f
  • when an initial value is not given then ~L = ~D
  • if iterable empty & no initial value given, return empty MB()
  • never returns if iterable generates an infinite iterator
def foldR( iterable: Reversible[~D], f: Callable[[~D, ~R], ~R], initial: Optional[~R] = None) -> grscheller.fp.woException.MB[~R]:
235def foldR(iterable: Reversible[D],
236          f: Callable[[D, R], R],
237          initial: Optional[R]=None) -> MB[R]:
238    """
239    Folds a reversible iterable right with an optional initial value.
240
241    * iterable needs to be reversible
242    * traditional FP type order given for function f
243    * when initial value is not given then ~R = ~D
244    * if iterable empty & no initial value given, return return empty MB()
245
246    """
247    acc: R
248    it = reversed(iterable)
249
250    if initial is None:
251        try:
252            acc = cast(R, next(it))  # in this case R = D
253        except StopIteration:
254            return MB()
255    else:
256        acc = initial
257
258    for v in it:
259        acc = f(v, acc)
260
261    return MB(acc)

Folds a reversible iterable right with an optional initial value.

  • iterable needs to be reversible
  • traditional FP type order given for function f
  • when initial value is not given then ~R = ~D
  • if iterable empty & no initial value given, return return empty MB()
def foldLsc( iterable: Iterable[~D], f: Callable[[~L, ~D], ~L], initial: Optional[~L] = None, stopfold: Callable[[~D, ~S], grscheller.fp.woException.MB[~S]] = <function <lambda>>, istate: Optional[~S] = None) -> grscheller.fp.woException.MB[~L]:
263def foldLsc(iterable: Iterable[D],
264            f: Callable[[L, D], L],
265            initial: Optional[L]=None,
266            stopfold: Callable[[D, S], MB[S]]=lambda d, s: MB(s),
267            istate: Optional[S]=None) -> MB[L]:
268    """
269    Short circuit version of foldL.
270
271    * Callable `stopfold` purpose is to prematurely stop fold before end
272      * useful for infinite iterables
273
274    """
275    state = cast(MB[S], MB(istate))
276
277    it = iter(iterable)
278
279    if initial is None:
280        try:
281            acc = cast(L, next(it))  # in this case L = D
282        except StopIteration:
283            return MB()
284    else:
285        acc = initial
286
287    for d in it:
288        if (state := stopfold(d, state.get())):
289            acc = f(acc, d)
290        else:
291            break
292
293    return MB(acc)

Short circuit version of foldL.

  • Callable stopfold purpose is to prematurely stop fold before end
    • useful for infinite iterables
def foldRsc( iterable: Iterable[~D], f: Callable[[~D, ~R], ~R], initial: Optional[~R] = None, startfold: Callable[[~D, ~S], grscheller.fp.woException.MB[~S]] = <function <lambda>>, istate: Optional[~S] = None) -> grscheller.fp.woException.MB[~R]:
295def foldRsc(iterable: Iterable[D],
296            f: Callable[[D, R], R],
297            initial: Optional[R]=None,
298            startfold: Callable[[D, S], MB[S]]=lambda d, s: MB(s),
299            istate: Optional[S]=None) -> MB[R]:
300    """
301    Short circuit version of foldR.
302
303    * Callable `startfold` purpose is to start fold before end
304      * does NOT start fold at end and prematurely stop
305      * useful for infinite and non-reversible iterables
306
307    """
308    state = cast(MB[S], MB(istate))
309
310    it = iter(iterable)
311
312    acc: R
313
314    ds: list[D] = []
315    for d in it:
316        if (state := startfold(d, state.get())):
317            ds.append(d)
318        else:
319            break
320
321    if initial is None:
322        if len(ds) == 0:
323            return MB()
324        else:
325            acc = cast(R, ds.pop())  # in this case R = D
326    else:
327        acc = initial
328
329    while ds:
330        acc = f(ds.pop(), acc)
331
332    return MB(acc)

Short circuit version of foldR.

  • Callable startfold purpose is to start fold before end
    • does NOT start fold at end and prematurely stop
    • useful for infinite and non-reversible iterables