grscheller.fp.iterables

  • iterables are not necessarily iterators
  • at all times iterator protocol is assumed to be followed, that is
    • for all iterators foo we assume iter(foo) is foo
    • all iterators are assumed to be iterable
  1# Copyright 2023-2024 Geoffrey R. Scheller
  2#
  3# Licensed under the Apache License, Version 2.0 (the "License");
  4# you may not use this file except in compliance with the License.
  5# You may obtain a copy of the License at
  6#
  7#     http://www.apache.org/licenses/LICENSE-2.0
  8#
  9# Unless required by applicable law or agreed to in writing, software
 10# distributed under the License is distributed on an "AS IS" BASIS,
 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12# See the License for the specific language governing permissions and
 13# limitations under the License.
 14
 15"""
 16#### Library of iterator related functions.
 17
 18* iterables are not necessarily iterators
 19* at all times iterator protocol is assumed to be followed, that is
 20  * for all iterators `foo` we assume `iter(foo) is foo`
 21  * all iterators are assumed to be iterable
 22
 23"""
 24
 25from __future__ import annotations
 26from enum import auto, Enum
 27from typing import Callable, cast, Final, Iterator, Iterable
 28from typing import overload, Optional, Reversible, TypeVar
 29from .nada import Nada, nada
 30
 31__all__ = [ 'accumulate', 'foldL', 'foldR',
 32            'concat', 'merge', 'exhaust', 'FM' ]
 33
 34class FM(Enum):
 35    CONCAT = auto()
 36    MERGE = auto()
 37    EXHAUST = auto()
 38
 39D = TypeVar('D')
 40L = TypeVar('L')
 41R = TypeVar('R')
 42S = TypeVar('S')
 43
 44## Iterate over multiple Iterables
 45
 46def concat(*iterables: Iterable[D]) -> Iterator[D]:
 47    """
 48    #### Sequentially concatenate multiple iterables together.
 49
 50    * pure Python version of standard library's itertools.chain
 51    * iterator yields Sequentially each iterable until all are exhausted
 52    * an infinite iterable will prevent subsequent iterables from yielding any values
 53    * performant to chain
 54
 55    """
 56    iterator: Iterator[D]
 57    for iterator in map(lambda x: iter(x), iterables):
 58        while True:
 59            try:
 60                value: D = next(iterator)
 61                yield value
 62            except StopIteration:
 63                break
 64
 65def exhaust(*iterables: Iterable[D]) -> Iterator[D]:
 66    """
 67    #### Shuffle together multiple iterables until all are exhausted.
 68
 69    * iterator yields until all iterables are exhausted
 70
 71    """
 72    iterList = list(map(lambda x: iter(x), iterables))
 73    if (numIters := len(iterList)) > 0:
 74        ii = 0
 75        values = []
 76        while True:
 77            try:
 78                while ii < numIters:
 79                    values.append(next(iterList[ii]))
 80                    ii += 1
 81                for value in values:
 82                    yield value
 83                ii = 0
 84                values.clear()
 85            except StopIteration:
 86                numIters -= 1
 87                if numIters < 1:
 88                    break
 89                del iterList[ii]
 90        for value in values:
 91            yield value
 92
 93def merge(*iterables: Iterable[D], yield_partials: bool=False) -> Iterator[D]:
 94    """
 95    #### Shuffle together multiple iterables until one is exhausted.
 96
 97    * iterator yields until one of the iterables is exhausted
 98    * if yield_partials is true, yield any unmatched yielded values from the other iterables
 99    * this prevents data lose if any of the iterables are iterators with external references
100
101    """
102    iterList = list(map(lambda x: iter(x), iterables))
103    if (numIters := len(iterList)) > 0:
104        values = []
105        while True:
106            try:
107                for ii in range(numIters):
108                    values.append(next(iterList[ii]))
109                for value in values:
110                    yield value
111                values.clear()
112            except StopIteration:
113                break
114        if yield_partials:
115            for value in values:
116                yield value
117
118## reducing and accumulating
119
120def accumulate(iterable: Iterable[D], f: Callable[[L, D], L],
121               initial: Optional[L]=None) -> Iterator[L]:
122    """
123    #### Returns an iterator of accumulated values.
124
125    * pure Python version of standard library's itertools.accumulate
126    * function f does not default to addition (for typing flexibility)
127    * begins accumulation with an optional starting value
128    * itertools.accumulate has mypy issues
129
130    """
131    it = iter(iterable)
132    try:
133        it0 = next(it)
134    except StopIteration:
135        if initial is None:
136            return
137        else:
138            yield initial
139    else:
140        if initial is not None:
141            yield initial
142            acc = f(initial, it0)
143            for ii in it:
144                yield acc
145                acc = f(acc, ii)
146            yield acc
147        else:
148            acc = cast(L, it0)  # in this case L = D
149            for ii in it:
150                yield acc
151                acc = f(acc, ii)
152            yield acc
153
154@overload
155def foldL(iterable: Iterable[D], f: Callable[[L, D], L], initial: Optional[L], default: S) -> L|S:
156    ...
157@overload
158def foldL(iterable: Iterable[D], f: Callable[[D, D], D]) -> D|Nada:
159    ...
160@overload
161def foldL(iterable: Iterable[D], f: Callable[[L, D], L], initial: L) -> L:
162    ...
163@overload
164def foldL(iterable: Iterable[D], f: Callable[[L, D], L], initial: Nada) -> Nada:
165    ...
166
167def foldL(iterable: Iterable[D], f: Callable[[L, D], L],
168          initial: Optional[L]=None, default: S|Nada=nada) -> L|S|Nada:
169    """
170    #### Folds iterable left with optional initial value.
171
172    * note that ~S can be the same type as ~L
173    * note that when an initial value is not given then ~L = ~D
174    * if iterable empty & no initial value given, return default
175    * traditional FP type order given for function f
176    * raises TypeError if the "iterable" is not iterable
177    * never returns if iterable generates an infinite iterator
178
179    """
180    acc: L
181    if hasattr(iterable, '__iter__'):
182        it = iter(iterable)
183    else:
184        msg = '"Iterable" is not iterable.'
185        raise TypeError(msg)
186
187    if initial is None:
188        try:
189            acc = cast(L, next(it))  # in this case L = D
190        except StopIteration:
191            return cast(S, default)  # if default = nothing, then S is Nothing
192    else:
193        acc = initial
194
195    for v in it:
196        acc = f(acc, v)
197
198    return acc
199
200@overload
201def foldR(iterable: Reversible[D], f: Callable[[D, R], R], initial: Optional[R], default: S) -> R|S:
202    ...
203@overload
204def foldR(iterable: Reversible[D], f: Callable[[D, D], D]) -> D|Nothing:
205    ...
206@overload
207def foldR(iterable: Reversible[D], f: Callable[[D, R], R], initial: R) -> R:
208    ...
209@overload
210def foldR(iterable: Reversible[D], f: Callable[[D, R], R], initial: Nada) -> R|Nada:
211    ...
212
213def foldR(iterable: Reversible[D], f: Callable[[D, R], R],
214          initial: Optional[R]=None, default: S|Nada=nada) -> R|S|Nada:
215    """
216    #### Folds reversible iterable right with an optional initial value.
217
218    * note that ~S can be the same type as ~R
219    * note that when an initial value not given then ~R = ~D
220    * if iterable empty & no initial value given, return default
221    * traditional FP type order given for function f
222    * raises TypeError if iterable is not reversible
223
224    """
225    acc: R
226    if hasattr(iterable, '__reversed__') or hasattr(iterable, '__len__') and hasattr(iterable, '__getitem__'):
227        it = reversed(iterable)
228    else:
229        msg = 'Iterable is not reversible.'
230        raise TypeError(msg)
231
232    if initial is None:
233        try:
234            acc = cast(R, next(it))  # in this case R = D
235        except StopIteration:
236            return cast(S, default)  # if default = nothing, then S is Nothing
237    else:
238        acc = initial
239
240    for v in it:
241        acc = f(v, acc)
242
243    return acc
244
245# @overload
246# def foldLsc(iterable: Iterable[D|S], f: Callable[[D, D|S], D], sentinel: S) -> D|S: ...
247# @overload
248# def foldLsc(iterable: Iterable[D|S], f: Callable[[L, D|S], L], sentinel: S, initial: L) -> L: ...
249# @overload
250# def foldLsc(iterable: Iterable[D|S], f: Callable[[L, D|S], L], sentinel: S, initial: Optional[L]=None) -> L|Nothing: ...
251# @overload
252# def foldLsc(iterable: Iterable[D|S], f: Callable[[L, D|S], L],
253#           initial: Optional[L]=None, default: S|Nothing=nothing) -> L|S: ...
254# def foldLsc(iterable: Iterable[D|S], f: Callable[[L, D|S], L],
255#           sentinel: S, initial: Optional[L|S]=None) -> L|S:
256#     """Folds an iterable from the left with an optional initial value.
257# 
258#     * if the iterable returns the sentinel value, stop the fold at that point
259#     * if f returns the sentinel value, stop the fold at that point
260#     * f is never passed the sentinel value
261#     * note that _S can be the same type as _D
262#     * if iterable empty & no initial value given, return sentinel
263#     * note that when initial not given, then _L = _D
264#     * traditional FP type order given for function f
265#     * raises TypeError if the iterable is not iterable (for the benefit of untyped code)
266#     * never returns if iterable generates an infinite iterator & f never returns the sentinel value
267# 
268#     """
269#     acc: L|S
270#     if hasattr(iterable, '__iter__'):
271#         it = iter(iterable)
272#     else:
273#         msg = '"Iterable" is not iterable.'
274#         raise TypeError(msg)
275# 
276#     if initial == sentinel:
277#         return sentinel
278#     elif initial is None:
279#         try:
280#             acc = cast(L, next(it))
281#         except StopIteration:
282#             return sentinel
283#     else:
284#         acc = initial
285# 
286#     for v in it:
287#         if v == sentinel:
288#             break
289#         facc = f(cast(L, acc), v)                    # if not L = S
290#                                                      # then type(acc) is not S
291#         if facc == sentinel:
292#             break
293#         else:
294#             acc = facc
295#     return acc
def accumulate( iterable: Iterable[~D], f: Callable[[~L, ~D], ~L], initial: Optional[~L] = None) -> Iterator[~L]:
121def accumulate(iterable: Iterable[D], f: Callable[[L, D], L],
122               initial: Optional[L]=None) -> Iterator[L]:
123    """
124    #### Returns an iterator of accumulated values.
125
126    * pure Python version of standard library's itertools.accumulate
127    * function f does not default to addition (for typing flexibility)
128    * begins accumulation with an optional starting value
129    * itertools.accumulate has mypy issues
130
131    """
132    it = iter(iterable)
133    try:
134        it0 = next(it)
135    except StopIteration:
136        if initial is None:
137            return
138        else:
139            yield initial
140    else:
141        if initial is not None:
142            yield initial
143            acc = f(initial, it0)
144            for ii in it:
145                yield acc
146                acc = f(acc, ii)
147            yield acc
148        else:
149            acc = cast(L, it0)  # in this case L = D
150            for ii in it:
151                yield acc
152                acc = f(acc, ii)
153            yield acc

Returns an iterator of accumulated values.

  • pure Python version of standard library's itertools.accumulate
  • function f does not default to addition (for typing flexibility)
  • begins accumulation with an optional starting value
  • itertools.accumulate has mypy issues
def foldL( iterable: Iterable[~D], f: Callable[[~L, ~D], ~L], initial: Optional[~L] = None, default: Union[~S, grscheller.fp.nada.Nada] = nada) -> Union[~L, ~S, grscheller.fp.nada.Nada]:
168def foldL(iterable: Iterable[D], f: Callable[[L, D], L],
169          initial: Optional[L]=None, default: S|Nada=nada) -> L|S|Nada:
170    """
171    #### Folds iterable left with optional initial value.
172
173    * note that ~S can be the same type as ~L
174    * note that when an initial value is not given then ~L = ~D
175    * if iterable empty & no initial value given, return default
176    * traditional FP type order given for function f
177    * raises TypeError if the "iterable" is not iterable
178    * never returns if iterable generates an infinite iterator
179
180    """
181    acc: L
182    if hasattr(iterable, '__iter__'):
183        it = iter(iterable)
184    else:
185        msg = '"Iterable" is not iterable.'
186        raise TypeError(msg)
187
188    if initial is None:
189        try:
190            acc = cast(L, next(it))  # in this case L = D
191        except StopIteration:
192            return cast(S, default)  # if default = nothing, then S is Nothing
193    else:
194        acc = initial
195
196    for v in it:
197        acc = f(acc, v)
198
199    return acc

Folds iterable left with optional initial value.

  • note that ~S can be the same type as ~L
  • note that when an initial value is not given then ~L = ~D
  • if iterable empty & no initial value given, return default
  • traditional FP type order given for function f
  • raises TypeError if the "iterable" is not iterable
  • never returns if iterable generates an infinite iterator
def foldR( iterable: Reversible[~D], f: Callable[[~D, ~R], ~R], initial: Optional[~R] = None, default: Union[~S, grscheller.fp.nada.Nada] = nada) -> Union[~R, ~S, grscheller.fp.nada.Nada]:
214def foldR(iterable: Reversible[D], f: Callable[[D, R], R],
215          initial: Optional[R]=None, default: S|Nada=nada) -> R|S|Nada:
216    """
217    #### Folds reversible iterable right with an optional initial value.
218
219    * note that ~S can be the same type as ~R
220    * note that when an initial value not given then ~R = ~D
221    * if iterable empty & no initial value given, return default
222    * traditional FP type order given for function f
223    * raises TypeError if iterable is not reversible
224
225    """
226    acc: R
227    if hasattr(iterable, '__reversed__') or hasattr(iterable, '__len__') and hasattr(iterable, '__getitem__'):
228        it = reversed(iterable)
229    else:
230        msg = 'Iterable is not reversible.'
231        raise TypeError(msg)
232
233    if initial is None:
234        try:
235            acc = cast(R, next(it))  # in this case R = D
236        except StopIteration:
237            return cast(S, default)  # if default = nothing, then S is Nothing
238    else:
239        acc = initial
240
241    for v in it:
242        acc = f(v, acc)
243
244    return acc

Folds reversible iterable right with an optional initial value.

  • note that ~S can be the same type as ~R
  • note that when an initial value not given then ~R = ~D
  • if iterable empty & no initial value given, return default
  • traditional FP type order given for function f
  • raises TypeError if iterable is not reversible
def concat(*iterables: Iterable[~D]) -> Iterator[~D]:
47def concat(*iterables: Iterable[D]) -> Iterator[D]:
48    """
49    #### Sequentially concatenate multiple iterables together.
50
51    * pure Python version of standard library's itertools.chain
52    * iterator yields Sequentially each iterable until all are exhausted
53    * an infinite iterable will prevent subsequent iterables from yielding any values
54    * performant to chain
55
56    """
57    iterator: Iterator[D]
58    for iterator in map(lambda x: iter(x), iterables):
59        while True:
60            try:
61                value: D = next(iterator)
62                yield value
63            except StopIteration:
64                break

Sequentially concatenate multiple iterables together.

  • pure Python version of standard library's itertools.chain
  • iterator yields Sequentially each iterable until all are exhausted
  • an infinite iterable will prevent subsequent iterables from yielding any values
  • performant to chain
def merge(*iterables: Iterable[~D], yield_partials: bool = False) -> Iterator[~D]:
 94def merge(*iterables: Iterable[D], yield_partials: bool=False) -> Iterator[D]:
 95    """
 96    #### Shuffle together multiple iterables until one is exhausted.
 97
 98    * iterator yields until one of the iterables is exhausted
 99    * if yield_partials is true, yield any unmatched yielded values from the other iterables
100    * this prevents data lose if any of the iterables are iterators with external references
101
102    """
103    iterList = list(map(lambda x: iter(x), iterables))
104    if (numIters := len(iterList)) > 0:
105        values = []
106        while True:
107            try:
108                for ii in range(numIters):
109                    values.append(next(iterList[ii]))
110                for value in values:
111                    yield value
112                values.clear()
113            except StopIteration:
114                break
115        if yield_partials:
116            for value in values:
117                yield value

Shuffle together multiple iterables until one is exhausted.

  • iterator yields until one of the iterables is exhausted
  • if yield_partials is true, yield any unmatched yielded values from the other iterables
  • this prevents data lose if any of the iterables are iterators with external references
def exhaust(*iterables: Iterable[~D]) -> Iterator[~D]:
66def exhaust(*iterables: Iterable[D]) -> Iterator[D]:
67    """
68    #### Shuffle together multiple iterables until all are exhausted.
69
70    * iterator yields until all iterables are exhausted
71
72    """
73    iterList = list(map(lambda x: iter(x), iterables))
74    if (numIters := len(iterList)) > 0:
75        ii = 0
76        values = []
77        while True:
78            try:
79                while ii < numIters:
80                    values.append(next(iterList[ii]))
81                    ii += 1
82                for value in values:
83                    yield value
84                ii = 0
85                values.clear()
86            except StopIteration:
87                numIters -= 1
88                if numIters < 1:
89                    break
90                del iterList[ii]
91        for value in values:
92            yield value

Shuffle together multiple iterables until all are exhausted.

  • iterator yields until all iterables are exhausted
class FM(enum.Enum):
35class FM(Enum):
36    CONCAT = auto()
37    MERGE = auto()
38    EXHAUST = auto()
CONCAT = <FM.CONCAT: 1>
MERGE = <FM.MERGE: 2>
EXHAUST = <FM.EXHAUST: 3>
Inherited Members
enum.Enum
name
value