grscheller.fp.iterables

  • iterables are not necessarily iterators
  • at all times iterator protocol is assumed to be followed, that is
    • for all iterators foo we assume iter(foo) is foo
    • all iterators are assumed to be iterable
  1# Copyright 2023-2024 Geoffrey R. Scheller
  2#
  3# Licensed under the Apache License, Version 2.0 (the "License");
  4# you may not use this file except in compliance with the License.
  5# You may obtain a copy of the License at
  6#
  7#     http://www.apache.org/licenses/LICENSE-2.0
  8#
  9# Unless required by applicable law or agreed to in writing, software
 10# distributed under the License is distributed on an "AS IS" BASIS,
 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12# See the License for the specific language governing permissions and
 13# limitations under the License.
 14
 15"""
 16#### Library of iterator related functions.
 17
 18* iterables are not necessarily iterators
 19* at all times iterator protocol is assumed to be followed, that is
 20  * for all iterators `foo` we assume `iter(foo) is foo`
 21  * all iterators are assumed to be iterable
 22
 23"""
 24
 25from __future__ import annotations
 26from typing import Callable, cast, Final, Iterator, Iterable
 27from typing import overload, Optional, Reversible, TypeVar
 28from .nada import Nada, nada
 29
 30__all__ = [ 'concat', 'merge', 'exhaust', 'foldL', 'foldR', 'accumulate' ]
 31
 32D = TypeVar('D')
 33L = TypeVar('L')
 34R = TypeVar('R')
 35S = TypeVar('S')
 36
 37## Iterate over multiple Iterables
 38
 39def concat(*iterables: Iterable[D]) -> Iterator[D]:
 40    """
 41    #### Sequentially concatenate multiple iterables together.
 42
 43    * pure Python version of standard library's itertools.chain
 44    * iterator yields Sequentially each iterable until all are exhausted
 45    * an infinite iterable will prevent subsequent iterables from yielding any values
 46    * performant to chain
 47
 48    """
 49    iterator: Iterator[D]
 50    for iterator in map(lambda x: iter(x), iterables):
 51        while True:
 52            try:
 53                value: D = next(iterator)
 54                yield value
 55            except StopIteration:
 56                break
 57
 58def exhaust(*iterables: Iterable[D]) -> Iterator[D]:
 59    """
 60    #### Shuffle together multiple iterables until all are exhausted.
 61
 62    * iterator yields until all iterables are exhausted
 63
 64    """
 65    iterList = list(map(lambda x: iter(x), iterables))
 66    if (numIters := len(iterList)) > 0:
 67        ii = 0
 68        values = []
 69        while True:
 70            try:
 71                while ii < numIters:
 72                    values.append(next(iterList[ii]))
 73                    ii += 1
 74                for value in values:
 75                    yield value
 76                ii = 0
 77                values.clear()
 78            except StopIteration:
 79                numIters -= 1
 80                if numIters < 1:
 81                    break
 82                del iterList[ii]
 83        for value in values:
 84            yield value
 85
 86def merge(*iterables: Iterable[D], yield_partials: bool=False) -> Iterator[D]:
 87    """
 88    #### Shuffle together multiple iterables until one is exhausted.
 89
 90    * iterator yields until one of the iterables is exhausted
 91    * if yield_partials is true, yield any unmatched yielded values from the other iterables
 92    * this prevents data lose if any of the iterables are iterators with external references
 93
 94    """
 95    iterList = list(map(lambda x: iter(x), iterables))
 96    if (numIters := len(iterList)) > 0:
 97        values = []
 98        while True:
 99            try:
100                for ii in range(numIters):
101                    values.append(next(iterList[ii]))
102                for value in values:
103                    yield value
104                values.clear()
105            except StopIteration:
106                break
107        if yield_partials:
108            for value in values:
109                yield value
110
111## reducing and accumulating
112
113@overload
114def foldL(iterable: Iterable[D], f: Callable[[L, D], L], initial: Optional[L], default: S) -> L|S:
115    ...
116@overload
117def foldL(iterable: Iterable[D], f: Callable[[D, D], D]) -> D|Nada:
118    ...
119@overload
120def foldL(iterable: Iterable[D], f: Callable[[L, D], L], initial: L) -> L:
121    ...
122@overload
123def foldL(iterable: Iterable[D], f: Callable[[L, D], L], initial: Nada) -> Nada:
124    ...
125
126def foldL(iterable: Iterable[D], f: Callable[[L, D], L],
127          initial: Optional[L]=None, default: S|Nada=nada) -> L|S|Nada:
128    """
129    #### Folds iterable left with optional initial value.
130
131    * note that ~S can be the same type as ~L
132    * note that when an initial value is not given then ~L = ~D
133    * if iterable empty & no initial value given, return default
134    * traditional FP type order given for function f
135    * raises TypeError if the "iterable" is not iterable
136    * never returns if iterable generates an infinite iterator
137
138    """
139    acc: L
140    if hasattr(iterable, '__iter__'):
141        it = iter(iterable)
142    else:
143        msg = '"Iterable" is not iterable.'
144        raise TypeError(msg)
145
146    if initial is None:
147        try:
148            acc = cast(L, next(it))  # in this case L = D
149        except StopIteration:
150            return cast(S, default)  # if default = nothing, then S is Nothing
151    else:
152        acc = initial
153
154    for v in it:
155        acc = f(acc, v)
156
157    return acc
158
159@overload
160def foldR(iterable: Reversible[D], f: Callable[[D, R], R], initial: Optional[R], default: S) -> R|S:
161    ...
162@overload
163def foldR(iterable: Reversible[D], f: Callable[[D, D], D]) -> D|Nothing:
164    ...
165@overload
166def foldR(iterable: Reversible[D], f: Callable[[D, R], R], initial: R) -> R:
167    ...
168@overload
169def foldR(iterable: Reversible[D], f: Callable[[D, R], R], initial: Nada) -> R|Nada:
170    ...
171
172def foldR(iterable: Reversible[D], f: Callable[[D, R], R],
173          initial: Optional[R]=None, default: S|Nada=nada) -> R|S|Nada:
174    """
175    #### Folds reversible iterable right with an optional initial value.
176
177    * note that ~S can be the same type as ~R
178    * note that when an initial value not given then ~R = ~D
179    * if iterable empty & no initial value given, return default
180    * traditional FP type order given for function f
181    * raises TypeError if iterable is not reversible
182
183    """
184    acc: R
185    if hasattr(iterable, '__reversed__') or hasattr(iterable, '__len__') and hasattr(iterable, '__getitem__'):
186        it = reversed(iterable)
187    else:
188        msg = 'Iterable is not reversible.'
189        raise TypeError(msg)
190
191    if initial is None:
192        try:
193            acc = cast(R, next(it))  # in this case R = D
194        except StopIteration:
195            return cast(S, default)  # if default = nothing, then S is Nothing
196    else:
197        acc = initial
198
199    for v in it:
200        acc = f(v, acc)
201
202    return acc
203
204# @overload
205# def foldLsc(iterable: Iterable[D|S], f: Callable[[D, D|S], D], sentinel: S) -> D|S: ...
206# @overload
207# def foldLsc(iterable: Iterable[D|S], f: Callable[[L, D|S], L], sentinel: S, initial: L) -> L: ...
208# @overload
209# def foldLsc(iterable: Iterable[D|S], f: Callable[[L, D|S], L], sentinel: S, initial: Optional[L]=None) -> L|Nothing: ...
210# @overload
211# def foldLsc(iterable: Iterable[D|S], f: Callable[[L, D|S], L],
212#           initial: Optional[L]=None, default: S|Nothing=nothing) -> L|S: ...
213# def foldLsc(iterable: Iterable[D|S], f: Callable[[L, D|S], L],
214#           sentinel: S, initial: Optional[L|S]=None) -> L|S:
215#     """Folds an iterable from the left with an optional initial value.
216# 
217#     * if the iterable returns the sentinel value, stop the fold at that point
218#     * if f returns the sentinel value, stop the fold at that point
219#     * f is never passed the sentinel value
220#     * note that _S can be the same type as _D
221#     * if iterable empty & no initial value given, return sentinel
222#     * note that when initial not given, then _L = _D
223#     * traditional FP type order given for function f
224#     * raises TypeError if the iterable is not iterable (for the benefit of untyped code)
225#     * never returns if iterable generates an infinite iterator & f never returns the sentinel value
226# 
227#     """
228#     acc: L|S
229#     if hasattr(iterable, '__iter__'):
230#         it = iter(iterable)
231#     else:
232#         msg = '"Iterable" is not iterable.'
233#         raise TypeError(msg)
234# 
235#     if initial == sentinel:
236#         return sentinel
237#     elif initial is None:
238#         try:
239#             acc = cast(L, next(it))
240#         except StopIteration:
241#             return sentinel
242#     else:
243#         acc = initial
244# 
245#     for v in it:
246#         if v == sentinel:
247#             break
248#         facc = f(cast(L, acc), v)                    # if not L = S
249#                                                      # then type(acc) is not S
250#         if facc == sentinel:
251#             break
252#         else:
253#             acc = facc
254#     return acc
255
256def accumulate(iterable: Iterable[D], f: Callable[[L, D], L],
257               initial: Optional[L]=None) -> Iterator[L]:
258    """
259    #### Returns an iterator of accumulated values.
260
261    * pure Python version of standard library's itertools.accumulate
262    * function f does not default to addition (for typing flexibility)
263    * begins accumulation with an optional starting value
264    * itertools.accumulate has mypy issues
265
266    """
267    it = iter(iterable)
268    try:
269        it0 = next(it)
270    except StopIteration:
271        if initial is None:
272            return
273        else:
274            yield initial
275    else:
276        if initial is not None:
277            yield initial
278            acc = f(initial, it0)
279            for ii in it:
280                yield acc
281                acc = f(acc, ii)
282            yield acc
283        else:
284            acc = cast(L, it0)  # in this case L = D
285            for ii in it:
286                yield acc
287                acc = f(acc, ii)
288            yield acc
def concat(*iterables: Iterable[~D]) -> Iterator[~D]:
40def concat(*iterables: Iterable[D]) -> Iterator[D]:
41    """
42    #### Sequentially concatenate multiple iterables together.
43
44    * pure Python version of standard library's itertools.chain
45    * iterator yields Sequentially each iterable until all are exhausted
46    * an infinite iterable will prevent subsequent iterables from yielding any values
47    * performant to chain
48
49    """
50    iterator: Iterator[D]
51    for iterator in map(lambda x: iter(x), iterables):
52        while True:
53            try:
54                value: D = next(iterator)
55                yield value
56            except StopIteration:
57                break

Sequentially concatenate multiple iterables together.

  • pure Python version of standard library's itertools.chain
  • iterator yields Sequentially each iterable until all are exhausted
  • an infinite iterable will prevent subsequent iterables from yielding any values
  • performant to chain
def merge(*iterables: Iterable[~D], yield_partials: bool = False) -> Iterator[~D]:
 87def merge(*iterables: Iterable[D], yield_partials: bool=False) -> Iterator[D]:
 88    """
 89    #### Shuffle together multiple iterables until one is exhausted.
 90
 91    * iterator yields until one of the iterables is exhausted
 92    * if yield_partials is true, yield any unmatched yielded values from the other iterables
 93    * this prevents data lose if any of the iterables are iterators with external references
 94
 95    """
 96    iterList = list(map(lambda x: iter(x), iterables))
 97    if (numIters := len(iterList)) > 0:
 98        values = []
 99        while True:
100            try:
101                for ii in range(numIters):
102                    values.append(next(iterList[ii]))
103                for value in values:
104                    yield value
105                values.clear()
106            except StopIteration:
107                break
108        if yield_partials:
109            for value in values:
110                yield value

Shuffle together multiple iterables until one is exhausted.

  • iterator yields until one of the iterables is exhausted
  • if yield_partials is true, yield any unmatched yielded values from the other iterables
  • this prevents data lose if any of the iterables are iterators with external references
def exhaust(*iterables: Iterable[~D]) -> Iterator[~D]:
59def exhaust(*iterables: Iterable[D]) -> Iterator[D]:
60    """
61    #### Shuffle together multiple iterables until all are exhausted.
62
63    * iterator yields until all iterables are exhausted
64
65    """
66    iterList = list(map(lambda x: iter(x), iterables))
67    if (numIters := len(iterList)) > 0:
68        ii = 0
69        values = []
70        while True:
71            try:
72                while ii < numIters:
73                    values.append(next(iterList[ii]))
74                    ii += 1
75                for value in values:
76                    yield value
77                ii = 0
78                values.clear()
79            except StopIteration:
80                numIters -= 1
81                if numIters < 1:
82                    break
83                del iterList[ii]
84        for value in values:
85            yield value

Shuffle together multiple iterables until all are exhausted.

  • iterator yields until all iterables are exhausted
def foldL( iterable: Iterable[~D], f: Callable[[~L, ~D], ~L], initial: Optional[~L] = None, default: Union[~S, grscheller.fp.nada.Nada] = nada) -> Union[~L, ~S, grscheller.fp.nada.Nada]:
127def foldL(iterable: Iterable[D], f: Callable[[L, D], L],
128          initial: Optional[L]=None, default: S|Nada=nada) -> L|S|Nada:
129    """
130    #### Folds iterable left with optional initial value.
131
132    * note that ~S can be the same type as ~L
133    * note that when an initial value is not given then ~L = ~D
134    * if iterable empty & no initial value given, return default
135    * traditional FP type order given for function f
136    * raises TypeError if the "iterable" is not iterable
137    * never returns if iterable generates an infinite iterator
138
139    """
140    acc: L
141    if hasattr(iterable, '__iter__'):
142        it = iter(iterable)
143    else:
144        msg = '"Iterable" is not iterable.'
145        raise TypeError(msg)
146
147    if initial is None:
148        try:
149            acc = cast(L, next(it))  # in this case L = D
150        except StopIteration:
151            return cast(S, default)  # if default = nothing, then S is Nothing
152    else:
153        acc = initial
154
155    for v in it:
156        acc = f(acc, v)
157
158    return acc

Folds iterable left with optional initial value.

  • note that ~S can be the same type as ~L
  • note that when an initial value is not given then ~L = ~D
  • if iterable empty & no initial value given, return default
  • traditional FP type order given for function f
  • raises TypeError if the "iterable" is not iterable
  • never returns if iterable generates an infinite iterator
def foldR( iterable: Reversible[~D], f: Callable[[~D, ~R], ~R], initial: Optional[~R] = None, default: Union[~S, grscheller.fp.nada.Nada] = nada) -> Union[~R, ~S, grscheller.fp.nada.Nada]:
173def foldR(iterable: Reversible[D], f: Callable[[D, R], R],
174          initial: Optional[R]=None, default: S|Nada=nada) -> R|S|Nada:
175    """
176    #### Folds reversible iterable right with an optional initial value.
177
178    * note that ~S can be the same type as ~R
179    * note that when an initial value not given then ~R = ~D
180    * if iterable empty & no initial value given, return default
181    * traditional FP type order given for function f
182    * raises TypeError if iterable is not reversible
183
184    """
185    acc: R
186    if hasattr(iterable, '__reversed__') or hasattr(iterable, '__len__') and hasattr(iterable, '__getitem__'):
187        it = reversed(iterable)
188    else:
189        msg = 'Iterable is not reversible.'
190        raise TypeError(msg)
191
192    if initial is None:
193        try:
194            acc = cast(R, next(it))  # in this case R = D
195        except StopIteration:
196            return cast(S, default)  # if default = nothing, then S is Nothing
197    else:
198        acc = initial
199
200    for v in it:
201        acc = f(v, acc)
202
203    return acc

Folds reversible iterable right with an optional initial value.

  • note that ~S can be the same type as ~R
  • note that when an initial value not given then ~R = ~D
  • if iterable empty & no initial value given, return default
  • traditional FP type order given for function f
  • raises TypeError if iterable is not reversible
def accumulate( iterable: Iterable[~D], f: Callable[[~L, ~D], ~L], initial: Optional[~L] = None) -> Iterator[~L]:
257def accumulate(iterable: Iterable[D], f: Callable[[L, D], L],
258               initial: Optional[L]=None) -> Iterator[L]:
259    """
260    #### Returns an iterator of accumulated values.
261
262    * pure Python version of standard library's itertools.accumulate
263    * function f does not default to addition (for typing flexibility)
264    * begins accumulation with an optional starting value
265    * itertools.accumulate has mypy issues
266
267    """
268    it = iter(iterable)
269    try:
270        it0 = next(it)
271    except StopIteration:
272        if initial is None:
273            return
274        else:
275            yield initial
276    else:
277        if initial is not None:
278            yield initial
279            acc = f(initial, it0)
280            for ii in it:
281                yield acc
282                acc = f(acc, ii)
283            yield acc
284        else:
285            acc = cast(L, it0)  # in this case L = D
286            for ii in it:
287                yield acc
288                acc = f(acc, ii)
289            yield acc

Returns an iterator of accumulated values.

  • pure Python version of standard library's itertools.accumulate
  • function f does not default to addition (for typing flexibility)
  • begins accumulation with an optional starting value
  • itertools.accumulate has mypy issues