Module sertit.ci
CI tools
You can use assert_raster_equal()
only if you have installed sertit[full] or sertit[rasters]
Expand source code
# -*- coding: utf-8 -*-
# Copyright 2021, SERTIT-ICube - France, https://sertit.unistra.fr/
# This file is part of sertit-utils project
# https://github.com/sertit/sertit-utils
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
CI tools
You can use `assert_raster_equal` only if you have installed sertit[full] or sertit[rasters]
"""
import filecmp
import os
from doctest import Example
import geopandas as gpd
import numpy as np
import rasterio
from lxml import etree
from lxml.doctestcompare import LXMLOutputChecker
def get_mnt_path() -> str:
"""
Return mounting directory `/mnt`.
.. WARNING::
This won't work on Windows !
```python
>>> get_mnt_path()
'/mnt'
```
Returns:
str: Mounting directory
"""
return r"/mnt"
def _get_db_path(db_nb=2) -> str:
"""
Return mounted directory `/mnt/ds2_db2` which corresponds to `\\ds2\database02`.
.. WARNING::
Use it carefully (OK in CI) as this directory may not exist ! This won't work on Windows !
```python
>>> get_db_path(db_nb=2)
'/mnt/ds2_db2'
```
"""
db_path = f"{get_mnt_path()}/ds2_db{db_nb}"
if not os.path.isdir(db_path):
raise NotADirectoryError(f"Directory not found: {db_path}")
return db_path
def get_db2_path() -> str:
"""
Return mounted directory `/mnt/ds2_db2` which corresponds to `\\ds2\database02`.
.. WARNING::
Use it carefully (OK in CI) as this directory may not exist ! This won't work on Windows !
```python
>>> get_db2_path()
'/mnt/ds2_db2'
```
Returns:
str: Mounted directory
"""
return _get_db_path(2)
def get_db3_path() -> str:
"""
Return mounted directory `/mnt/ds2_db3` which corresponds to `\\ds2\database03`.
.. WARNING::
Use it carefully (OK in CI) as this directory may not exist ! This won't work on Windows !
```python
>>> get_db3_path()
'/mnt/ds2_db3'
```
Returns:
str: Mounted directory
"""
return _get_db_path(3)
def get_db4_path() -> str:
"""
Return mounted directory `/mnt/ds2_db4` which corresponds to `\\ds2\database04`.
.. WARNING::
Use it carefully (OK in CI) as this directory may not exist ! This won't work on Windows !
```python
>>> get_db4_path()
'/mnt/ds2_db4'
```
Returns:
str: Mounted directory
"""
return _get_db_path(4)
def assert_raster_equal(path_1: str, path_2: str) -> None:
"""
Assert that two rasters are equal.
-> Useful for pytests.
```python
>>> path = r"CI\DATA\rasters\raster.tif"
>>> assert_raster_equal(path, path)
>>> # Raises AssertionError if sth goes wrong
```
Args:
path_1 (str): Raster 1
path_2 (str): Raster 2
"""
with rasterio.open(path_1) as dst_1:
with rasterio.open(path_2) as dst_2:
assert dst_1.meta == dst_2.meta
np.testing.assert_array_equal(dst_1.read(), dst_2.read())
def assert_raster_almost_equal(path_1: str, path_2: str, decimal=7) -> None:
"""
Assert that two rasters are almost equal.
(everything is equal except the transform and the arrays that are almost equal)
Accepts an offset of `1E{decimal}` on the array and the transform
-> Useful for pytests.
```python
>>> path = r"CI\DATA\rasters\raster.tif"
>>> path2 = r"CI\DATA\rasters\raster_almost.tif"
>>> assert_raster_equal(path, path2)
>>> # Raises AssertionError if sth goes wrong
```
Args:
path_1 (str): Raster 1
path_2 (str): Raster 2
"""
with rasterio.open(path_1) as dst_1:
with rasterio.open(path_2) as dst_2:
assert dst_1.meta["driver"] == dst_2.meta["driver"]
assert dst_1.meta["dtype"] == dst_2.meta["dtype"]
assert dst_1.meta["nodata"] == dst_2.meta["nodata"]
assert dst_1.meta["width"] == dst_2.meta["width"]
assert dst_1.meta["height"] == dst_2.meta["height"]
assert dst_1.meta["count"] == dst_2.meta["count"]
assert dst_1.meta["crs"] == dst_2.meta["crs"]
dst_1.meta["transform"].almost_equals(
dst_1.meta["transform"], precision=float(f"1E-{decimal}")
)
np.testing.assert_almost_equal(dst_1.read(), dst_2.read(), decimal=decimal)
def assert_dir_equal(path_1: str, path_2: str) -> None:
"""
Assert that two directories are equal.
# Useful for pytests.
```python
>>> path = r"CI\DATA\rasters"
>>> assert_dir_equal(path, path)
>>> # Raises AssertionError if sth goes wrong
```
Args:
path_1 (str): Directory 1
path_2 (str): Directory 2
"""
dcmp = filecmp.dircmp(path_1, path_2)
assert os.path.isdir(path_1)
assert os.path.isdir(path_2)
assert dcmp.left_only == []
assert dcmp.right_only == []
# def assert_archive_equal(path_1: str, path_2: str) -> None:
# """
# Assert that two archives are equal, by creating hashes that should be equal
#
# Args:
# path_1 (str): Archive 1
# path_2 (str): Archive 2
# """
# filecmp.cmp(path_1, path_2)
#
# file_1 = hashlib.sha256(open(path_1, 'rb').read()).digest()
# file_2 = hashlib.sha256(open(path_2, 'rb').read()).digest()
# assert file_1 == file_2
def assert_geom_equal(geom_1: gpd.GeoDataFrame, geom_2: gpd.GeoDataFrame) -> None:
"""
Assert that two geometries are equal
(do not check equality between geodataframe as they may differ on other fields).
-> Useful for pytests.
```python
>>> path = r"CI\DATA\vectors\aoi.geojson"
>>> assert_geom_equal(path, path)
>>> # Raises AssertionError if sth goes wrong
```
.. WARNING::
Only checks:
- valid geometries
- length of GeoDataFrame
- CRS
Args:
geom_1 (gpd.GeoDataFrame): Geometry 1
geom_2 (gpd.GeoDataFrame): Geometry 2
"""
assert len(geom_1) == len(geom_2)
assert geom_1.crs == geom_2.crs
for idx in range(len(geom_1)):
if geom_1.geometry.iat[idx].is_valid and geom_2.geometry.iat[idx].is_valid:
# If valid geometries, assert that the both are equal
assert geom_1.geometry.iat[idx].equals(geom_2.geometry.iat[idx])
def assert_xml_equal(xml_elem_1: etree._Element, xml_elem_2: etree._Element) -> None:
"""
Assert that 2 XML (as etree Elements) are equal.
-> Useful for pytests.
Args:
xml_elem_1 (etree._Element): 1st Element
xml_elem_2 (etree._Element): 2nd Element
"""
str_1 = etree.tounicode(xml_elem_1)
str_2 = etree.tounicode(xml_elem_2)
checker = LXMLOutputChecker()
if not checker.check_output(str_1, str_2, 0):
message = checker.output_difference(Example("", str_1), str_2, 0)
raise AssertionError(message)
Functions
def get_mnt_path(
)
-
Return mounting directory
/mnt
.Warning
This won't work on Windows !
>>> get_mnt_path() '/mnt'
Returns
str
- Mounting directory
Expand source code
def get_mnt_path() -> str: """ Return mounting directory `/mnt`. .. WARNING:: This won't work on Windows ! ```python >>> get_mnt_path() '/mnt' ``` Returns: str: Mounting directory """ return r"/mnt"
def get_db2_path(
)
-
Return mounted directory
/mnt/ds2_db2
which corresponds to\ds2\database02
.Warning
Use it carefully (OK in CI) as this directory may not exist ! This won't work on Windows !
>>> get_db2_path() '/mnt/ds2_db2'
Returns
str
- Mounted directory
Expand source code
def get_db2_path() -> str: """ Return mounted directory `/mnt/ds2_db2` which corresponds to `\\ds2\database02`. .. WARNING:: Use it carefully (OK in CI) as this directory may not exist ! This won't work on Windows ! ```python >>> get_db2_path() '/mnt/ds2_db2' ``` Returns: str: Mounted directory """ return _get_db_path(2)
def get_db3_path(
)
-
Return mounted directory
/mnt/ds2_db3
which corresponds to\ds2\database03
.Warning
Use it carefully (OK in CI) as this directory may not exist ! This won't work on Windows !
>>> get_db3_path() '/mnt/ds2_db3'
Returns
str
- Mounted directory
Expand source code
def get_db3_path() -> str: """ Return mounted directory `/mnt/ds2_db3` which corresponds to `\\ds2\database03`. .. WARNING:: Use it carefully (OK in CI) as this directory may not exist ! This won't work on Windows ! ```python >>> get_db3_path() '/mnt/ds2_db3' ``` Returns: str: Mounted directory """ return _get_db_path(3)
def get_db4_path(
)
-
Return mounted directory
/mnt/ds2_db4
which corresponds to\ds2\database04
.Warning
Use it carefully (OK in CI) as this directory may not exist ! This won't work on Windows !
>>> get_db4_path() '/mnt/ds2_db4'
Returns
str
- Mounted directory
Expand source code
def get_db4_path() -> str: """ Return mounted directory `/mnt/ds2_db4` which corresponds to `\\ds2\database04`. .. WARNING:: Use it carefully (OK in CI) as this directory may not exist ! This won't work on Windows ! ```python >>> get_db4_path() '/mnt/ds2_db4' ``` Returns: str: Mounted directory """ return _get_db_path(4)
def assert_raster_equal(
path_1,
path_2)-
Assert that two rasters are equal.
-> Useful for pytests.
>>> path = r"CI\DATA asters aster.tif" >>> assert_raster_equal(path, path) >>> # Raises AssertionError if sth goes wrong
Args
path_1
:str
- Raster 1
path_2
:str
- Raster 2
Expand source code
def assert_raster_equal(path_1: str, path_2: str) -> None: """ Assert that two rasters are equal. -> Useful for pytests. ```python >>> path = r"CI\DATA\rasters\raster.tif" >>> assert_raster_equal(path, path) >>> # Raises AssertionError if sth goes wrong ``` Args: path_1 (str): Raster 1 path_2 (str): Raster 2 """ with rasterio.open(path_1) as dst_1: with rasterio.open(path_2) as dst_2: assert dst_1.meta == dst_2.meta np.testing.assert_array_equal(dst_1.read(), dst_2.read())
def assert_raster_almost_equal(
path_1,
path_2,
decimal=7)-
Assert that two rasters are almost equal. (everything is equal except the transform and the arrays that are almost equal)
Accepts an offset of
1E{decimal}
on the array and the transform-> Useful for pytests.
>>> path = r"CI\DATA asters aster.tif" >>> path2 = r"CI\DATA asters aster_almost.tif" >>> assert_raster_equal(path, path2) >>> # Raises AssertionError if sth goes wrong
Args
path_1
:str
- Raster 1
path_2
:str
- Raster 2
Expand source code
def assert_raster_almost_equal(path_1: str, path_2: str, decimal=7) -> None: """ Assert that two rasters are almost equal. (everything is equal except the transform and the arrays that are almost equal) Accepts an offset of `1E{decimal}` on the array and the transform -> Useful for pytests. ```python >>> path = r"CI\DATA\rasters\raster.tif" >>> path2 = r"CI\DATA\rasters\raster_almost.tif" >>> assert_raster_equal(path, path2) >>> # Raises AssertionError if sth goes wrong ``` Args: path_1 (str): Raster 1 path_2 (str): Raster 2 """ with rasterio.open(path_1) as dst_1: with rasterio.open(path_2) as dst_2: assert dst_1.meta["driver"] == dst_2.meta["driver"] assert dst_1.meta["dtype"] == dst_2.meta["dtype"] assert dst_1.meta["nodata"] == dst_2.meta["nodata"] assert dst_1.meta["width"] == dst_2.meta["width"] assert dst_1.meta["height"] == dst_2.meta["height"] assert dst_1.meta["count"] == dst_2.meta["count"] assert dst_1.meta["crs"] == dst_2.meta["crs"] dst_1.meta["transform"].almost_equals( dst_1.meta["transform"], precision=float(f"1E-{decimal}") ) np.testing.assert_almost_equal(dst_1.read(), dst_2.read(), decimal=decimal)
def assert_dir_equal(
path_1,
path_2)-
Assert that two directories are equal.
Useful for pytests.
>>> path = r"CI\DATA asters" >>> assert_dir_equal(path, path) >>> # Raises AssertionError if sth goes wrong
Args
path_1
:str
- Directory 1
path_2
:str
- Directory 2
Expand source code
def assert_dir_equal(path_1: str, path_2: str) -> None: """ Assert that two directories are equal. # Useful for pytests. ```python >>> path = r"CI\DATA\rasters" >>> assert_dir_equal(path, path) >>> # Raises AssertionError if sth goes wrong ``` Args: path_1 (str): Directory 1 path_2 (str): Directory 2 """ dcmp = filecmp.dircmp(path_1, path_2) assert os.path.isdir(path_1) assert os.path.isdir(path_2) assert dcmp.left_only == [] assert dcmp.right_only == []
def assert_geom_equal(
geom_1,
geom_2)-
Assert that two geometries are equal (do not check equality between geodataframe as they may differ on other fields).
-> Useful for pytests.
>>> path = r"CI\DATAectorsoi.geojson" >>> assert_geom_equal(path, path) >>> # Raises AssertionError if sth goes wrong
Warning
Only checks: - valid geometries - length of GeoDataFrame - CRS
Args
geom_1
:gpd.GeoDataFrame
- Geometry 1
geom_2
:gpd.GeoDataFrame
- Geometry 2
Expand source code
def assert_geom_equal(geom_1: gpd.GeoDataFrame, geom_2: gpd.GeoDataFrame) -> None: """ Assert that two geometries are equal (do not check equality between geodataframe as they may differ on other fields). -> Useful for pytests. ```python >>> path = r"CI\DATA\vectors\aoi.geojson" >>> assert_geom_equal(path, path) >>> # Raises AssertionError if sth goes wrong ``` .. WARNING:: Only checks: - valid geometries - length of GeoDataFrame - CRS Args: geom_1 (gpd.GeoDataFrame): Geometry 1 geom_2 (gpd.GeoDataFrame): Geometry 2 """ assert len(geom_1) == len(geom_2) assert geom_1.crs == geom_2.crs for idx in range(len(geom_1)): if geom_1.geometry.iat[idx].is_valid and geom_2.geometry.iat[idx].is_valid: # If valid geometries, assert that the both are equal assert geom_1.geometry.iat[idx].equals(geom_2.geometry.iat[idx])
def assert_xml_equal(
xml_elem_1,
xml_elem_2)-
Assert that 2 XML (as etree Elements) are equal.
-> Useful for pytests.
Args
xml_elem_1
:etree._Element
- 1st Element
xml_elem_2
:etree._Element
- 2nd Element
Expand source code
def assert_xml_equal(xml_elem_1: etree._Element, xml_elem_2: etree._Element) -> None: """ Assert that 2 XML (as etree Elements) are equal. -> Useful for pytests. Args: xml_elem_1 (etree._Element): 1st Element xml_elem_2 (etree._Element): 2nd Element """ str_1 = etree.tounicode(xml_elem_1) str_2 = etree.tounicode(xml_elem_2) checker = LXMLOutputChecker() if not checker.check_output(str_1, str_2, 0): message = checker.output_difference(Example("", str_1), str_2, 0) raise AssertionError(message)