tests.test_events

  1import os
  2import time
  3
  4import cv2
  5import numpy as np
  6import pandas as pd
  7
  8from csi_images.csi_scans import Scan
  9from csi_images.csi_tiles import Tile
 10from csi_images.csi_events import Event, EventArray
 11
 12if os.environ.get("DEBIAN_FRONTEND") == "noninteractive":
 13    SHOW_PLOTS = False
 14else:
 15    # Change this to your preference for local testing, but commit as True
 16    SHOW_PLOTS = False
 17
 18
 19def test_getting_event():
 20    scan = Scan.load_txt("tests/data")
 21    tile = Tile(scan, 1000)
 22    event = Event(
 23        scan,
 24        tile,
 25        515,
 26        411,
 27    )
 28    images = event.extract_images()
 29    assert len(images) == 4
 30    images = event.extract_images(crop_size=100, in_pixels=True)
 31    assert images[0].shape == (100, 100)
 32    images = event.extract_images(crop_size=50, in_pixels=True)
 33    assert images[0].shape == (50, 50)
 34
 35    if SHOW_PLOTS:
 36        for image in images:
 37            cv2.imshow("Bright DAPI event in the center", image)
 38            cv2.waitKey(0)
 39        cv2.destroyAllWindows()
 40
 41    # Test a corner event
 42    event = Event(
 43        scan,
 44        tile,
 45        2,
 46        1000,
 47    )
 48    images = event.extract_images()
 49    assert len(images) == 4
 50    images = event.extract_images(crop_size=100, in_pixels=True)
 51    assert images[0].shape == (100, 100)
 52    images = event.extract_images(crop_size=200, in_pixels=True)
 53    assert images[0].shape == (200, 200)
 54
 55    if SHOW_PLOTS:
 56        for image in images:
 57            cv2.imshow("Events in the corner of a tile", image)
 58            cv2.waitKey(0)
 59        cv2.destroyAllWindows()
 60
 61
 62def test_getting_many_events():
 63    scan = Scan.load_txt("tests/data")
 64    tile = Tile(scan, 1000)
 65    tile2 = Tile(scan, 0)
 66    events = [
 67        Event(scan, tile, 515, 411),
 68        Event(scan, tile2, 2, 1000),
 69        Event(scan, tile, 1000, 1000),
 70        Event(scan, tile, 87, 126),
 71        Event(scan, tile, 1000, 2),
 72        Event(scan, tile2, 800, 800),
 73        Event(scan, tile, 1000, 662),
 74    ]
 75    # Test time to extract images sequentially
 76    start_time = time.time()
 77    images_1 = []
 78    for event in events:
 79        images_1.append(event.extract_images())
 80    sequential_time = time.time() - start_time
 81
 82    # Test time to extract images in parallel
 83    start_time = time.time()
 84    images_2 = Event.extract_images_for_list(events, crop_size=100)
 85    parallel_time = time.time() - start_time
 86    assert parallel_time < sequential_time
 87    for list_a, list_b in zip(images_1, images_2):
 88        assert len(list_a) == len(list_b)
 89        for image_a, image_b in zip(list_a, list_b):
 90            assert np.array_equal(image_a, image_b)
 91
 92    # Test that it works after converting to EventArray and back
 93    event_array = EventArray.from_events(events)
 94    remade_events = event_array.to_events(
 95        [scan], ignore_metadata=True, ignore_features=True
 96    )
 97    images_3 = Event.extract_images_for_list(remade_events, crop_size=100)
 98    for list_a, list_b in zip(images_1, images_3):
 99        assert len(list_a) == len(list_b)
100        for image_a, image_b in zip(list_a, list_b):
101            assert np.array_equal(image_a, image_b)
102
103
104def test_event_coordinates_for_bzscanner():
105    scan = Scan.load_txt("tests/data")
106    # Origin
107    tile = Tile(scan, 0)
108    event = Event(scan, tile, 0, 0)
109    scan_origin = event.get_scan_position()
110    assert 2500 <= scan_origin[0] <= 3500
111    assert 2500 <= scan_origin[1] <= 3500
112    scan_origin_on_slide = event.get_slide_position()
113    assert 71500 <= scan_origin_on_slide[0] <= 72500
114    assert 21500 <= scan_origin_on_slide[1] <= 22500
115    # Within the same tile, "bottom-right corner"
116    event = Event(scan, tile, 1000, 1000)
117    scan_position = event.get_scan_position()
118    assert scan_origin[0] <= scan_position[0]
119    assert scan_origin[1] <= scan_position[1]
120    slide_position = event.get_slide_position()
121    assert slide_position[0] <= scan_origin_on_slide[0]
122    assert slide_position[1] <= scan_origin_on_slide[1]
123
124    # Next row, opposite side
125    tile = Tile(scan, (scan.roi[0].tile_cols - 1, 1))
126    event = Event(scan, tile, 1000, 1000)
127    scan_position = event.get_scan_position()
128    assert scan_origin[0] <= scan_position[0]
129    assert scan_origin[1] <= scan_position[1]
130    slide_position = event.get_slide_position()
131    assert slide_position[0] <= scan_origin_on_slide[0]
132    assert slide_position[1] <= scan_origin_on_slide[1]
133
134    # Opposite corner
135    tile = Tile(scan, (scan.roi[0].tile_cols - 1, scan.roi[0].tile_rows - 1))
136    event = Event(scan, tile, 1361, 1003)
137    scan_position = event.get_scan_position()
138    assert 21500 <= scan_position[0] <= 22500
139    assert 58500 <= scan_position[1] <= 60500
140    slide_position = event.get_slide_position()
141    assert 14500 <= slide_position[0] <= 15500
142    assert 2500 <= slide_position[1] <= 3500
143
144
145def test_event_coordinates_for_axioscan():
146    scan = Scan.load_yaml("tests/data")
147    # Origin
148    tile = Tile(scan, 0)
149    event = Event(scan, tile, 0, 0)
150    scan_position = event.get_scan_position()
151    assert -59000 <= scan_position[0] < -55000
152    assert 0 <= scan_position[1] < 4000
153    slide_position = event.get_slide_position()
154    assert 16000 <= slide_position[0] < 20000
155    assert scan_position[1] == slide_position[1]
156
157    # Opposite corner
158    tile = Tile(scan, (scan.roi[0].tile_cols - 1, scan.roi[0].tile_rows - 1))
159    event = Event(scan, tile, 2000, 2000)
160    scan_position = event.get_scan_position()
161    assert -4000 <= scan_position[0] <= 0
162    assert 21000 <= scan_position[1] <= 25000
163    slide_position = event.get_slide_position()
164    assert 71000 <= slide_position[0] <= 75000
165    assert scan_position[1] == slide_position[1]
166
167
168def test_eventarray_conversions():
169    scan = Scan.load_yaml("tests/data")
170    # Origin
171    tile = Tile(scan, 0)
172    event0 = Event(scan, tile, 0, 0)
173    event1 = Event(scan, tile, 1000, 1000)
174    event2 = Event(scan, tile, 2000, 2000)
175
176    event_array = EventArray.from_events([event0, event1, event2])
177
178    assert len(event_array) == 3
179    assert event_array.metadata is None
180    assert event_array.features is None
181
182    event0.metadata = pd.Series({"event0": 0})
183
184    try:
185        event_array = EventArray.from_events([event0, event1, event2])
186        # Should throw error
187        assert False
188    except ValueError:
189        pass
190
191    event1.metadata = pd.Series({"event0": 1})
192    event2.metadata = pd.Series({"event0": 2})
193
194    event_array = EventArray.from_events([event0, event1, event2])
195
196    assert len(event_array) == 3
197
198    events_df = event_array.to_dataframe()
199
200    assert len(events_df) == 3
201
202    assert event_array == EventArray.from_dataframe(events_df)
203
204    # Test adding different dtypes and converting back and forth
205    event_array.features = pd.DataFrame(
206        {"feature1": [1, 2, 3], "feature2": [4.0, 5.0, 6.0]}
207    )
208    remade_event_list = event_array.to_events([scan])
209    assert len(remade_event_list) == 3
210    remade_event_array = EventArray.from_events(remade_event_list)
211    assert event_array == remade_event_array
212    # Test saving and loading
213    assert event_array.save_csv("tests/data/events.csv")
214    assert event_array == EventArray.load_csv("tests/data/events.csv")
215    os.remove("tests/data/events.csv")
216
217    assert event_array.save_hdf5("tests/data/events.h5")
218    assert event_array == EventArray.load_hdf5("tests/data/events.h5")
219    os.remove("tests/data/events.h5")
220
221
222def test_ocular_conversions():
223    scan = Scan.load_txt("tests/data")
224    input_path = "/mnt/HDSCA_Development/DZ/0B58703/ocular"
225    result = EventArray.load_ocular(input_path)
226    # For the purposes of this test, we will manually relabel "clust" == nan to 0
227    # These come from ocular_interesting.rds, which does not have clusters
228    result.metadata["clust"] = result.metadata["clust"].fillna(0)
229    result.metadata["hcpc"] = result.metadata["hcpc"].fillna(0)
230    result.save_ocular("tests/data")
231    new_result = EventArray.load_ocular("tests/data")
232    # # Sort them so that they are in the same order
233    result = result.sort(["tile", "x", "y"])
234    new_result = new_result.sort(["tile", "x", "y"])
235    # Note: hcpc method within ocularr and here are different
236    result.metadata["hcpc"] = new_result.metadata["hcpc"].copy()
237    assert result == new_result
238    # Clean up
239    os.remove("tests/data/rc-final.csv")
240    os.remove("tests/data/rc-final.rds")
241    os.remove("tests/data/rc-final1.rds")
242    os.remove("tests/data/rc-final2.rds")
243    os.remove("tests/data/rc-final3.rds")
244    os.remove("tests/data/rc-final4.rds")
245
246    # Try it with "others" files
247    result = EventArray.load_ocular(input_path, event_type="others")
248    result.save_ocular("tests/data", event_type="others")
249    new_result = EventArray.load_ocular("tests/data", event_type="others")
250    result = result.sort(["tile", "x", "y"])
251    new_result = new_result.sort(["tile", "x", "y"])
252    # Note: hcpc method within ocularr and here are different
253    result.metadata["hcpc"] = new_result.metadata["hcpc"].copy()
254    assert result == new_result
255    # Clean up
256    os.remove("tests/data/others-final.csv")
257    os.remove("tests/data/others-final.rds")
258    os.remove("tests/data/others-final1.rds")
259    os.remove("tests/data/others-final2.rds")
260    os.remove("tests/data/others-final3.rds")
261    os.remove("tests/data/others-final4.rds")
262
263
264def test_copy_sort_rows_get():
265    scan = Scan.load_yaml("tests/data")
266    # Origin
267    tile = Tile(scan, 0)
268    events = [
269        Event(scan, tile, 0, 100),
270        Event(scan, tile, 0, 0),
271        Event(scan, tile, 1000, 1000),
272        Event(scan, tile, 1000, 1),
273        Event(scan, tile, 2000, 2000),
274    ]
275
276    events = EventArray.from_events(events)
277
278    # Copy
279    events_copy = events.copy()
280    events_copy.info["x"] = np.uint16(1)
281    # Check that changes to the copy did not change the original
282    assert events_copy.info["x"].equals(pd.Series([1, 1, 1, 1, 1], dtype=np.uint16))
283    assert events.info["x"].equals(pd.Series([0, 0, 1000, 1000, 2000], dtype=np.uint16))
284
285    # Sort
286    events = events.sort(["x", "y"], ascending=[False, True])
287    assert events.info["x"].equals(pd.Series([2000, 1000, 1000, 0, 0], dtype=np.uint16))
288    assert events.info["y"].equals(pd.Series([2000, 1, 1000, 0, 100], dtype=np.uint16))
289
290    # Get
291    events_get = events.get(["x", "y"])
292    assert events_get["x"].equals(pd.Series([2000, 1000, 1000, 0, 0], dtype=np.uint16))
293    assert events_get["y"].equals(pd.Series([2000, 1, 1000, 0, 100], dtype=np.uint16))
294    assert events_get.columns.equals(pd.Index(["x", "y"]))
295
296    # Rows
297    events_get = events.rows([0, 1, 3])
298    assert len(events_get) == 3
299    assert events_get.info["x"].equals(pd.Series([2000, 1000, 0], dtype=np.uint16))
300    assert events_get.info["y"].equals(pd.Series([2000, 1, 0], dtype=np.uint16))
301    events_get = events.rows([True, False, False, True, True])
302    assert len(events_get) == 3
303    assert events_get.info["x"].equals(pd.Series([2000, 0, 0], dtype=np.uint16))
304    assert events_get.info["y"].equals(pd.Series([2000, 0, 100], dtype=np.uint16))
def test_getting_event():
20def test_getting_event():
21    scan = Scan.load_txt("tests/data")
22    tile = Tile(scan, 1000)
23    event = Event(
24        scan,
25        tile,
26        515,
27        411,
28    )
29    images = event.extract_images()
30    assert len(images) == 4
31    images = event.extract_images(crop_size=100, in_pixels=True)
32    assert images[0].shape == (100, 100)
33    images = event.extract_images(crop_size=50, in_pixels=True)
34    assert images[0].shape == (50, 50)
35
36    if SHOW_PLOTS:
37        for image in images:
38            cv2.imshow("Bright DAPI event in the center", image)
39            cv2.waitKey(0)
40        cv2.destroyAllWindows()
41
42    # Test a corner event
43    event = Event(
44        scan,
45        tile,
46        2,
47        1000,
48    )
49    images = event.extract_images()
50    assert len(images) == 4
51    images = event.extract_images(crop_size=100, in_pixels=True)
52    assert images[0].shape == (100, 100)
53    images = event.extract_images(crop_size=200, in_pixels=True)
54    assert images[0].shape == (200, 200)
55
56    if SHOW_PLOTS:
57        for image in images:
58            cv2.imshow("Events in the corner of a tile", image)
59            cv2.waitKey(0)
60        cv2.destroyAllWindows()
def test_getting_many_events():
 63def test_getting_many_events():
 64    scan = Scan.load_txt("tests/data")
 65    tile = Tile(scan, 1000)
 66    tile2 = Tile(scan, 0)
 67    events = [
 68        Event(scan, tile, 515, 411),
 69        Event(scan, tile2, 2, 1000),
 70        Event(scan, tile, 1000, 1000),
 71        Event(scan, tile, 87, 126),
 72        Event(scan, tile, 1000, 2),
 73        Event(scan, tile2, 800, 800),
 74        Event(scan, tile, 1000, 662),
 75    ]
 76    # Test time to extract images sequentially
 77    start_time = time.time()
 78    images_1 = []
 79    for event in events:
 80        images_1.append(event.extract_images())
 81    sequential_time = time.time() - start_time
 82
 83    # Test time to extract images in parallel
 84    start_time = time.time()
 85    images_2 = Event.extract_images_for_list(events, crop_size=100)
 86    parallel_time = time.time() - start_time
 87    assert parallel_time < sequential_time
 88    for list_a, list_b in zip(images_1, images_2):
 89        assert len(list_a) == len(list_b)
 90        for image_a, image_b in zip(list_a, list_b):
 91            assert np.array_equal(image_a, image_b)
 92
 93    # Test that it works after converting to EventArray and back
 94    event_array = EventArray.from_events(events)
 95    remade_events = event_array.to_events(
 96        [scan], ignore_metadata=True, ignore_features=True
 97    )
 98    images_3 = Event.extract_images_for_list(remade_events, crop_size=100)
 99    for list_a, list_b in zip(images_1, images_3):
100        assert len(list_a) == len(list_b)
101        for image_a, image_b in zip(list_a, list_b):
102            assert np.array_equal(image_a, image_b)
def test_event_coordinates_for_bzscanner():
105def test_event_coordinates_for_bzscanner():
106    scan = Scan.load_txt("tests/data")
107    # Origin
108    tile = Tile(scan, 0)
109    event = Event(scan, tile, 0, 0)
110    scan_origin = event.get_scan_position()
111    assert 2500 <= scan_origin[0] <= 3500
112    assert 2500 <= scan_origin[1] <= 3500
113    scan_origin_on_slide = event.get_slide_position()
114    assert 71500 <= scan_origin_on_slide[0] <= 72500
115    assert 21500 <= scan_origin_on_slide[1] <= 22500
116    # Within the same tile, "bottom-right corner"
117    event = Event(scan, tile, 1000, 1000)
118    scan_position = event.get_scan_position()
119    assert scan_origin[0] <= scan_position[0]
120    assert scan_origin[1] <= scan_position[1]
121    slide_position = event.get_slide_position()
122    assert slide_position[0] <= scan_origin_on_slide[0]
123    assert slide_position[1] <= scan_origin_on_slide[1]
124
125    # Next row, opposite side
126    tile = Tile(scan, (scan.roi[0].tile_cols - 1, 1))
127    event = Event(scan, tile, 1000, 1000)
128    scan_position = event.get_scan_position()
129    assert scan_origin[0] <= scan_position[0]
130    assert scan_origin[1] <= scan_position[1]
131    slide_position = event.get_slide_position()
132    assert slide_position[0] <= scan_origin_on_slide[0]
133    assert slide_position[1] <= scan_origin_on_slide[1]
134
135    # Opposite corner
136    tile = Tile(scan, (scan.roi[0].tile_cols - 1, scan.roi[0].tile_rows - 1))
137    event = Event(scan, tile, 1361, 1003)
138    scan_position = event.get_scan_position()
139    assert 21500 <= scan_position[0] <= 22500
140    assert 58500 <= scan_position[1] <= 60500
141    slide_position = event.get_slide_position()
142    assert 14500 <= slide_position[0] <= 15500
143    assert 2500 <= slide_position[1] <= 3500
def test_event_coordinates_for_axioscan():
146def test_event_coordinates_for_axioscan():
147    scan = Scan.load_yaml("tests/data")
148    # Origin
149    tile = Tile(scan, 0)
150    event = Event(scan, tile, 0, 0)
151    scan_position = event.get_scan_position()
152    assert -59000 <= scan_position[0] < -55000
153    assert 0 <= scan_position[1] < 4000
154    slide_position = event.get_slide_position()
155    assert 16000 <= slide_position[0] < 20000
156    assert scan_position[1] == slide_position[1]
157
158    # Opposite corner
159    tile = Tile(scan, (scan.roi[0].tile_cols - 1, scan.roi[0].tile_rows - 1))
160    event = Event(scan, tile, 2000, 2000)
161    scan_position = event.get_scan_position()
162    assert -4000 <= scan_position[0] <= 0
163    assert 21000 <= scan_position[1] <= 25000
164    slide_position = event.get_slide_position()
165    assert 71000 <= slide_position[0] <= 75000
166    assert scan_position[1] == slide_position[1]
def test_eventarray_conversions():
169def test_eventarray_conversions():
170    scan = Scan.load_yaml("tests/data")
171    # Origin
172    tile = Tile(scan, 0)
173    event0 = Event(scan, tile, 0, 0)
174    event1 = Event(scan, tile, 1000, 1000)
175    event2 = Event(scan, tile, 2000, 2000)
176
177    event_array = EventArray.from_events([event0, event1, event2])
178
179    assert len(event_array) == 3
180    assert event_array.metadata is None
181    assert event_array.features is None
182
183    event0.metadata = pd.Series({"event0": 0})
184
185    try:
186        event_array = EventArray.from_events([event0, event1, event2])
187        # Should throw error
188        assert False
189    except ValueError:
190        pass
191
192    event1.metadata = pd.Series({"event0": 1})
193    event2.metadata = pd.Series({"event0": 2})
194
195    event_array = EventArray.from_events([event0, event1, event2])
196
197    assert len(event_array) == 3
198
199    events_df = event_array.to_dataframe()
200
201    assert len(events_df) == 3
202
203    assert event_array == EventArray.from_dataframe(events_df)
204
205    # Test adding different dtypes and converting back and forth
206    event_array.features = pd.DataFrame(
207        {"feature1": [1, 2, 3], "feature2": [4.0, 5.0, 6.0]}
208    )
209    remade_event_list = event_array.to_events([scan])
210    assert len(remade_event_list) == 3
211    remade_event_array = EventArray.from_events(remade_event_list)
212    assert event_array == remade_event_array
213    # Test saving and loading
214    assert event_array.save_csv("tests/data/events.csv")
215    assert event_array == EventArray.load_csv("tests/data/events.csv")
216    os.remove("tests/data/events.csv")
217
218    assert event_array.save_hdf5("tests/data/events.h5")
219    assert event_array == EventArray.load_hdf5("tests/data/events.h5")
220    os.remove("tests/data/events.h5")
def test_ocular_conversions():
223def test_ocular_conversions():
224    scan = Scan.load_txt("tests/data")
225    input_path = "/mnt/HDSCA_Development/DZ/0B58703/ocular"
226    result = EventArray.load_ocular(input_path)
227    # For the purposes of this test, we will manually relabel "clust" == nan to 0
228    # These come from ocular_interesting.rds, which does not have clusters
229    result.metadata["clust"] = result.metadata["clust"].fillna(0)
230    result.metadata["hcpc"] = result.metadata["hcpc"].fillna(0)
231    result.save_ocular("tests/data")
232    new_result = EventArray.load_ocular("tests/data")
233    # # Sort them so that they are in the same order
234    result = result.sort(["tile", "x", "y"])
235    new_result = new_result.sort(["tile", "x", "y"])
236    # Note: hcpc method within ocularr and here are different
237    result.metadata["hcpc"] = new_result.metadata["hcpc"].copy()
238    assert result == new_result
239    # Clean up
240    os.remove("tests/data/rc-final.csv")
241    os.remove("tests/data/rc-final.rds")
242    os.remove("tests/data/rc-final1.rds")
243    os.remove("tests/data/rc-final2.rds")
244    os.remove("tests/data/rc-final3.rds")
245    os.remove("tests/data/rc-final4.rds")
246
247    # Try it with "others" files
248    result = EventArray.load_ocular(input_path, event_type="others")
249    result.save_ocular("tests/data", event_type="others")
250    new_result = EventArray.load_ocular("tests/data", event_type="others")
251    result = result.sort(["tile", "x", "y"])
252    new_result = new_result.sort(["tile", "x", "y"])
253    # Note: hcpc method within ocularr and here are different
254    result.metadata["hcpc"] = new_result.metadata["hcpc"].copy()
255    assert result == new_result
256    # Clean up
257    os.remove("tests/data/others-final.csv")
258    os.remove("tests/data/others-final.rds")
259    os.remove("tests/data/others-final1.rds")
260    os.remove("tests/data/others-final2.rds")
261    os.remove("tests/data/others-final3.rds")
262    os.remove("tests/data/others-final4.rds")
def test_copy_sort_rows_get():
265def test_copy_sort_rows_get():
266    scan = Scan.load_yaml("tests/data")
267    # Origin
268    tile = Tile(scan, 0)
269    events = [
270        Event(scan, tile, 0, 100),
271        Event(scan, tile, 0, 0),
272        Event(scan, tile, 1000, 1000),
273        Event(scan, tile, 1000, 1),
274        Event(scan, tile, 2000, 2000),
275    ]
276
277    events = EventArray.from_events(events)
278
279    # Copy
280    events_copy = events.copy()
281    events_copy.info["x"] = np.uint16(1)
282    # Check that changes to the copy did not change the original
283    assert events_copy.info["x"].equals(pd.Series([1, 1, 1, 1, 1], dtype=np.uint16))
284    assert events.info["x"].equals(pd.Series([0, 0, 1000, 1000, 2000], dtype=np.uint16))
285
286    # Sort
287    events = events.sort(["x", "y"], ascending=[False, True])
288    assert events.info["x"].equals(pd.Series([2000, 1000, 1000, 0, 0], dtype=np.uint16))
289    assert events.info["y"].equals(pd.Series([2000, 1, 1000, 0, 100], dtype=np.uint16))
290
291    # Get
292    events_get = events.get(["x", "y"])
293    assert events_get["x"].equals(pd.Series([2000, 1000, 1000, 0, 0], dtype=np.uint16))
294    assert events_get["y"].equals(pd.Series([2000, 1, 1000, 0, 100], dtype=np.uint16))
295    assert events_get.columns.equals(pd.Index(["x", "y"]))
296
297    # Rows
298    events_get = events.rows([0, 1, 3])
299    assert len(events_get) == 3
300    assert events_get.info["x"].equals(pd.Series([2000, 1000, 0], dtype=np.uint16))
301    assert events_get.info["y"].equals(pd.Series([2000, 1, 0], dtype=np.uint16))
302    events_get = events.rows([True, False, False, True, True])
303    assert len(events_get) == 3
304    assert events_get.info["x"].equals(pd.Series([2000, 0, 0], dtype=np.uint16))
305    assert events_get.info["y"].equals(pd.Series([2000, 0, 100], dtype=np.uint16))