Usage examples
from comparisonframe import ComparisonFrame
1. Creating validation set
1.1 Initialize comparison class
comparer = ComparisonFrame(
# optionally
## mocker default parameters
mocker_params = {
'file_path' : "./comparisonframe_storage",
'persist' : True},
## scores to calculate
compare_scores = ['word_count_diff','semantic_similarity'],
aggr_scores = ['median']
)
1.2 Recording queries and expected responses (validation set)
comparer.record_queries(
queries = ["Black metal",
"Tribulation"],
expected_texts = ["Black metal is an extreme subgenre of heavy metal music.",
"Tribulation are a Swedish heavy metal band from Arvika that formed in 2005."],
metadata = {'name' : 'metal_bands'})
2. Comparing newly generated data with expected results
2.1 Initialize new comparison class
comparer = ComparisonFrame(
# optionally
## mocker default parameters
mocker_params = {
'file_path' : "./comparisonframe_storage",
'persist' : True},
## scores to calculate
compare_scores = ['word_count_diff','semantic_similarity'],
aggr_scores = ['median']
)
2.2 Show validation set
untested_queries = comparer.get_all_queries(
## optional
metadata_filters={'name' : 'metal_bands'})
print(untested_queries)
['Black metal', 'Tribulation']
comparer.get_all_records()
[{'record_id': '0cc157453395b440f36d1a1aee24aa76a03f5f9ab0a7a8bd7b663c92f2f16e87',
'query': 'Black metal',
'expected_text': 'Black metal is an extreme subgenre of heavy metal music.'},
{'record_id': 'eecd9c2a5b25ee6053891b894157fa30372ed694763385e1ada1dc9ad8e41625',
'query': 'Tribulation',
'expected_text': 'Tribulation are a Swedish heavy metal band from Arvika that formed in 2005.'}]
comparer.get_all_records_df()
|
record_id |
query |
expected_text |
0 |
0cc157453395b440f36d1a1aee24aa76a03f5f9ab0a7a8... |
Black metal |
Black metal is an extreme subgenre of heavy me... |
1 |
eecd9c2a5b25ee6053891b894157fa30372ed694763385... |
Tribulation |
Tribulation are a Swedish heavy metal band fro... |
2.3 Insert newly generated with records
valid_answer_query_1 = "Black metal is an extreme subgenre of heavy metal music."
very_similar_answer_query_1 = "Black metal is a subgenre of heavy metal music."
unexpected_answer_query_1 = "Black metals are beautiful and are often used in jewelry design."
comparer.record_runs(queries = ["Black metal"],
provided_texts = [valid_answer_query_1,
very_similar_answer_query_1,
unexpected_answer_query_1],
metadata={'desc' : 'definitions'})
comparer.get_all_runs()
[{'timestamp': '2024-09-24 03:43:48',
'run_id': 'e8f5f87186ac81a38896e602c7d8a4e1b973aa00973b68cbfdc983e6c1762d6e',
'query': 'Black metal',
'provided_text': 'Black metal is an extreme subgenre of heavy metal music.'},
{'timestamp': '2024-09-24 03:43:48',
'run_id': 'e0e4dfaff0b16227dbfde7a75eb9f060ccd2b1b5acc8b4b48aa9d305a3591353',
'query': 'Black metal',
'provided_text': 'Black metal is a subgenre of heavy metal music.'},
{'timestamp': '2024-09-24 03:43:48',
'run_id': '4e806f8cf2bd2184e63d41c85322e129b14c8d5206472c7668077452386257af',
'query': 'Black metal',
'provided_text': 'Black metals are beautiful and are often used in jewelry design.'}]
df = comparer.get_all_runs_df()
df
|
timestamp |
run_id |
query |
provided_text |
0 |
2024-09-24 03:43:48 |
e8f5f87186ac81a38896e602c7d8a4e1b973aa00973b68... |
Black metal |
Black metal is an extreme subgenre of heavy me... |
1 |
2024-09-24 03:43:48 |
e0e4dfaff0b16227dbfde7a75eb9f060ccd2b1b5acc8b4... |
Black metal |
Black metal is a subgenre of heavy metal music. |
2 |
2024-09-24 03:43:48 |
4e806f8cf2bd2184e63d41c85322e129b14c8d5206472c... |
Black metal |
Black metals are beautiful and are often used ... |
2.4 Comparing runs with records
comparer.compare_runs_with_records()
WARNING:ComparisonFrame:No data was found with applied filters!
comparer.get_all_run_scores()
[{'timestamp': '2024-09-24 03:43:57',
'run_id': 'e8f5f87186ac81a38896e602c7d8a4e1b973aa00973b68cbfdc983e6c1762d6e',
'query': 'Black metal',
'provided_text': 'Black metal is an extreme subgenre of heavy metal music.',
'record_id': '0cc157453395b440f36d1a1aee24aa76a03f5f9ab0a7a8bd7b663c92f2f16e87',
'word_count_diff': 0,
'semantic_similarity': 1.0000001192092896,
'comparison_id': '9376f9245bf3a237ef1eb4920cc2c86918fdba5e5934014058b14286fbd31f32'},
{'timestamp': '2024-09-24 03:43:57',
'run_id': 'e0e4dfaff0b16227dbfde7a75eb9f060ccd2b1b5acc8b4b48aa9d305a3591353',
'query': 'Black metal',
'provided_text': 'Black metal is a subgenre of heavy metal music.',
'record_id': '0cc157453395b440f36d1a1aee24aa76a03f5f9ab0a7a8bd7b663c92f2f16e87',
'word_count_diff': 1,
'semantic_similarity': 0.9859851002693176,
'comparison_id': '756abbce6c86dea26ce9f3d0cc11668b76df40120faf2fbee1dad5daa1e201ec'},
{'timestamp': '2024-09-24 03:43:57',
'run_id': '4e806f8cf2bd2184e63d41c85322e129b14c8d5206472c7668077452386257af',
'query': 'Black metal',
'provided_text': 'Black metals are beautiful and are often used in jewelry design.',
'record_id': '0cc157453395b440f36d1a1aee24aa76a03f5f9ab0a7a8bd7b663c92f2f16e87',
'word_count_diff': 1,
'semantic_similarity': 0.4940534234046936,
'comparison_id': 'f6ab2967695a560f4ff208e03c2804b09f26b9356ab5da592186e503a900ab56'}]
comparer.get_all_run_scores_df()
|
timestamp |
run_id |
query |
provided_text |
record_id |
word_count_diff |
semantic_similarity |
comparison_id |
0 |
2024-09-24 03:43:57 |
e8f5f87186ac81a38896e602c7d8a4e1b973aa00973b68... |
Black metal |
Black metal is an extreme subgenre of heavy me... |
0cc157453395b440f36d1a1aee24aa76a03f5f9ab0a7a8... |
0 |
1.000000 |
9376f9245bf3a237ef1eb4920cc2c86918fdba5e593401... |
1 |
2024-09-24 03:43:57 |
e0e4dfaff0b16227dbfde7a75eb9f060ccd2b1b5acc8b4... |
Black metal |
Black metal is a subgenre of heavy metal music. |
0cc157453395b440f36d1a1aee24aa76a03f5f9ab0a7a8... |
1 |
0.985985 |
756abbce6c86dea26ce9f3d0cc11668b76df40120faf2f... |
2 |
2024-09-24 03:43:57 |
4e806f8cf2bd2184e63d41c85322e129b14c8d5206472c... |
Black metal |
Black metals are beautiful and are often used ... |
0cc157453395b440f36d1a1aee24aa76a03f5f9ab0a7a8... |
1 |
0.494053 |
f6ab2967695a560f4ff208e03c2804b09f26b9356ab5da... |
3 Calculating aggregate comparison scores
comparer.calculate_aggr_scores()
WARNING:ComparisonFrame:No data was found with applied filters!
comparer.get_all_aggr_scores()
[{'timestamp': '2024-09-24 03:44:07',
'comparison_id': ['9376f9245bf3a237ef1eb4920cc2c86918fdba5e5934014058b14286fbd31f32',
'756abbce6c86dea26ce9f3d0cc11668b76df40120faf2fbee1dad5daa1e201ec',
'f6ab2967695a560f4ff208e03c2804b09f26b9356ab5da592186e503a900ab56'],
'query': 'Black metal',
'median_word_count_diff': 1.0,
'median_semantic_similarity': 0.9859851002693176,
'record_status_id': '2723311824179b0f89b65a513368bfe05298a625b7c73b5a06ff33702689d56a'}]
comparer.get_all_aggr_scores_df()
|
timestamp |
comparison_id |
query |
median_word_count_diff |
median_semantic_similarity |
record_status_id |
0 |
2024-09-24 03:44:07 |
[9376f9245bf3a237ef1eb4920cc2c86918fdba5e59340... |
Black metal |
1.0 |
0.985985 |
2723311824179b0f89b65a513368bfe05298a625b7c73b... |
4. Recording test statuses
comparer.calculate_test_statuses(test_query = "median_semantic_similarity > 0.9")
comparer.get_test_statuses()
[{'timestamp': '2024-09-24 03:44:13',
'record_id': '0cc157453395b440f36d1a1aee24aa76a03f5f9ab0a7a8bd7b663c92f2f16e87',
'record_status_id': '2723311824179b0f89b65a513368bfe05298a625b7c73b5a06ff33702689d56a',
'query': 'Black metal',
'test': 'median_semantic_similarity > 0.9',
'valid': True}]
comparer.get_test_statuses_df()
|
timestamp |
record_id |
record_status_id |
query |
test |
valid |
0 |
2024-09-24 03:44:13 |
0cc157453395b440f36d1a1aee24aa76a03f5f9ab0a7a8... |
2723311824179b0f89b65a513368bfe05298a625b7c73b... |
Black metal |
median_semantic_similarity > 0.9 |
True |
5. Reseting statuses, flushing records and comparison results
comparer.flush_records()
comparer.flush_runs()
comparer.flush_comparison_scores()
comparer.flush_aggregate_scores()
comparer.flush_test_statuses()