Skip to content

Usage examples

from comparisonframe import ComparisonFrame

1. Creating validation set

1.1 Initialize comparison class

comparer = ComparisonFrame(
    # optionally
    ## mocker default parameters
    mocker_params = {
        'file_path' : "./comparisonframe_storage",
         'persist' : True},

    ## scores to calculate
    compare_scores = ['word_count_diff','semantic_similarity'],
    aggr_scores = ['median']
)

1.2 Recording queries and expected responses (validation set)

comparer.record_queries(
    queries = ["Black metal", 
               "Tribulation"],
    expected_texts = ["Black metal is an extreme subgenre of heavy metal music.",
    "Tribulation are a Swedish heavy metal band from Arvika that formed in 2005."],
    metadata = {'name' : 'metal_bands'})

2. Comparing newly generated data with expected results

2.1 Initialize new comparison class

comparer = ComparisonFrame(
    # optionally
    ## mocker default parameters
    mocker_params = {
        'file_path' : "./comparisonframe_storage",
         'persist' : True},

    ## scores to calculate
    compare_scores = ['word_count_diff','semantic_similarity'],
    aggr_scores = ['median']
)

2.2 Show validation set

untested_queries = comparer.get_all_queries(
    ## optional
    metadata_filters={'name' : 'metal_bands'})
print(untested_queries)
['Black metal', 'Tribulation']
comparer.get_all_records()
[{'record_id': '0cc157453395b440f36d1a1aee24aa76a03f5f9ab0a7a8bd7b663c92f2f16e87',
  'query': 'Black metal',
  'expected_text': 'Black metal is an extreme subgenre of heavy metal music.'},
 {'record_id': 'eecd9c2a5b25ee6053891b894157fa30372ed694763385e1ada1dc9ad8e41625',
  'query': 'Tribulation',
  'expected_text': 'Tribulation are a Swedish heavy metal band from Arvika that formed in 2005.'}]
comparer.get_all_records_df()
record_id query expected_text
0 0cc157453395b440f36d1a1aee24aa76a03f5f9ab0a7a8... Black metal Black metal is an extreme subgenre of heavy me...
1 eecd9c2a5b25ee6053891b894157fa30372ed694763385... Tribulation Tribulation are a Swedish heavy metal band fro...

2.3 Insert newly generated with records

valid_answer_query_1 = "Black metal is an extreme subgenre of heavy metal music."
very_similar_answer_query_1 = "Black metal is a subgenre of heavy metal music."
unexpected_answer_query_1 = "Black metals are beautiful and are often used in jewelry design."
comparer.record_runs(queries = ["Black metal"],
                     provided_texts = [valid_answer_query_1,
                                      very_similar_answer_query_1,
                                      unexpected_answer_query_1],
                    metadata={'desc' : 'definitions'})
comparer.get_all_runs()
[{'timestamp': '2024-09-24 03:43:48',
  'run_id': 'e8f5f87186ac81a38896e602c7d8a4e1b973aa00973b68cbfdc983e6c1762d6e',
  'query': 'Black metal',
  'provided_text': 'Black metal is an extreme subgenre of heavy metal music.'},
 {'timestamp': '2024-09-24 03:43:48',
  'run_id': 'e0e4dfaff0b16227dbfde7a75eb9f060ccd2b1b5acc8b4b48aa9d305a3591353',
  'query': 'Black metal',
  'provided_text': 'Black metal is a subgenre of heavy metal music.'},
 {'timestamp': '2024-09-24 03:43:48',
  'run_id': '4e806f8cf2bd2184e63d41c85322e129b14c8d5206472c7668077452386257af',
  'query': 'Black metal',
  'provided_text': 'Black metals are beautiful and are often used in jewelry design.'}]
df = comparer.get_all_runs_df()
df
timestamp run_id query provided_text
0 2024-09-24 03:43:48 e8f5f87186ac81a38896e602c7d8a4e1b973aa00973b68... Black metal Black metal is an extreme subgenre of heavy me...
1 2024-09-24 03:43:48 e0e4dfaff0b16227dbfde7a75eb9f060ccd2b1b5acc8b4... Black metal Black metal is a subgenre of heavy metal music.
2 2024-09-24 03:43:48 4e806f8cf2bd2184e63d41c85322e129b14c8d5206472c... Black metal Black metals are beautiful and are often used ...

2.4 Comparing runs with records

comparer.compare_runs_with_records()
WARNING:ComparisonFrame:No data was found with applied filters!
comparer.get_all_run_scores()
[{'timestamp': '2024-09-24 03:43:57',
  'run_id': 'e8f5f87186ac81a38896e602c7d8a4e1b973aa00973b68cbfdc983e6c1762d6e',
  'query': 'Black metal',
  'provided_text': 'Black metal is an extreme subgenre of heavy metal music.',
  'record_id': '0cc157453395b440f36d1a1aee24aa76a03f5f9ab0a7a8bd7b663c92f2f16e87',
  'word_count_diff': 0,
  'semantic_similarity': 1.0000001192092896,
  'comparison_id': '9376f9245bf3a237ef1eb4920cc2c86918fdba5e5934014058b14286fbd31f32'},
 {'timestamp': '2024-09-24 03:43:57',
  'run_id': 'e0e4dfaff0b16227dbfde7a75eb9f060ccd2b1b5acc8b4b48aa9d305a3591353',
  'query': 'Black metal',
  'provided_text': 'Black metal is a subgenre of heavy metal music.',
  'record_id': '0cc157453395b440f36d1a1aee24aa76a03f5f9ab0a7a8bd7b663c92f2f16e87',
  'word_count_diff': 1,
  'semantic_similarity': 0.9859851002693176,
  'comparison_id': '756abbce6c86dea26ce9f3d0cc11668b76df40120faf2fbee1dad5daa1e201ec'},
 {'timestamp': '2024-09-24 03:43:57',
  'run_id': '4e806f8cf2bd2184e63d41c85322e129b14c8d5206472c7668077452386257af',
  'query': 'Black metal',
  'provided_text': 'Black metals are beautiful and are often used in jewelry design.',
  'record_id': '0cc157453395b440f36d1a1aee24aa76a03f5f9ab0a7a8bd7b663c92f2f16e87',
  'word_count_diff': 1,
  'semantic_similarity': 0.4940534234046936,
  'comparison_id': 'f6ab2967695a560f4ff208e03c2804b09f26b9356ab5da592186e503a900ab56'}]
comparer.get_all_run_scores_df()
timestamp run_id query provided_text record_id word_count_diff semantic_similarity comparison_id
0 2024-09-24 03:43:57 e8f5f87186ac81a38896e602c7d8a4e1b973aa00973b68... Black metal Black metal is an extreme subgenre of heavy me... 0cc157453395b440f36d1a1aee24aa76a03f5f9ab0a7a8... 0 1.000000 9376f9245bf3a237ef1eb4920cc2c86918fdba5e593401...
1 2024-09-24 03:43:57 e0e4dfaff0b16227dbfde7a75eb9f060ccd2b1b5acc8b4... Black metal Black metal is a subgenre of heavy metal music. 0cc157453395b440f36d1a1aee24aa76a03f5f9ab0a7a8... 1 0.985985 756abbce6c86dea26ce9f3d0cc11668b76df40120faf2f...
2 2024-09-24 03:43:57 4e806f8cf2bd2184e63d41c85322e129b14c8d5206472c... Black metal Black metals are beautiful and are often used ... 0cc157453395b440f36d1a1aee24aa76a03f5f9ab0a7a8... 1 0.494053 f6ab2967695a560f4ff208e03c2804b09f26b9356ab5da...

3 Calculating aggregate comparison scores

comparer.calculate_aggr_scores()
WARNING:ComparisonFrame:No data was found with applied filters!
comparer.get_all_aggr_scores()
[{'timestamp': '2024-09-24 03:44:07',
  'comparison_id': ['9376f9245bf3a237ef1eb4920cc2c86918fdba5e5934014058b14286fbd31f32',
   '756abbce6c86dea26ce9f3d0cc11668b76df40120faf2fbee1dad5daa1e201ec',
   'f6ab2967695a560f4ff208e03c2804b09f26b9356ab5da592186e503a900ab56'],
  'query': 'Black metal',
  'median_word_count_diff': 1.0,
  'median_semantic_similarity': 0.9859851002693176,
  'record_status_id': '2723311824179b0f89b65a513368bfe05298a625b7c73b5a06ff33702689d56a'}]
comparer.get_all_aggr_scores_df()
timestamp comparison_id query median_word_count_diff median_semantic_similarity record_status_id
0 2024-09-24 03:44:07 [9376f9245bf3a237ef1eb4920cc2c86918fdba5e59340... Black metal 1.0 0.985985 2723311824179b0f89b65a513368bfe05298a625b7c73b...

4. Recording test statuses

comparer.calculate_test_statuses(test_query = "median_semantic_similarity > 0.9")

comparer.get_test_statuses()
[{'timestamp': '2024-09-24 03:44:13',
  'record_id': '0cc157453395b440f36d1a1aee24aa76a03f5f9ab0a7a8bd7b663c92f2f16e87',
  'record_status_id': '2723311824179b0f89b65a513368bfe05298a625b7c73b5a06ff33702689d56a',
  'query': 'Black metal',
  'test': 'median_semantic_similarity > 0.9',
  'valid': True}]
comparer.get_test_statuses_df()
timestamp record_id record_status_id query test valid
0 2024-09-24 03:44:13 0cc157453395b440f36d1a1aee24aa76a03f5f9ab0a7a8... 2723311824179b0f89b65a513368bfe05298a625b7c73b... Black metal median_semantic_similarity > 0.9 True

5. Reseting statuses, flushing records and comparison results

comparer.flush_records()
comparer.flush_runs()
comparer.flush_comparison_scores()
comparer.flush_aggregate_scores()
comparer.flush_test_statuses()