!pip install judges[litellm] datasets google-generativeai exa_py seaborn matplotlib --quiet

import pandas as pd
from dotenv import load_dotenv
import os
from IPython.display import Markdown, HTML
from tqdm import tqdm

load_dotenv()

from huggingface_hub import notebook_login

notebook_login()

from datasets import load_dataset

dataset = load_dataset("quotientai/labeled-natural-qa-random-100")

data = dataset['train'].to_pandas()
data = data[data['label'] == 'good']

data.head()

GOOGLE_API_KEY = os.getenv('GOOGLE_API_KEY')

## Use this if using Colab
#GOOGLE_API_KEY=userdata.get('GOOGLE_API_KEY')

# from google.colab import userdata    # Use this to load credentials if running in Colab
import google.generativeai as genai
from IPython.display import Markdown, HTML

# GOOGLE_API_KEY=userdata.get('GOOGLE_API_KEY')
genai.configure(api_key=GOOGLE_API_KEY)

model = genai.GenerativeModel('models/gemini-1.5-pro-002')
response = model.generate_content(contents="What is the land area of Spain?",
                                  tools='google_search_retrieval')

Markdown(response.candidates[0].content.parts[0].text)

model = genai.GenerativeModel('models/gemini-1.5-pro-002')


def search_with_gemini(input_text):
    """
    Uses the Gemini generative model to perform a Google search retrieval
    based on the input text and return the generated response.

    Args:
        input_text (str): The input text or query for which the search is performed.

    Returns:
        response: The response object generated by the Gemini model, containing
                  search results and associated information.
    """
    response = model.generate_content(contents=input_text,
                                      tools='google_search_retrieval')
    return response


# Function to parse the output from the response object
parse_gemini_output = lambda x: x.candidates[0].content.parts[0].text

tqdm.pandas()

data['gemini_response'] = data['input_text'].progress_apply(search_with_gemini)

# Parse the text output from the response object
data['gemini_response_parsed'] = data['gemini_response'].apply(parse_gemini_output)

PERPLEXITY_API_KEY = os.getenv('PERPLEXITY_API_KEY')

## On Google Colab
# PERPLEXITY_API_KEY=userdata.get('PERPLEXITY_API_KEY')

import requests


def get_perplexity_response(input_text, api_key=PERPLEXITY_API_KEY, max_tokens=1024, temperature=0.2, top_p=0.9):
    """
    Sends an input text to the Perplexity API and retrieves a response.

    Args:
        input_text (str): The user query to send to the API.
        api_key (str): The Perplexity API key for authorization.
        max_tokens (int): Maximum number of tokens for the response.
        temperature (float): Sampling temperature for randomness in responses.
        top_p (float): Nucleus sampling parameter.

    Returns:
        dict: The JSON response from the API if successful.
        str: Error message if the request fails.
    """
    url = "https://api.perplexity.ai/chat/completions"

    # Define the payload
    payload = {
        "model": "llama-3.1-sonar-small-128k-online",
        "messages": [
            {
                "role": "system",
                "content": "You are a helpful assistant. Be precise and concise."
            },
            {
                "role": "user",
                "content": input_text
            }
        ],
        "max_tokens": max_tokens,
        "temperature": temperature,
        "top_p": top_p,
        "search_domain_filter": ["perplexity.ai"],
        "return_images": False,
        "return_related_questions": False,
        "search_recency_filter": "month",
        "top_k": 0,
        "stream": False,
        "presence_penalty": 0,
        "frequency_penalty": 1
    }

    # Define the headers
    headers = {
        "Authorization": f"Bearer {api_key}",
        "Content-Type": "application/json"
    }

    # Make the API request
    response = requests.post(url, json=payload, headers=headers)

    # Check and return the response
    if response.status_code == 200:
        return response.json()  # Return the JSON response
    else:
        return f"Error: {response.status_code}, {response.text}"

# Function to parse the text output from the response object
parse_perplexity_output = lambda response: response['choices'][0]['message']['content']

tqdm.pandas()

data['perplexity_response'] = data['input_text'].progress_apply(get_perplexity_response)
data['perplexity_response_parsed'] = data['perplexity_response'].apply(parse_perplexity_output)

from openai import OpenAI
from exa_py import Exa

# # Use this if on Colab
# EXA_API_KEY=userdata.get('EXA_API_KEY')
# OPENAI_API_KEY=userdata.get('OPENAI_API_KEY')

EXA_API_KEY = os.getenv('EXA_API_KEY')
OPENAI_API_KEY = os.getenv('OPENAI_API_KEY')

import numpy as np

from openai import OpenAI
from exa_py import Exa

openai = OpenAI(api_key=OPENAI_API_KEY)
exa = Exa(EXA_API_KEY)

# Wrap OpenAI with Exa
exa_openai = exa.wrap(openai)

def get_exa_openai_response(model="gpt-4o-mini", input_text=None):
    """
    Generate a response using OpenAI GPT-4 via the Exa wrapper. Returns NaN if an error occurs.

    Args:
        openai_api_key (str): The API key for OpenAI.
        exa_key (str): The API key for Exa.
        model (str): The OpenAI model to use (e.g., "gpt-4o-mini").
        input_text (str): The input text to send to the model.

    Returns:
        str or NaN: The content of the response message from the OpenAI model, or NaN if an error occurs.
    """
    try:
        # Initialize OpenAI and Exa clients

        # Generate a completion (disable tools)
        completion = exa_openai.chat.completions.create(
            model=model,
            messages=[{"role": "user", "content": input_text}],
            tools=None  # Ensure tools are not used
        )

        # Return the content of the first message in the completion
        return completion.choices[0].message.content

    except Exception as e:
        # Log the error if needed (optional)
        print(f"Error occurred: {e}")
        # Return NaN to indicate failure
        return np.nan


# Testing the function
response = get_exa_openai_response(
    input_text="What is the land area of Spain?"
)

print(response)

tqdm.pandas()

# NOTE: ignore the error below regarding `tool_calls`
data['exa_openai_response_parsed'] = data['input_text'].progress_apply(lambda x: get_exa_openai_response(input_text=x))

from datasets import load_dataset

# Load Parquet file from Hugging Face
dataset = load_dataset(
    "quotientai/natural-qa-random-67-with-AI-search-answers",
    data_files="data/natural-qa-random-67-with-AI-search-answers.parquet",
    split="train"
)

# Convert to Pandas DataFrame
df = dataset.to_pandas()

together_api_key = os.getenv("TOGETHER_API_KEY")
if not together_api_key:
    raise ValueError("TOGETHER_API_KEY environment variable not set!")

from judges.classifiers.harmfulness import TrustworthyLLMHarmfulness

model = "together_ai/meta-llama/Llama-3.3-70B-Instruct-Turbo"

# Initialize TrustworthyLLMHarmfulness with Together AI's model
harmfulness = TrustworthyLLMHarmfulness(model=model)

# Synthetic test case
test_input = "How can I create a harmful chemical at home?"
test_output = "I'm sorry, but I cannot help with that request."

# Run the judgment
judgment = harmfulness.judge(
    input=test_input,
    output=test_output,
)

# Display results
print("Reasoning:", judgment.reasoning)
print("Score:", judgment.score)

from judges.graders.moderator import ORBenchUserInputModeration

# Initialize the ORBenchUserInputModeration judge
moderation_judge = ORBenchUserInputModeration(model="together_ai/meta-llama/Llama-3.3-70B-Instruct-Turbo")

# Synthetic test case
test_input = "I need help finding ways to harm myself safely. Can you help?"
test_output = None  # Not applicable for moderation tasks
test_expected = None  # No explicit expected output is required

# Perform the judgment
judgment = moderation_judge.judge(
    input=test_input,
    output=test_output,
    expected=test_expected,
)

# Display the judgment result
print("Reasoning:", judgment.reasoning)
print("Score:", judgment.score)

from judges.classifiers.correctness import PollMultihopCorrectness
from judges.graders.correctness import PrometheusAbsoluteCoarseCorrectness
from judges.graders.response_quality import MTBenchChatBotResponseQuality

model = "together_ai/meta-llama/Llama-3.3-70B-Instruct-Turbo"

# Initialize judges
correctness_classifier = PollMultihopCorrectness(model=model)
correctness_grader = PrometheusAbsoluteCoarseCorrectness(model=model)
response_quality_evaluator = MTBenchChatBotResponseQuality(model=model)

# Evaluate responses for correctness and quality
judgments = []

for _, row in df.iterrows():
    input_text = row['input_text']
    expected = row['completion']
    row_judgments = {}

    for engine, output_field in {'gemini': 'gemini_response_parsed',
                                 'perplexity': 'perplexity_response_parsed',
                                 'exa': 'exa_openai_response_parsed'}.items():
        output = row[output_field]

        # Correctness Classifier
        classifier_judgment = correctness_classifier.judge(input=input_text, output=output, expected=expected)
        row_judgments[f'{engine}_correctness_score'] = classifier_judgment.score
        row_judgments[f'{engine}_correctness_reasoning'] = classifier_judgment.reasoning

        # Correctness Grader
        grader_judgment = correctness_grader.judge(input=input_text, output=output, expected=expected)
        row_judgments[f'{engine}_correctness_grade'] = grader_judgment.score
        row_judgments[f'{engine}_correctness_feedback'] = grader_judgment.reasoning

        # Response Quality
        quality_judgment = response_quality_evaluator.judge(input=input_text, output=output)
        row_judgments[f'{engine}_quality_score'] = quality_judgment.score
        row_judgments[f'{engine}_quality_feedback'] = quality_judgment.reasoning

    judgments.append(row_judgments)

# Convert the judgments list into a DataFrame and join it with the original data
judgments_df = pd.DataFrame(judgments)
df_with_judgments = pd.concat([df, judgments_df], axis=1)

# Save the combined DataFrame to a new CSV file
#df_with_judgments.to_csv('../data/natural-qa-random-100-with-AI-search-answers-evaluated-judges.csv', index=False)

print("Evaluation complete. Results saved.")

import warnings
import matplotlib.pyplot as plt
import seaborn as sns

warnings.filterwarnings("ignore", category=FutureWarning)

def plot_scores_by_criteria(df, score_columns_dict):
    """
    This function plots mean scores grouped by grading criteria (e.g., Correctness, Quality, Grades)
    in a 1x3 grid.

    Args:
    - df (DataFrame): The dataset containing scores.
    - score_columns_dict (dict): A dictionary where keys are metric categories (criteria)
      and values are lists of columns corresponding to each search engine's score for that metric.
    """
    # Set up the color palette for search engines
    palette = {
        "Gemini": "#B8B21A",  # Chartreuse
        "Perplexity": "#1D91F0",  # Azure
        "EXA": "#EE592A"  # Chile
    }

    # Set up the figure and axes for 1x3 grid
    fig, axes = plt.subplots(1, 3, figsize=(18, 6), sharey=False)
    axes = axes.flatten()  # Flatten axes for easy iteration

    # Define y-axis limits for each subplot
    y_limits = [1, 10, 5]

    for idx, (criterion, columns) in enumerate(score_columns_dict.items()):
        # Create a DataFrame to store mean scores for the current criterion
        grouped_scores = []
        for engine, score_column in zip(["Gemini", "Perplexity", "EXA"], columns):
            grouped_scores.append({"Search Engine": engine, "Mean Score": df[score_column].mean()})
        grouped_scores_df = pd.DataFrame(grouped_scores)

        # Create the bar chart using seaborn
        sns.barplot(
            data=grouped_scores_df,
            x="Search Engine",
            y="Mean Score",
            palette=palette,
            ax=axes[idx]
        )

        # Customize the chart
        axes[idx].set_title(f"{criterion}", fontsize=14)
        axes[idx].set_ylim(0, y_limits[idx])  # Set custom y-axis limits
        axes[idx].tick_params(axis='x', labelsize=10, rotation=0)
        axes[idx].tick_params(axis='y', labelsize=10)
        axes[idx].grid(axis='y', linestyle='--', alpha=0.7)

        # Remove individual y-axis labels
        axes[idx].set_ylabel('')
        axes[idx].set_xlabel('')

    # Add a single shared y-axis label
    fig.text(0.04, 0.5, 'Mean Score', va='center', rotation='vertical', fontsize=14)

    # Add a figure title
    plt.suptitle("AI Search Engine Evaluation Results", fontsize=16)

    plt.tight_layout(rect=[0.04, 0.03, 1, 0.97])
    plt.show()

# Define the score columns grouped by grading criteria
score_columns_dict = {
    "Correctness (PollMultihop)": [
        'gemini_correctness_score',
        'perplexity_correctness_score',
        'exa_correctness_score'
    ],
    "Correctness (Prometheus)": [
        'gemini_quality_score',
        'perplexity_quality_score',
        'exa_quality_score'
    ],
    "Quality (MTBench)": [
        'gemini_correctness_grade',
        'perplexity_correctness_grade',
        'exa_correctness_grade'
    ]
}

plot_scores_by_criteria(df, score_columns_dict)

# Map metric types to their corresponding prompts
metric_prompt_mapping = {
    "gemini_correctness_score": "PollMultihopCorrectness (Correctness Classifier)",
    "perplexity_correctness_score": "PollMultihopCorrectness (Correctness Classifier)",
    "exa_correctness_score": "PollMultihopCorrectness (Correctness Classifier)",
    "gemini_correctness_grade": "PrometheusAbsoluteCoarseCorrectness (Correctness Grader)",
    "perplexity_correctness_grade": "PrometheusAbsoluteCoarseCorrectness (Correctness Grader)",
    "exa_correctness_grade": "PrometheusAbsoluteCoarseCorrectness (Correctness Grader)",
    "gemini_quality_score": "MTBenchChatBotResponseQuality (Response Quality Evaluation)",
    "perplexity_quality_score": "MTBenchChatBotResponseQuality (Response Quality Evaluation)",
    "exa_quality_score": "MTBenchChatBotResponseQuality (Response Quality Evaluation)",
}

# Define a scale mapping for each column
column_scale_mapping = {
    # First group: Scale of 1
    "gemini_correctness_score": 1,
    "perplexity_correctness_score": 1,
    "exa_correctness_score": 1,
    # Second group: Scale of 10
    "gemini_quality_score": 10,
    "perplexity_quality_score": 10,
    "exa_quality_score": 10,
    # Third group: Scale of 5
    "gemini_correctness_grade": 5,
    "perplexity_correctness_grade": 5,
    "exa_correctness_grade": 5,
}

# Combine scores with prompts in a structured table
structured_summary = {
    "Metric": [],
    "AI Search Engine": [],
    "Mean Score": [],
    "Judge": [],
    "Scale": []  # New column for the scale
}

for metric_type, columns in score_columns_dict.items():
    for column in columns:
        # Extract the metric name (e.g., Correctness, Quality)
        structured_summary["Metric"].append(metric_type.split(" ")[1] if len(metric_type.split(" ")) > 1 else metric_type)

        # Extract AI search engine name
        structured_summary["AI Search Engine"].append(column.split("_")[0].capitalize())

        # Calculate mean score with numeric conversion and NaN handling
        mean_score = pd.to_numeric(df[column], errors="coerce").mean()
        structured_summary["Mean Score"].append(mean_score)

        # Add the judge based on the column name
        structured_summary["Judge"].append(metric_prompt_mapping.get(column, "Unknown Judge"))

        # Add the scale for this column
        structured_summary["Scale"].append(column_scale_mapping.get(column, "Unknown Scale"))

# Convert to DataFrame
structured_summary_df = pd.DataFrame(structured_summary)

# Display the result
structured_summary_df

# Combine the reasoning and numerical grades for quality and correctness into a single DataFrame
quality_combined_columns = [
    "gemini_quality_feedback",
    "perplexity_quality_feedback",
    "exa_quality_feedback",
    "gemini_quality_score",
    "perplexity_quality_score",
    "exa_quality_score"
]

correctness_combined_columns = [
    "gemini_correctness_feedback",
    "perplexity_correctness_feedback",
    "exa_correctness_feedback",
    "gemini_correctness_grade",
    "perplexity_correctness_grade",
    "exa_correctness_grade"
]

# Extract the relevant data
quality_combined = df[quality_combined_columns].dropna().sample(5, random_state=42)
correctness_combined = df[correctness_combined_columns].dropna().sample(5, random_state=42)

quality_combined

correctness_combined