Merge branch 'main' into feature/litellm

Giskard-AI · Nov 18, 2024 · 5f51327 · 5f51327
2 parents 7eaf007 + 94b5a37
commit 5f51327
Show file tree

Hide file tree

Showing 4 changed files with 787 additions and 443 deletions.
diff --git a/.github/workflows/build-python.yml b/.github/workflows/build-python.yml
@@ -202,7 +202,7 @@ jobs:
 
       - name: SonarCloud Scan
         if: ${{ matrix.os == 'ubuntu-latest' && matrix.python-version == '3.10' && !matrix.langchain_minimal && !matrix.pandas_v1 && !matrix.pydantic_v1 && (github.event.ref == 'refs/heads/main' || github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) }}
-        uses: SonarSource/sonarcloud-github-action@v3.0.0
+        uses: SonarSource/sonarcloud-github-action@v3.1.0
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
@@ -233,15 +233,14 @@ jobs:
 
   retry-on-failure:
     if: failure() && fromJSON(github.run_attempt) < 2
-    needs: [ build-python ]
+    needs: [build-python]
     runs-on: ubuntu-latest
     steps:
       - env:
           GH_REPO: ${{ github.repository }}
           GH_TOKEN: ${{ github.token }}
         run: gh workflow run retry-workflow.yml --ref ${{ github.ref }} -F run_id=${{ github.run_id }}
 
-
   install-poetry:
     name: "Check if wheel can be installed with using Poetry"
     runs-on: ubuntu-latest

diff --git a/giskard/rag/metrics/ragas_metrics.py b/giskard/rag/metrics/ragas_metrics.py
@@ -2,6 +2,8 @@
 
 import logging
 
+import ragas
+
 from ...llm.client import ChatMessage, LLMClient, get_default_client
 from ...llm.embeddings import BaseEmbedding, get_default_embedding
 from ..base import AgentAnswer
@@ -104,20 +106,38 @@ def __call__(self, question_sample: dict, answer: AgentAnswer) -> dict:
 
         self.metric.init(run_config)
         if self.requires_context and answer.documents is None:
-            logger.warn(
+            logger.warning(
                 f"No retrieved documents are passed to the evaluation function, computation of {self.name} cannot be done without it."
-                "Make sure you pass 'retrieved_documents' to the evaluate function or that the 'answer_fn' return documents alongside the answer."
+                "Make sure that the 'answer_fn' return documents alongside the answer, wrapped by the 'AgentAnswer' class."
             )
             return {self.name: 0}
 
-        ragas_sample = {
-            "question": question_sample["question"],
-            "answer": answer.message,
-            "contexts": answer.documents,
-            "ground_truth": question_sample["reference_answer"],
-        }
+        ragas_sample = self.prepare_ragas_sample(question_sample, answer)
+
         return {self.name: self.metric.score(ragas_sample)}
 
+    @staticmethod
+    def prepare_ragas_sample(question_sample: dict, answer: AgentAnswer) -> dict:
+        if ragas.__version__.startswith("0.1"):
+            logger.warning(
+                f"{DeprecationWarning.__name__}: You are using an older version (v0.1) of `ragas` package. "
+                "Support for v0.1 is deprecated and may be removed in future versions. "
+                "Please consider updating `ragas` to a later version."
+            )
+            return {
+                "question": question_sample["question"],
+                "answer": answer.message,
+                "contexts": answer.documents,
+                "ground_truth": question_sample["reference_answer"],
+            }
+
+        return {
+            "user_input": question_sample["question"],
+            "response": answer.message,
+            "retrieved_contexts": answer.documents,
+            "reference": question_sample["reference_answer"],
+        }
+
 
 ragas_context_precision = RagasMetric(name="RAGAS Context Precision", metric=context_precision, requires_context=True)
 ragas_faithfulness = RagasMetric(name="RAGAS Faithfulness", metric=faithfulness, requires_context=True)