Add validate_with_llm

thmsmlr · Jan 13, 2024 · 3145553 · 3145553
1 parent 16410d5
commit 3145553
Show file tree

Hide file tree

Showing 3 changed files with 114 additions and 1 deletion.
diff --git a/README.md b/README.md
@@ -76,7 +76,7 @@ is_spam?.("Hello I am a Nigerian prince and I would like to send you money")
 Simply create an ecto schema, optionally provide a `@doc` to the schema definition which we pass down to the LLM, then make a call to `Instructor.chat_completion/1` with context about the task you'd like the LLM to complete.
 You can also provide a `validate_changeset/1` function via the `use Instructor.Validator` which will provide a set of code level ecto changeset validations. You can use this in conjunction with `max_retries: 3` to automatically, iteratively go back and forth with the LLM up to `n` times with any validation errors so that it has a chance to fix them.
 
-**Curious to learn more? Unsure of how you'd use this? Check out our extensive set of [tutorials](https://hexdocs.pm/instructor/tutorial.html)**
+**Curious to learn more? Unsure of how you'd use this? Check out our [Gettings Started Guide](https://hexdocs.pm/instructor/tutorial.html)**
 
 * [Tutorial - Basic Usage & Features](https://hexdocs.pm/instructor/tutorial.html)
 * [Text Classification](https://hexdocs.pm/instructor/text-classification.html)

diff --git a/lib/instructor/validator.ex b/lib/instructor/validator.ex
@@ -55,6 +55,75 @@ defmodule Instructor.Validator do
   defmacro __using__(_) do
     quote do
       @behaviour Instructor.Validator
+
+      import Instructor.Validator
+    end
+  end
+
+  defmodule Validation do
+    use Ecto.Schema
+
+    @doc """
+    Validate if an attribute is correct and if not, return an error message
+    """
+    @primary_key false
+    embedded_schema do
+      field(:valid?, :boolean)
+      field(:reason, :string)
     end
   end
+
+  @doc """
+  Validate a changeset field using a language model
+
+  ## Example
+
+    defmodule QuestionAnswer do
+      use Ecto.Schema
+
+      @primary_key false
+      embedded_schema do
+          field :question, :string
+          field :answer, :string
+      end
+
+      @impl true
+      def validate_changeset(changeset) do
+          changeset
+          |> validate_with_llm(:answer, "do not say anything objectionable")
+      end
+    end
+  """
+  def validate_with_llm(changeset, field, statement, opts \\ []) do
+    Ecto.Changeset.validate_change(changeset, field, fn field, value ->
+      {:ok, response} =
+        Instructor.chat_completion(
+          model: Keyword.get(opts, :model, "gpt-3.5-turbo"),
+          temperature: Keyword.get(opts, :temperature, 0),
+          response_model: Validation,
+          messages: [
+            %{
+              role: "system",
+              content: """
+              You are a world class validation model. Capable to determine if the following value is valid for the statement, if it is not, explain why and suggest a new value.
+              """
+            },
+            %{
+              role: "user",
+              content: "Does `#{value}` follow the rules: #{statement}"
+            }
+          ]
+        )
+
+      case response do
+        %Validation{valid?: true} ->
+          []
+
+        %Validation{reason: reason} ->
+          [
+            {field, "is invalid, #{reason}"}
+          ]
+      end
+    end)
+  end
 end
diff --git a/test/instructor_test.exs b/test/instructor_test.exs
@@ -274,4 +274,48 @@ defmodule InstructorTest do
              ]
            ] = Enum.to_list(result)
   end
+
+  defmodule QuestionAnswer do
+    use Ecto.Schema
+    use Instructor.Validator
+
+    @primary_key false
+    embedded_schema do
+      field(:question, :string)
+      field(:answer, :string)
+    end
+
+    @impl true
+    def validate_changeset(changeset) do
+      changeset
+      |> validate_with_llm(:answer, "do not say anything objectionable")
+    end
+  end
+
+  test "llm validator" do
+    TestHelpers.mock_openai_response(%{
+      question: "What is the meaning of life?",
+      answer:
+        "The meaning of life, according to the context, is to live a life of sin and debauchery."
+    })
+
+    TestHelpers.mock_openai_response(%{
+      valid?: false,
+      reason: "The statement promotes sin and debauchery, which is objectionable."
+    })
+
+    result =
+      Instructor.chat_completion(
+        model: "gpt-3.5-turbo",
+        response_model: QuestionAnswer,
+        messages: [
+          %{
+            role: "user",
+            content: "What is the meaning of life?"
+          }
+        ]
+      )
+
+    assert {:error, %Ecto.Changeset{valid?: false}} = result
+  end
 end