feat: Response Bot using GPT and Webpage Sources (#7518)

This commit introduces the ability to associate response sources to an inbox, allowing external webpages to be parsed by Chatwoot. The parsed data is converted into embeddings for use with GPT models when managing customer queries. The implementation relies on the `pgvector` extension for PostgreSQL. Database migrations related to this feature are handled separately by `Features::ResponseBotService`. A future update will integrate these migrations into the default rails migrations, once compatibility with Postgres extensions across all self-hosted installation options is confirmed. Additionally, a new GitHub action has been added to the CI pipeline to ensure the execution of specs related to this feature.
chatwoot · Jul 21, 2023 · 480f348 · 480f348
1 parent 30f3928
commit 480f348
Show file tree

Hide file tree

Showing 41 changed files with 976 additions and 10 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -12,7 +12,7 @@ defaults: &defaults
     # Specify service dependencies here if necessary
     # CircleCI maintains a library of pre-built images
     # documented at https://circleci.com/docs/2.0/circleci-images/
-    - image: cimg/postgres:14.1
+    - image: cimg/postgres:15.3
     - image: cimg/redis:6.2.6
   environment:
       - RAILS_LOG_TO_STDOUT: false

diff --git a/.env.example b/.env.example
@@ -231,5 +231,10 @@ AZURE_APP_SECRET=
 # control the concurrency setting of sidekiq
 # SIDEKIQ_CONCURRENCY=10
 
+
+# AI powered features
+## OpenAI key
+# OPENAI_API_KEY=
+
 # Sentiment analysis model file path
 SENTIMENT_FILE_PATH=
diff --git a/.github/workflows/run_foss_spec.yml b/.github/workflows/run_foss_spec.yml
@@ -18,11 +18,12 @@ jobs:
     runs-on: ubuntu-20.04
     services:
       postgres:
-        image: postgres:10.8
+        image: postgres:15.3
         env:
           POSTGRES_USER: postgres
           POSTGRES_PASSWORD: ""
           POSTGRES_DB: postgres
+          POSTGRES_HOST_AUTH_METHOD: trust
         ports:
           - 5432:5432
         # needed because the postgres container does not provide a healthcheck

diff --git a/.github/workflows/run_response_bot_spec.yml b/.github/workflows/run_response_bot_spec.yml
@@ -0,0 +1,78 @@
+# #
+# # This workflow will run specs related to response bot
+# # This can only be activated in installations Where vector extension is available.
+# #
+
+name: Run Response Bot spec
+on:
+  push:
+    branches:
+      - develop
+      - master
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  test:
+    runs-on: ubuntu-20.04
+    services:
+      postgres:
+        image: ankane/pgvector
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: ""
+          POSTGRES_DB: postgres
+          POSTGRES_HOST_AUTH_METHOD: trust
+        ports:
+          - 5432:5432
+        # needed because the postgres container does not provide a healthcheck
+        # tmpfs makes DB faster by using RAM
+        options: >-
+          --mount type=tmpfs,destination=/var/lib/postgresql/data
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+      redis:
+        image: redis
+        ports:
+          - 6379:6379
+        options: --entrypoint redis-server
+
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        ref: ${{ github.event.pull_request.head.ref }}
+        repository: ${{ github.event.pull_request.head.repo.full_name }}
+
+    - uses: ruby/setup-ruby@v1
+      with:
+        bundler-cache: true # runs 'bundle install' and caches installed gems automatically
+
+    - uses: actions/setup-node@v3
+      with:
+        node-version: 16
+
+    - name: yarn
+      run: yarn install
+
+    - name: Create database
+      run: bundle exec rake db:create
+
+    - name: Seed database
+      run: bundle exec rake db:schema:load
+
+    - name: Enable ResponseBotService in installation
+      run: RAILS_ENV=test bundle exec rails runner "Features::ResponseBotService.new.enable_in_installation"
+
+    # Run  Response Bot specs
+    - name: Run backend tests
+      run: |
+        bundle exec rspec spec/enterprise/controllers/api/v1/accounts/response_sources_controller_spec.rb spec/enterprise/controllers/enterprise/api/v1/accounts/inboxes_controller_spec.rb:47 --profile=10 --format documentation
+
+    - name: Upload rails log folder
+      uses: actions/upload-artifact@v3
+      if: always()
+      with:
+        name: rails-log-folder
+        path: log
diff --git a/.rubocop.yml b/.rubocop.yml
@@ -87,6 +87,7 @@ Style/ClassAndModuleChildren:
   EnforcedStyle: compact
   Exclude:
     - 'config/application.rb'
+    - 'config/initializers/monkey_patches/*'
 Style/MapToHash:
   Enabled: false
 Style/HashSyntax:

diff --git a/Gemfile b/Gemfile
@@ -165,6 +165,13 @@ gem 'omniauth'
 gem 'omniauth-google-oauth2'
 gem 'omniauth-rails_csrf_protection', '~> 1.0'
 
+## Gems for reponse bot
+# adds cosine similarity to postgres using vector extension
+gem 'neighbor'
+gem 'pgvector'
+# Convert Website HTML to Markdown
+gem 'reverse_markdown'
+
 # Sentiment analysis
 gem 'informers'
 

diff --git a/Gemfile.lock b/Gemfile.lock
@@ -459,6 +459,8 @@ GEM
     multi_json (1.15.0)
     multi_xml (0.6.0)
     multipart-post (2.3.0)
+    neighbor (0.2.3)
+      activerecord (>= 5.2)
     net-http-persistent (4.0.2)
       connection_pool (~> 2.2)
     net-imap (0.3.6)
@@ -532,6 +534,7 @@ GEM
     pg_search (2.3.6)
       activerecord (>= 5.2)
       activesupport (>= 5.2)
+    pgvector (0.1.1)
     procore-sift (1.0.0)
       activerecord (>= 6.1)
     pry (0.14.2)
@@ -617,6 +620,8 @@ GEM
       mime-types (>= 1.16, < 4.0)
       netrc (~> 0.8)
     retriable (3.1.2)
+    reverse_markdown (2.1.1)
+      nokogiri
     rexml (3.2.5)
     rspec-core (3.12.2)
       rspec-support (~> 3.12.0)
@@ -884,6 +889,7 @@ DEPENDENCIES
   lograge (~> 0.12.0)
   maxminddb
   mock_redis
+  neighbor
   newrelic-sidekiq-metrics
   newrelic_rpm
   omniauth
@@ -892,6 +898,7 @@ DEPENDENCIES
   omniauth-rails_csrf_protection (~> 1.0)
   pg
   pg_search
+  pgvector
   procore-sift
   pry-rails
   puma
@@ -905,6 +912,7 @@ DEPENDENCIES
   redis-namespace
   responders
   rest-client
+  reverse_markdown
   rspec-rails
   rspec_junit_formatter
   rubocop

diff --git a/app/models/account.rb b/app/models/account.rb
@@ -151,5 +151,5 @@ def remove_account_sequences
 end
 
 Account.prepend_mod_with('Account')
-Account.include_mod_with('EnterpriseAccountConcern')
+Account.include_mod_with('Concerns::Account')
 Account.include_mod_with('Audit::Account')
diff --git a/app/models/inbox.rb b/app/models/inbox.rb
@@ -172,3 +172,4 @@ def check_channel_type?
 
 Inbox.prepend_mod_with('Inbox')
 Inbox.include_mod_with('Audit::Inbox')
+Inbox.include_mod_with('Concerns::Inbox')
diff --git a/app/policies/inbox_policy.rb b/app/policies/inbox_policy.rb
@@ -38,6 +38,10 @@ def campaigns?
     @account_user.administrator?
   end
 
+  def response_sources?
+    @account_user.administrator?
+  end
+
   def create?
     @account_user.administrator?
   end

diff --git a/app/policies/response_source_policy.rb b/app/policies/response_source_policy.rb
@@ -0,0 +1,17 @@
+class ResponseSourcePolicy < ApplicationPolicy
+  def parse?
+    @account_user.administrator?
+  end
+
+  def create?
+    @account_user.administrator?
+  end
+
+  def add_document?
+    @account_user.administrator?
+  end
+
+  def remove_document?
+    @account_user.administrator?
+  end
+end
diff --git a/app/services/message_templates/hook_execution_service.rb b/app/services/message_templates/hook_execution_service.rb
@@ -70,3 +70,4 @@ def should_send_csat_survey?
     true
   end
 end
+MessageTemplates::HookExecutionService.prepend_mod_with('MessageTemplates::HookExecutionService')
diff --git a/app/views/api/v1/accounts/inboxes/response_sources.json.jbuilder b/app/views/api/v1/accounts/inboxes/response_sources.json.jbuilder
@@ -0,0 +1,3 @@
+json.array! @response_sources do |response_source|
+  json.partial! 'api/v1/models/response_source', formats: [:json], resource: response_source
+end
diff --git a/app/views/api/v1/accounts/response_sources/add_document.json.jbuilder b/app/views/api/v1/accounts/response_sources/add_document.json.jbuilder
@@ -0,0 +1 @@
+json.partial! 'api/v1/models/response_source', formats: [:json], resource: @response_source
diff --git a/app/views/api/v1/accounts/response_sources/create.json.jbuilder b/app/views/api/v1/accounts/response_sources/create.json.jbuilder
@@ -0,0 +1 @@
+json.partial! 'api/v1/models/response_source', formats: [:json], resource: @response_source
diff --git a/app/views/api/v1/accounts/response_sources/remove_document.json.jbuilder b/app/views/api/v1/accounts/response_sources/remove_document.json.jbuilder
@@ -0,0 +1 @@
+json.partial! 'api/v1/models/response_source', formats: [:json], resource: @response_source
diff --git a/app/views/api/v1/models/_response_source.json.jbuilder b/app/views/api/v1/models/_response_source.json.jbuilder
@@ -0,0 +1,16 @@
+json.id resource.id
+json.name resource.name
+json.source_link resource.source_link
+json.source_type resource.source_type
+json.inbox_id resource.inbox_id
+json.account_id resource.account_id
+json.created_at resource.created_at.to_i
+json.updated_at resource.updated_at.to_i
+json.response_documents do
+  json.array! resource.response_documents do |response_document|
+    json.id response_document.id
+    json.document_link response_document.document_link
+    json.created_at response_document.created_at.to_i
+    json.updated_at response_document.updated_at.to_i
+  end
+end
diff --git a/config/features.yml b/config/features.yml
@@ -55,3 +55,5 @@
   enabled: false
 - name: audit_logs
   enabled: false
+- name: response_bot
+  enabled: false
diff --git a/config/initializers/monkey_patches/schema_dumper.rb b/config/initializers/monkey_patches/schema_dumper.rb
@@ -0,0 +1,35 @@
+# When working with experimental extensions, which doesn't have support on all providers
+# This monkey patch will help us to ignore the extensions when dumping the schema
+# Additionally we will also ignore the tables associated with those features and exentions
+
+# Once the feature stabilizes, we can remove the tables/extension from the ignore list
+# Ensure you write appropriate migrations when you do that.
+
+module ActiveRecord
+  module ConnectionAdapters
+    module PostgreSQL
+      class SchemaDumper < ConnectionAdapters::SchemaDumper
+        cattr_accessor :ignore_extentions, default: []
+
+        private
+
+        def extensions(stream)
+          extensions = @connection.extensions
+          return unless extensions.any?
+
+          stream.puts '  # These are extensions that must be enabled in order to support this database'
+          extensions.sort.each do |extension|
+            stream.puts "  enable_extension #{extension.inspect}" unless ignore_extentions.include?(extension)
+          end
+          stream.puts
+        end
+      end
+    end
+  end
+end
+
+## Extentions / Tables to be ignored
+ActiveRecord::ConnectionAdapters::PostgreSQL::SchemaDumper.ignore_extentions << 'vector'
+ActiveRecord::SchemaDumper.ignore_tables << 'responses'
+ActiveRecord::SchemaDumper.ignore_tables << 'response_sources'
+ActiveRecord::SchemaDumper.ignore_tables << 'response_documents'
diff --git a/config/routes.rb b/config/routes.rb
@@ -140,6 +140,7 @@
           resources :inboxes, only: [:index, :show, :create, :update, :destroy] do
             get :assignable_agents, on: :member
             get :campaigns, on: :member
+            get :response_sources, on: :member
             get :agent_bot, on: :member
             post :set_agent_bot, on: :member
             delete :avatar, on: :member
@@ -151,6 +152,15 @@
             end
           end
           resources :labels, only: [:index, :show, :create, :update, :destroy]
+          resources :response_sources, only: [:create] do
+            collection do
+              post :parse
+            end
+            member do
+              post :add_document
+              post :remove_document
+            end
+          end
 
           resources :notifications, only: [:index, :update] do
             collection do

diff --git a/enterprise/app/controllers/api/v1/accounts/response_sources_controller.rb b/enterprise/app/controllers/api/v1/accounts/response_sources_controller.rb
@@ -0,0 +1,34 @@
+class Api::V1::Accounts::ResponseSourcesController < Api::V1::Accounts::BaseController
+  before_action :current_account
+  before_action :check_authorization
+  before_action :find_response_source, only: [:add_document, :remove_document]
+
+  def parse
+    links = PageCrawlerService.new(params[:link]).page_links
+    render json: { links: links }
+  end
+
+  def create
+    @response_source = Current.account.response_sources.new(response_source_params)
+    @response_source.save!
+  end
+
+  def add_document
+    @response_source.response_documents.create!(document_link: params[:document_link])
+  end
+
+  def remove_document
+    @response_source.response_documents.find(params[:document_id]).destroy!
+  end
+
+  private
+
+  def find_response_source
+    @response_source = Current.account.response_sources.find(params[:id])
+  end
+
+  def response_source_params
+    params.require(:response_source).permit(:name, :source_link, :inbox_id,
+                                            response_documents_attributes: [:document_link])
+  end
+end
diff --git a/enterprise/app/controllers/enterprise/api/v1/accounts/inboxes_controller.rb b/enterprise/app/controllers/enterprise/api/v1/accounts/inboxes_controller.rb
@@ -1,4 +1,8 @@
 module Enterprise::Api::V1::Accounts::InboxesController
+  def response_sources
+    @response_sources = @inbox.response_sources
+  end
+
   def inbox_attributes
     super + ee_inbox_attributes
   end

diff --git a/enterprise/app/jobs/response_bot_job.rb b/enterprise/app/jobs/response_bot_job.rb
@@ -0,0 +1,7 @@
+class ResponseBotJob < ApplicationJob
+  queue_as :medium
+
+  def perform(conversation)
+    ::Enterprise::MessageTemplates::ResponseBotService.new(conversation: conversation).perform
+  end
+end
Original file line number	Diff line number	Diff line change
Expand Up		@@ -172,3 +172,4 @@ def check_channel_type?

		Inbox.prepend_mod_with('Inbox')
		Inbox.include_mod_with('Audit::Inbox')
		Inbox.include_mod_with('Concerns::Inbox')
-Original file line number
+Diff line change
@@ Expand Up / @@ -70,3 +70,4 @@ def should_send_csat_survey? @@
         true
       end
     end
+    MessageTemplates::HookExecutionService.prepend_mod_with('MessageTemplates::HookExecutionService')
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		json.partial! 'api/v1/models/response_source', formats: [:json], resource: @response_source