chatwoot · sojan-official · Jul 21, 2023 · May 22, 2023 · May 29, 2023 · Mar 20, 2023
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -12,7 +12,7 @@ defaults: &defaults
     # Specify service dependencies here if necessary
     # CircleCI maintains a library of pre-built images
     # documented at https://circleci.com/docs/2.0/circleci-images/
-    - image: cimg/postgres:14.1
+    - image: cimg/postgres:15.3
     - image: cimg/redis:6.2.6
   environment:
       - RAILS_LOG_TO_STDOUT: false

diff --git a/.env.example b/.env.example
@@ -231,5 +231,10 @@ AZURE_APP_SECRET=
 # control the concurrency setting of sidekiq
 # SIDEKIQ_CONCURRENCY=10
 
+
+# AI powered features
+## OpenAI key
+# OPENAI_API_KEY=
+
 # Sentiment analysis model file path
 SENTIMENT_FILE_PATH=
diff --git a/.github/workflows/run_foss_spec.yml b/.github/workflows/run_foss_spec.yml
@@ -18,11 +18,12 @@ jobs:
     runs-on: ubuntu-20.04
     services:
       postgres:
-        image: postgres:10.8
+        image: postgres:15.3
         env:
           POSTGRES_USER: postgres
           POSTGRES_PASSWORD: ""
           POSTGRES_DB: postgres
+          POSTGRES_HOST_AUTH_METHOD: trust
         ports:
           - 5432:5432
         # needed because the postgres container does not provide a healthcheck

diff --git a/.github/workflows/run_response_bot_spec.yml b/.github/workflows/run_response_bot_spec.yml
@@ -0,0 +1,78 @@
+# #
+# # This workflow will run specs related to response bot
+# # This can only be activated in installations Where vector extension is available.
+# #
+
+name: Run Response Bot spec
+on:
+  push:
+    branches:
+      - develop
+      - master
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  test:
+    runs-on: ubuntu-20.04
+    services:
+      postgres:
+        image: ankane/pgvector
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: ""
+          POSTGRES_DB: postgres
+          POSTGRES_HOST_AUTH_METHOD: trust
+        ports:
+          - 5432:5432
+        # needed because the postgres container does not provide a healthcheck
+        # tmpfs makes DB faster by using RAM
+        options: >-
+          --mount type=tmpfs,destination=/var/lib/postgresql/data
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+      redis:
+        image: redis
+        ports:
+          - 6379:6379
+        options: --entrypoint redis-server
+
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        ref: ${{ github.event.pull_request.head.ref }}
+        repository: ${{ github.event.pull_request.head.repo.full_name }}
+
+    - uses: ruby/setup-ruby@v1
+      with:
+        bundler-cache: true # runs 'bundle install' and caches installed gems automatically
+
+    - uses: actions/setup-node@v3
+      with:
+        node-version: 16
+
+    - name: yarn
+      run: yarn install
+
+    - name: Create database
+      run: bundle exec rake db:create
+
+    - name: Seed database
+      run: bundle exec rake db:schema:load
+
+    - name: Enable ResponseBotService in installation
+      run: RAILS_ENV=test bundle exec rails runner "Features::ResponseBotService.new.enable_in_installation"
+
+    # Run  Response Bot specs
+    - name: Run backend tests
+      run: |
+        bundle exec rspec spec/enterprise/controllers/api/v1/accounts/response_sources_controller_spec.rb spec/enterprise/controllers/enterprise/api/v1/accounts/inboxes_controller_spec.rb:47 --profile=10 --format documentation
+
+    - name: Upload rails log folder
+      uses: actions/upload-artifact@v3
+      if: always()
+      with:
+        name: rails-log-folder
+        path: log
diff --git a/.rubocop.yml b/.rubocop.yml
@@ -87,6 +87,7 @@ Style/ClassAndModuleChildren:
   EnforcedStyle: compact
   Exclude:
     - 'config/application.rb'
+    - 'config/initializers/monkey_patches/*'
 Style/MapToHash:
   Enabled: false
 Style/HashSyntax:

diff --git a/Gemfile b/Gemfile
@@ -165,6 +165,13 @@ gem 'omniauth'
 gem 'omniauth-google-oauth2'
 gem 'omniauth-rails_csrf_protection', '~> 1.0'
 
+## Gems for reponse bot
+# adds cosine similarity to postgres using vector extension
+gem 'neighbor'
+gem 'pgvector'
+# Convert Website HTML to Markdown
+gem 'reverse_markdown'
+
 # Sentiment analysis
 gem 'informers'
 

diff --git a/Gemfile.lock b/Gemfile.lock
@@ -459,6 +459,8 @@ GEM
     multi_json (1.15.0)
     multi_xml (0.6.0)
     multipart-post (2.3.0)
+    neighbor (0.2.3)
+      activerecord (>= 5.2)
     net-http-persistent (4.0.2)
       connection_pool (~> 2.2)
     net-imap (0.3.6)
@@ -532,6 +534,7 @@ GEM
     pg_search (2.3.6)
       activerecord (>= 5.2)
       activesupport (>= 5.2)
+    pgvector (0.1.1)
     procore-sift (1.0.0)
       activerecord (>= 6.1)
     pry (0.14.2)
@@ -617,6 +620,8 @@ GEM
       mime-types (>= 1.16, < 4.0)
       netrc (~> 0.8)
     retriable (3.1.2)
+    reverse_markdown (2.1.1)
+      nokogiri
     rexml (3.2.5)
     rspec-core (3.12.2)
       rspec-support (~> 3.12.0)
@@ -884,6 +889,7 @@ DEPENDENCIES
   lograge (~> 0.12.0)
   maxminddb
   mock_redis
+  neighbor
   newrelic-sidekiq-metrics
   newrelic_rpm
   omniauth
@@ -892,6 +898,7 @@ DEPENDENCIES
   omniauth-rails_csrf_protection (~> 1.0)
   pg
   pg_search
+  pgvector
   procore-sift
   pry-rails
   puma
@@ -905,6 +912,7 @@ DEPENDENCIES
   redis-namespace
   responders
   rest-client
+  reverse_markdown
   rspec-rails
   rspec_junit_formatter
   rubocop

diff --git a/app/models/account.rb b/app/models/account.rb
@@ -151,5 +151,5 @@ def remove_account_sequences
 end
 
 Account.prepend_mod_with('Account')
-Account.include_mod_with('EnterpriseAccountConcern')
+Account.include_mod_with('Concerns::Account')
 Account.include_mod_with('Audit::Account')
diff --git a/app/models/inbox.rb b/app/models/inbox.rb
@@ -172,3 +172,4 @@ def check_channel_type?
 
 Inbox.prepend_mod_with('Inbox')
 Inbox.include_mod_with('Audit::Inbox')
+Inbox.include_mod_with('Concerns::Inbox')
diff --git a/app/policies/inbox_policy.rb b/app/policies/inbox_policy.rb
@@ -38,6 +38,10 @@ def campaigns?
     @account_user.administrator?
   end
 
+  def response_sources?
+    @account_user.administrator?
+  end
+
   def create?
     @account_user.administrator?
   end

diff --git a/app/policies/response_source_policy.rb b/app/policies/response_source_policy.rb
@@ -0,0 +1,17 @@
+class ResponseSourcePolicy < ApplicationPolicy
+  def parse?
+    @account_user.administrator?
+  end
+
+  def create?
+    @account_user.administrator?
+  end
+
+  def add_document?
+    @account_user.administrator?
+  end
+
+  def remove_document?
+    @account_user.administrator?
+  end
+end
diff --git a/app/services/message_templates/hook_execution_service.rb b/app/services/message_templates/hook_execution_service.rb
@@ -70,3 +70,4 @@ def should_send_csat_survey?
     true
   end
 end
+MessageTemplates::HookExecutionService.prepend_mod_with('MessageTemplates::HookExecutionService')
diff --git a/app/views/api/v1/accounts/inboxes/response_sources.json.jbuilder b/app/views/api/v1/accounts/inboxes/response_sources.json.jbuilder
@@ -0,0 +1,3 @@
+json.array! @response_sources do |response_source|
+  json.partial! 'api/v1/models/response_source', formats: [:json], resource: response_source
+end
diff --git a/app/views/api/v1/accounts/response_sources/add_document.json.jbuilder b/app/views/api/v1/accounts/response_sources/add_document.json.jbuilder
@@ -0,0 +1 @@
+json.partial! 'api/v1/models/response_source', formats: [:json], resource: @response_source
diff --git a/app/views/api/v1/accounts/response_sources/create.json.jbuilder b/app/views/api/v1/accounts/response_sources/create.json.jbuilder
@@ -0,0 +1 @@
+json.partial! 'api/v1/models/response_source', formats: [:json], resource: @response_source
diff --git a/app/views/api/v1/accounts/response_sources/remove_document.json.jbuilder b/app/views/api/v1/accounts/response_sources/remove_document.json.jbuilder
@@ -0,0 +1 @@
+json.partial! 'api/v1/models/response_source', formats: [:json], resource: @response_source
diff --git a/app/views/api/v1/models/_response_source.json.jbuilder b/app/views/api/v1/models/_response_source.json.jbuilder
@@ -0,0 +1,16 @@
+json.id resource.id
+json.name resource.name
+json.source_link resource.source_link
+json.source_type resource.source_type
+json.inbox_id resource.inbox_id
+json.account_id resource.account_id
+json.created_at resource.created_at.to_i
+json.updated_at resource.updated_at.to_i
+json.response_documents do
+  json.array! resource.response_documents do |response_document|
+    json.id response_document.id
+    json.document_link response_document.document_link
+    json.created_at response_document.created_at.to_i
+    json.updated_at response_document.updated_at.to_i
+  end
+end
diff --git a/config/features.yml b/config/features.yml
@@ -55,3 +55,5 @@
   enabled: false
 - name: audit_logs
   enabled: false
+- name: response_bot
+  enabled: false
diff --git a/config/initializers/monkey_patches/schema_dumper.rb b/config/initializers/monkey_patches/schema_dumper.rb
@@ -0,0 +1,35 @@
+# When working with experimental extensions, which doesn't have support on all providers
+# This monkey patch will help us to ignore the extensions when dumping the schema
+# Additionally we will also ignore the tables associated with those features and exentions
+
+# Once the feature stabilizes, we can remove the tables/extension from the ignore list
+# Ensure you write appropriate migrations when you do that.
+
+module ActiveRecord
+  module ConnectionAdapters
+    module PostgreSQL
+      class SchemaDumper < ConnectionAdapters::SchemaDumper
+        cattr_accessor :ignore_extentions, default: []
+
+        private
+
+        def extensions(stream)
+          extensions = @connection.extensions
+          return unless extensions.any?
+
+          stream.puts '  # These are extensions that must be enabled in order to support this database'
+          extensions.sort.each do |extension|
+            stream.puts "  enable_extension #{extension.inspect}" unless ignore_extentions.include?(extension)
+          end
+          stream.puts
+        end
+      end
+    end
+  end
+end
+
+## Extentions / Tables to be ignored
+ActiveRecord::ConnectionAdapters::PostgreSQL::SchemaDumper.ignore_extentions << 'vector'
+ActiveRecord::SchemaDumper.ignore_tables << 'responses'
+ActiveRecord::SchemaDumper.ignore_tables << 'response_sources'
+ActiveRecord::SchemaDumper.ignore_tables << 'response_documents'
diff --git a/config/routes.rb b/config/routes.rb
@@ -140,6 +140,7 @@
           resources :inboxes, only: [:index, :show, :create, :update, :destroy] do
             get :assignable_agents, on: :member
             get :campaigns, on: :member
+            get :response_sources, on: :member
             get :agent_bot, on: :member
             post :set_agent_bot, on: :member
             delete :avatar, on: :member
@@ -151,6 +152,15 @@
             end
           end
           resources :labels, only: [:index, :show, :create, :update, :destroy]
+          resources :response_sources, only: [:create] do
+            collection do
+              post :parse
+            end
+            member do
+              post :add_document
+              post :remove_document
+            end
+          end
 
           resources :notifications, only: [:index, :update] do
             collection do

diff --git a/enterprise/app/controllers/api/v1/accounts/response_sources_controller.rb b/enterprise/app/controllers/api/v1/accounts/response_sources_controller.rb
@@ -0,0 +1,34 @@
+class Api::V1::Accounts::ResponseSourcesController < Api::V1::Accounts::BaseController
+  before_action :current_account
+  before_action :check_authorization
+  before_action :find_response_source, only: [:add_document, :remove_document]
+
+  def parse
+    links = PageCrawlerService.new(params[:link]).page_links
+    render json: { links: links }
+  end
+
+  def create
+    @response_source = Current.account.response_sources.new(response_source_params)
+    @response_source.save!
+  end
+
+  def add_document
+    @response_source.response_documents.create!(document_link: params[:document_link])
+  end
+
+  def remove_document
+    @response_source.response_documents.find(params[:document_id]).destroy!
+  end
+
+  private
+
+  def find_response_source
+    @response_source = Current.account.response_sources.find(params[:id])
+  end
+
+  def response_source_params
+    params.require(:response_source).permit(:name, :source_link, :inbox_id,
+                                            response_documents_attributes: [:document_link])
+  end
+end
diff --git a/enterprise/app/controllers/enterprise/api/v1/accounts/inboxes_controller.rb b/enterprise/app/controllers/enterprise/api/v1/accounts/inboxes_controller.rb
@@ -1,4 +1,8 @@
 module Enterprise::Api::V1::Accounts::InboxesController
+  def response_sources
+    @response_sources = @inbox.response_sources
+  end
+
   def inbox_attributes
     super + ee_inbox_attributes
   end

diff --git a/enterprise/app/jobs/response_bot_job.rb b/enterprise/app/jobs/response_bot_job.rb
@@ -0,0 +1,7 @@
+class ResponseBotJob < ApplicationJob
+  queue_as :medium
+
+  def perform(conversation)
+    ::Enterprise::MessageTemplates::ResponseBotService.new(conversation: conversation).perform
+  end
+end
Original file line number	Diff line number	Diff line change
Expand Up		@@ -172,3 +172,4 @@ def check_channel_type?

		Inbox.prepend_mod_with('Inbox')
		Inbox.include_mod_with('Audit::Inbox')
		Inbox.include_mod_with('Concerns::Inbox')
-Original file line number
+Diff line change
@@ Expand Up / @@ -70,3 +70,4 @@ def should_send_csat_survey? @@
         true
       end
     end
+    MessageTemplates::HookExecutionService.prepend_mod_with('MessageTemplates::HookExecutionService')
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		json.partial! 'api/v1/models/response_source', formats: [:json], resource: @response_source