Merge pull request #2 from gbaptista/gb-utf8

Fixing encoding issues (incompatible character encodings: UTF-8 and ASCII-8BIT (Encoding::CompatibilityError))
gbaptista · Apr 13, 2024 · 24d05b1 · 24d05b1
2 parents 37dd4b3 + 756ec0f
commit 24d05b1
Show file tree

Hide file tree

Showing 2 changed files with 19 additions and 3 deletions.
diff --git a/controllers/client.rb b/controllers/client.rb
@@ -92,7 +92,7 @@ def request(path, payload = nil, server_sent_events: nil, request_method: 'POST'
 
         method_to_call = request_method.to_s.strip.downcase.to_sym
 
-        partial_json = ''
+        partial_json = String.new.force_encoding('UTF-8')
 
         response = Faraday.new(request: @request_options) do |faraday|
           faraday.adapter @faraday_adapter
@@ -110,7 +110,13 @@ def request(path, payload = nil, server_sent_events: nil, request_method: 'POST'
                 raise_error.on_complete(env.merge(body: chunk))
               end
 
-              partial_json += chunk
+              utf8_chunk = chunk.force_encoding('UTF-8')
+
+              partial_json += if utf8_chunk.valid_encoding?
+                                utf8_chunk
+                              else
+                                utf8_chunk.encode('UTF-8', invalid: :replace, undef: :replace)
+                              end
 
               parsed_json = safe_parse_json(partial_json)
 
@@ -121,7 +127,7 @@ def request(path, payload = nil, server_sent_events: nil, request_method: 'POST'
 
                 results << result
 
-                partial_json = ''
+                partial_json = String.new.force_encoding('UTF-8')
               end
             end
           end

diff --git a/spec/tasks/test-encoding.rb b/spec/tasks/test-encoding.rb
@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+require_relative '../../ports/dsl/ollama-ai'
+
+client = Ollama.new(
+  credentials: { address: 'http://localhost:11434' },
+  options: { server_sent_events: true }
+)
+
+puts client.show({ name: 'yi:latest' })[0]['license']