From 290f8cc803ad1d8b17d43177c4a1af730bc672c3 Mon Sep 17 00:00:00 2001 From: "Aleksandar N. Kostadinov" Date: Wed, 10 Dec 2025 22:33:18 +0200 Subject: [PATCH 1/5] remove DEPRECATION --- features/support/hooks.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/features/support/hooks.rb b/features/support/hooks.rb index 4bf7a29514..4eff6149b2 100644 --- a/features/support/hooks.rb +++ b/features/support/hooks.rb @@ -7,7 +7,7 @@ end After do - ActiveRecord::Base.clear_active_connections! + ActiveRecord::Base.connection_handler.clear_active_connections! end Around '@security' do |scenario, block| From 9297bb61b70955d41960a203dd156b9a28ce7419 Mon Sep 17 00:00:00 2001 From: "Aleksandar N. Kostadinov" Date: Wed, 10 Dec 2025 23:03:36 +0200 Subject: [PATCH 2/5] oracle performance --- Gemfile | 2 ++ config/initializers/oracle.rb | 29 +++++++++++++++++++++++++++-- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/Gemfile b/Gemfile index 3d3ccfe5d4..be5f117be0 100644 --- a/Gemfile +++ b/Gemfile @@ -250,6 +250,8 @@ gem 'unicorn', require: false, group: %i[production] # NOTE: Use ENV['DB'] only to install oracle dependencies group :oracle do oracle = -> { (ENV['ORACLE'] == '1') || ENV.fetch('DATABASE_URL', ENV['DB'])&.start_with?('oracle') } + # ENV['NLS_LANG'] ||= 'AMERICAN_AMERICA.AL32UTF8' if oracle + ENV['NLS_LANG'] ||= 'AMERICAN_AMERICA.UTF8' if oracle gem 'activerecord-oracle_enhanced-adapter', '~> 7.1.0', install_if: oracle gem 'ruby-oci8', require: false, install_if: oracle end diff --git a/config/initializers/oracle.rb b/config/initializers/oracle.rb index a40511a009..4e62c65a53 100644 --- a/config/initializers/oracle.rb +++ b/config/initializers/oracle.rb @@ -2,7 +2,7 @@ ActiveSupport.on_load(:active_record) do if System::Database.oracle? - require 'arel/visitors/oracle12_hack' + require 'arel/visitors/oracle12_hack' || next # once done, we can skip setup # in 6.0.6 automatic detection of max identifier length was introduced # see https://github.com/rsim/oracle-enhanced/pull/1703 @@ -30,9 +30,20 @@ def column(name, type, **options) end end) - ENV['NLS_LANG'] ||= 'AMERICAN_AMERICA.UTF8' + # clean-up prepared statements/cursors on connection return to pool + module OracleStatementCleanup + def self.included(base) + base.set_callback :checkin, :after, :close_and_clear_statements + end + + def close_and_clear_statements + @statements&.clear + end + end ActiveRecord::ConnectionAdapters::OracleEnhancedAdapter.class_eval do + include OracleStatementCleanup + # Fixing OCIError: ORA-01741: illegal zero-length identifier # because of https://github.com/rails/rails/commit/c18a95e38e9860953236aed94c1bfb877fa3be84 # the value of `columns` is [ "\"ACCOUNTS\".\"ID\"" ] which forms an incorrect query @@ -273,5 +284,19 @@ def column_definitions(table_name) end ActiveRecord::ConnectionAdapters::OracleEnhancedAdapter.prepend OracleEnhancedAdapterSchemaIssue2276 + + # see https://github.com/kubo/ruby-oci8/pull/271 + module OCI8DisableArrayFetch + private + def define_one_column(pos, param) + @fetch_array_size = nil # disable memory array fetching anytime + super # call original + end + end + + OCI8::Cursor.prepend(OCI8DisableArrayFetch) + + OCI8::BindType::Mapping[:clob] = OCI8::BindType::Long + OCI8::BindType::Mapping[:blob] = OCI8::BindType::LongRaw end end From 0f1b5da77ad6f7d3a6dc7818498c3223895e0eff Mon Sep 17 00:00:00 2001 From: "Aleksandar N. Kostadinov" Date: Fri, 12 Dec 2025 21:50:55 +0200 Subject: [PATCH 3/5] proper fix for writing lobs --- config/initializers/oracle.rb | 67 +++++++++++++++++++++++++---------- 1 file changed, 49 insertions(+), 18 deletions(-) diff --git a/config/initializers/oracle.rb b/config/initializers/oracle.rb index 4e62c65a53..6e4e59adae 100644 --- a/config/initializers/oracle.rb +++ b/config/initializers/oracle.rb @@ -41,8 +41,53 @@ def close_and_clear_statements end end + ActiveRecord::Base.skip_callback(:update, :after, :enhanced_write_lobs) + + # We need to patch Oracle Adapter quoting to actually serialize CLOB columns. + # https://github.com/rsim/oracle-enhanced/issues/1588#issue-272289146 + # The default behaviour is to serialize them to 'empty_clob()' basically wiping out the data. + # The team behind it believes `Table.update_all(column: 'text')` + # should wipe all your data in that column: https://github.com/rsim/oracle-enhanced/issues/1588#issuecomment-343353756 + # So we try to convert the text to using `to_clob` function. + module OracleEnhancedSmartQuoting + CLOB_INLINE_LIMIT = 32767 # 32KB - 1 + BLOB_INLINE_LIMIT = 16383 # 16KB - 1 (hex encoding doubles size) + + def quote(value) + case value + when ActiveModel::Type::Binary::Data, ActiveRecord::Type::OracleEnhanced::Text::Data + raise ArgumentError, "trying to prove that we never reach here" + when ActiveModel::Type::Binary::Data + raw = value.to_s + size = raw.bytesize + + if size == 0 + "empty_blob()" + elsif size <= BLOB_INLINE_LIMIT + "hextoraw('#{raw.unpack1('H*')}')" + else + raise ArgumentError, "BLOB too large for inline quoting (#{size} bytes, max #{BLOB_INLINE_LIMIT} bytes). Use bind parameters instead." + end + when ActiveRecord::Type::OracleEnhanced::Text::Data + text = value.to_s + size = text.bytesize + + if size == 0 + "empty_clob()" + elsif size <= CLOB_INLINE_LIMIT + "to_clob(#{super(text)})" + else + raise ArgumentError, "CLOB too large for inline quoting (#{size} bytes, max #{CLOB_INLINE_LIMIT} bytes). Use bind parameters instead." + end + else + super + end + end + end + ActiveRecord::ConnectionAdapters::OracleEnhancedAdapter.class_eval do include OracleStatementCleanup + prepend OracleEnhancedSmartQuoting # Fixing OCIError: ORA-01741: illegal zero-length identifier # because of https://github.com/rails/rails/commit/c18a95e38e9860953236aed94c1bfb877fa3be84 @@ -76,24 +121,6 @@ def add_column(table_name, column_name, type, **options) end end - # We need to patch Oracle Adapter quoting to actually serialize CLOB columns. - # https://github.com/rsim/oracle-enhanced/issues/1588#issue-272289146 - # The default behaviour is to serialize them to 'empty_clob()' basically wiping out the data. - # The team behind it believes `Table.update_all(column: 'text')` - # should wipe all your data in that column: https://github.com/rsim/oracle-enhanced/issues/1588#issuecomment-343353756 - # So we try to convert the text to using `to_clob` function. - def _quote(value) - case value - when ActiveModel::Type::Binary::Data - # I know this looks ugly, but that just modified copy paste of what the adapter does (minus the rescue). - # It is a bit improved in next version due to ActiveRecord Attributes API. - %{to_blob(#{quote(value.to_s)})} - when ActiveRecord::Type::OracleEnhanced::Text::Data - %{to_clob(#{quote(value.to_s)})} - else - super - end - end end) end @@ -296,6 +323,10 @@ def define_one_column(pos, param) OCI8::Cursor.prepend(OCI8DisableArrayFetch) + # see https://github.com/kubo/ruby-oci8/pull/271 + # Enable piecewise retrieval for both CLOBs and BLOBs + # With the OCIConnectionCursorLobFix above, we can safely use both mappings + # because LOBs are bound as OCI8::CLOB/BLOB objects, not LONG data OCI8::BindType::Mapping[:clob] = OCI8::BindType::Long OCI8::BindType::Mapping[:blob] = OCI8::BindType::LongRaw end From 8172bc17a51d76365c5555345841b9538e5e1855 Mon Sep 17 00:00:00 2001 From: "Aleksandar N. Kostadinov" Date: Mon, 15 Dec 2025 20:52:32 +0200 Subject: [PATCH 4/5] reliable fetch, inline quoting and tests --- config/initializers/oracle.rb | 105 +++++++++----- .../oracle_lob_large_update_test.rb | 129 ++++++++++++++++++ 2 files changed, 198 insertions(+), 36 deletions(-) create mode 100644 test/integration/oracle_lob_large_update_test.rb diff --git a/config/initializers/oracle.rb b/config/initializers/oracle.rb index 6e4e59adae..2a25f8a1d0 100644 --- a/config/initializers/oracle.rb +++ b/config/initializers/oracle.rb @@ -43,48 +43,79 @@ def close_and_clear_statements ActiveRecord::Base.skip_callback(:update, :after, :enhanced_write_lobs) - # We need to patch Oracle Adapter quoting to actually serialize CLOB columns. - # https://github.com/rsim/oracle-enhanced/issues/1588#issue-272289146 - # The default behaviour is to serialize them to 'empty_clob()' basically wiping out the data. - # The team behind it believes `Table.update_all(column: 'text')` - # should wipe all your data in that column: https://github.com/rsim/oracle-enhanced/issues/1588#issuecomment-343353756 - # So we try to convert the text to using `to_clob` function. + # For more information see https://github.com/rsim/oracle-enhanced/pull/2483 module OracleEnhancedSmartQuoting - CLOB_INLINE_LIMIT = 32767 # 32KB - 1 - BLOB_INLINE_LIMIT = 16383 # 16KB - 1 (hex encoding doubles size) + SQL_UTF8_CHUNK_CHARS = 8191 # (32767÷4), 4 bytes max character; 1000 without MAX_STRING_SIZE=EXTENDED + BLOB_INLINE_LIMIT = 16383 # (32767÷2) 2000 without MAX_STRING_SIZE=EXTENDED + PLSQL_BASE64_CHUNK_SIZE = 24_573 def quote(value) case value - when ActiveModel::Type::Binary::Data, ActiveRecord::Type::OracleEnhanced::Text::Data - raise ArgumentError, "trying to prove that we never reach here" when ActiveModel::Type::Binary::Data - raw = value.to_s - size = raw.bytesize - - if size == 0 + data = value.to_s + if data.empty? "empty_blob()" - elsif size <= BLOB_INLINE_LIMIT - "hextoraw('#{raw.unpack1('H*')}')" + elsif data.bytesize <= BLOB_INLINE_LIMIT + "to_blob(hextoraw('#{data.unpack1('H*')}'))" else - raise ArgumentError, "BLOB too large for inline quoting (#{size} bytes, max #{BLOB_INLINE_LIMIT} bytes). Use bind parameters instead." + quote_blob_as_subquery(data) end when ActiveRecord::Type::OracleEnhanced::Text::Data text = value.to_s - size = text.bytesize - - if size == 0 - "empty_clob()" - elsif size <= CLOB_INLINE_LIMIT - "to_clob(#{super(text)})" - else - raise ArgumentError, "CLOB too large for inline quoting (#{size} bytes, max #{CLOB_INLINE_LIMIT} bytes). Use bind parameters instead." - end + text.empty? ? "empty_clob()" : + value.to_s.scan(/.{1,#{SQL_UTF8_CHUNK_CHARS}}/m) + .map { |chunk| "to_clob('#{quote_string(chunk)}')" } + .join(" || ") else super end end + + # Generate a scalar subquery with PL/SQL function to build large BLOBs. + # Uses DBMS_LOB.WRITEAPPEND with base64-encoded chunks for efficiency. + # Testing showed hextoraw() unusable for being more than 100x slower. + def quote_blob_as_subquery(data) + out = +"" + out << "(\n" + out << " WITH FUNCTION make_blob RETURN BLOB IS\n" + out << " l_blob BLOB;\n" + out << " BEGIN\n" + out << " DBMS_LOB.CREATETEMPORARY(l_blob, TRUE, DBMS_LOB.CALL);\n" + offset = 0 + while offset < data.bytesize + chunk = data.byteslice(offset, PLSQL_BASE64_CHUNK_SIZE) + out << " DBMS_LOB.WRITEAPPEND(l_blob, " + out << chunk.bytesize.to_s + out << ", UTL_ENCODE.BASE64_DECODE(UTL_RAW.CAST_TO_RAW('" + out << [chunk].pack("m0") # Base64 encoding without newlines + out << "')));\n" + offset += PLSQL_BASE64_CHUNK_SIZE + end + out << " RETURN l_blob;\n" + out << " END;\n" + out << " SELECT make_blob() FROM dual\n" + out << ")" + out + end end + # this is also needed for inline quoting of large BLOBs work + module OracleEnhancedSmartQuotingPreprocess + # Add /*+ WITH_PLSQL */ hint for INSERT/UPDATE statements containing + # PL/SQL function definitions. Oracle requires this hint for DML + # statements that use PL/SQL in a WITH clause. + # in Rails 8.0 this method was renamed to preprocess_query(sql) + def transform_query(sql) + sql = super + if sql =~ /\A\s*(INSERT|UPDATE)\b(?=.*\bBEGIN\b)/im + sql = sql.sub($1, "#{$1} /*+ WITH_PLSQL */") + end + sql + end + end + + ActiveRecord::ConnectionAdapters::OracleEnhanced::DatabaseStatements.prepend OracleEnhancedSmartQuotingPreprocess + ActiveRecord::ConnectionAdapters::OracleEnhancedAdapter.class_eval do include OracleStatementCleanup prepend OracleEnhancedSmartQuoting @@ -182,12 +213,12 @@ def adapter_for(model) return default.new(model) if default adapter = adapter_type_for(model) - klass = case adapter - when :oracle - ThinkingSphinx::ActiveRecord::DatabaseAdapters::OracleAdapter - else - super - end + klass = case adapter + when :oracle + ThinkingSphinx::ActiveRecord::DatabaseAdapters::OracleAdapter + else + super + end klass.new model end @@ -216,10 +247,10 @@ def adapter_type_for(model) # delta column being within the threshold. In the latter's case, no condition # is needed, so nil is returned. def clause(*args) - model = (args.length >= 2 ? args[0] : nil) + model = (args.length >= 2 ? args[0] : nil) is_delta = (args.length >= 2 ? args[1] : args[0]) || false - table_name = (model.nil? ? adapter.quoted_table_name : model.quoted_table_name) + table_name = (model.nil? ? adapter.quoted_table_name : model.quoted_table_name) column_name = (model.nil? ? adapter.quote(@column.to_s) : model.connection.quote_column_name(@column.to_s)) if is_delta @@ -315,6 +346,7 @@ def column_definitions(table_name) # see https://github.com/kubo/ruby-oci8/pull/271 module OCI8DisableArrayFetch private + def define_one_column(pos, param) @fetch_array_size = nil # disable memory array fetching anytime super # call original @@ -327,7 +359,8 @@ def define_one_column(pos, param) # Enable piecewise retrieval for both CLOBs and BLOBs # With the OCIConnectionCursorLobFix above, we can safely use both mappings # because LOBs are bound as OCI8::CLOB/BLOB objects, not LONG data - OCI8::BindType::Mapping[:clob] = OCI8::BindType::Long - OCI8::BindType::Mapping[:blob] = OCI8::BindType::LongRaw + # Note: disable temporary for issues with NLS_LANG=AMERICAN_AMERICA.UTF8 + # OCI8::BindType::Mapping[:clob] = OCI8::BindType::Long + # OCI8::BindType::Mapping[:blob] = OCI8::BindType::LongRaw end end diff --git a/test/integration/oracle_lob_large_update_test.rb b/test/integration/oracle_lob_large_update_test.rb new file mode 100644 index 0000000000..7ecbfc347b --- /dev/null +++ b/test/integration/oracle_lob_large_update_test.rb @@ -0,0 +1,129 @@ +# frozen_string_literal: true + +require 'test_helper' + +class OracleLobLargeUpdateTest < ActiveSupport::TestCase + # Test large LOB (CLOB and BLOB) updates with OCI8::BindType::Long/LongRaw piecewise retrieval + # This verifies the OCIConnectionCursorLobFix in config/initializers/oracle.rb works correctly + # by using OCI8::CLOB.new() and OCI8::BLOB.new() for direct binding + + setup do + skip "Only run on Oracle database" unless System::Database.oracle? + + @service = FactoryBot.create(:simple_service) + @proxy = @service.proxy + end + + test "update and retrieve large CLOB (policies_config) with 512KB data" do + # Generate ~512KB of JSON data for policies_config + large_policy_data = generate_large_policies_config(2.kilobytes) + + # Initial save using ActiveRecord + @proxy.policies_config = large_policy_data + @proxy.save! + + @proxy.reload + retrieved_policies = @proxy.policies_config + assert retrieved_policies.to_json.bytesize >= 2.kilobytes + + expected_policy = JSON.parse(large_policy_data).first + assert_includes retrieved_policies.map(&:to_h), expected_policy + + # NOW UPDATE with different data + large_policy_data_v2 = generate_large_policies_config(512.kilobytes, version: "2.0") + @proxy.reload + @proxy.policies_config = large_policy_data_v2 + @proxy.save! + + @proxy.reload + retrieved_policies_v2 = @proxy.policies_config + + assert retrieved_policies_v2.to_json.bytesize >= 512.kilobytes + + expected_policy_v2 = JSON.parse(large_policy_data_v2).first + assert_includes retrieved_policies_v2.map(&:to_h), expected_policy_v2 + end + + test "update and retrieve large BLOB (MemberPermission service_ids) with 512KB data" do + # Simple test model double for MemberPermission to avoid the JSON serialization logic + test_class = Class.new(ActiveRecord::Base) do + self.table_name = 'member_permissions' + end + + small_binary_data = Random.bytes(2.kilobytes) + test_record = test_class.create!(service_ids: small_binary_data) + test_record.reload + assert_equal small_binary_data, test_record.service_ids + + # NOW UPDATE with different random binary data + large_binary_data = Random.bytes(512.kilobytes) + test_record.service_ids = large_binary_data + test_record.save! + + test_record.reload + retrieved_value = test_record.service_ids + + assert_equal large_binary_data.bytesize, retrieved_value.bytesize,"Updated binary data size should match" + assert_equal large_binary_data, retrieved_value,"Updated service_ids should match new binary data" + end + + test "test large blobs inline quoting" do + sizes = [ + 15.kilobytes, # within EXTENDED limit + 160.kilobytes # Large size + ] + + sizes.each do |size| + inline_data = Random.bytes(size) + blob_data = ActiveModel::Type::Binary::Data.new(inline_data) + ActiveRecord::Base.connection_pool.with_connection do |conn| + quoted = conn.quote(blob_data) + conn.execute("INSERT INTO member_permissions (id, service_ids) VALUES (member_permissions_seq.nextval, #{quoted})") + res = conn.uncached { conn.select_all("SELECT service_ids FROM member_permissions") } + assert_equal inline_data, res.first["service_ids"] + conn.exec_delete("DELETE member_permissions") + end + end + end + + # practical max is 2GB - 8 bytes; can be increased with a OCI8::BLOB and CLOB fixes to write big data in chunks + test "test large CLOBs inline quoting" do + sizes = [ + 8.kilobytes - 1, # VARCHAR2 within EXTENDED limit + 160.kilobytes # Large size + ] + + sizes.each do |size| + description = "\u20AC" * size # 3 bytes character because we use UTF8 instead of AL32UTF8 + # description = "\u{1F600}" * size # 3 bytes character because we use UTF8 instead of AL32UTF8 + large_data = ActiveRecord::Type::OracleEnhanced::Text::Data.new(description) + conn = ActiveRecord::Base.connection + quoted = conn.quote(large_data) + conn.execute("update services SET description = #{quoted} where id=#{@service.id}") + + actual_description = @service.reload.description + assert_equal description, actual_description, "CLOB size #{size} did not match" + end + end + + private + + # Generate large JSON policies config data + def generate_large_policies_config(target_size, version="1.0") + # Generate a test JSON and calculate actual overhead + test_policy = { + "name" => "test_policy", + "version" => version, + "configuration" => { "data" => "" }, + "enabled" => true + } + + test_json = [test_policy].to_json + padding_size = target_size - test_json.bytesize + + return test_json if padding_size <= 0 + + test_policy["configuration"]["data"] = "X" * padding_size + [test_policy].to_json + end +end From c698c64dadf9ae0f0e7412f6eaa73648c5bce74e Mon Sep 17 00:00:00 2001 From: "Aleksandar N. Kostadinov" Date: Fri, 23 Jan 2026 20:00:51 +0200 Subject: [PATCH 5/5] installing oracle oci on Fedora 43 --- SETUP_ORACLE.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/SETUP_ORACLE.md b/SETUP_ORACLE.md index 7495694ec1..08c7baecc1 100644 --- a/SETUP_ORACLE.md +++ b/SETUP_ORACLE.md @@ -4,8 +4,15 @@ Go to [the official Oracle Instant Client Downloads site](https://www.oracle.com/database/technologies/instant-client/downloads.html) and install basic and SDK RPMs like this: +On Fedora 43, because of required file digests, for version 19 you need: ``` -sudo dnf install https://download.oracle.com/otn_software/linux/instantclient/1918000/oracle-instantclient19.18-basic-19.18.0.0.0-2.x86_64.rpm https://download.oracle.com/otn_software/linux/instantclient/1918000/oracle-instantclient19.18-devel-19.18.0.0.0-2.x86_64.rpm +sudo dnf install -y libnsl +sudo rpm -ivh --nodigest --nofiledigest https://download.oracle.com/otn_software/linux/instantclient/1930000/oracle-instantclient19.30-basic-19.30.0.0.0-1.el9.x86_64.rpm https://download.oracle.com/otn_software/linux/instantclient/1930000/oracle-instantclient19.30-devel-19.30.0.0.0-1.x86_64.rpm +``` + +Earlier can do with dnf only: +``` +sudo dnf install https://download.oracle.com/otn_software/linux/instantclient/1930000/oracle-instantclient19.30-basic-19.30.0.0.0-1.el9.x86_64.rpm https://download.oracle.com/otn_software/linux/instantclient/1930000/oracle-instantclient19.30-devel-19.30.0.0.0-1.x86_64.rpm ``` If you wish, you can also install SQLPLus client from same location as well.