CHEF-34003: Support ~/.chef/ruby/<version>/gems for Habitat gem persistence

binstub_patch.rb

@@ -1,4 +1,5 @@
 unless ENV["APPBUNDLER_ALLOW_RVM"]
   ENV["APPBUNDLER_ALLOW_RVM"] = "true"
-  ENV["GEM_PATH"] = [File.expand_path(File.join(__dir__, "..", "vendor")), ENV["GEM_PATH"]].compact.join(File::PATH_SEPARATOR)
+  user_gem_home = File.expand_path(File.join("~", ".chef", "ruby", RbConfig::CONFIG["ruby_version"], "gems"))
+  ENV["GEM_PATH"] = [user_gem_home, File.expand_path(File.join(__dir__, "..", "vendor")), ENV["GEM_PATH"]].compact.join(File::PATH_SEPARATOR)
 end

habitat/plan.ps1

@@ -36,6 +36,11 @@ function Invoke-SetupEnvironment {
     Set-RuntimeEnv FORCE_FFI_YAJL "ext"
     Set-RuntimeEnv LANG "en_US.UTF-8"
     Set-RuntimeEnv LC_CTYPE "en_US.UTF-8"
+
+    # Allow user-installed gems to persist across package upgrades.
+    # The actual GEM_HOME/GEM_PATH will be resolved at runtime to include
+    # ~/.chef/ruby/<ruby_version>/gems via the chef-cli Ruby code.
+    Set-RuntimeEnv CHEF_GEM_HOME_ENABLED "true"
 }
 
 function Invoke-Build {

habitat/plan.sh

@@ -17,6 +17,11 @@ do_setup_environment() {
   set_runtime_env APPBUNDLER_ALLOW_RVM "true" # prevent appbundler from clearing out the carefully constructed runtime GEM_PATH
   set_runtime_env LANG "en_US.UTF-8"
   set_runtime_env LC_CTYPE "en_US.UTF-8"
+
+  # Allow user-installed gems to persist across package upgrades.
+  # The actual GEM_HOME/GEM_PATH will be resolved at runtime via the wrapper
+  # script to include ~/.chef/ruby/<ruby_version>/gems.
+  set_runtime_env CHEF_GEM_HOME_ENABLED "true"
 }
 
 do_prepare() {
@@ -43,6 +48,7 @@ do_build() {
 
     build_line "Setting GEM_PATH=$GEM_HOME"
     export GEM_PATH="$GEM_HOME"
+    bundle config unset with
     bundle config --local without integration deploy maintenance test development profile
     bundle config --local jobs 4
     bundle config --local retry 5
@@ -89,10 +95,17 @@ do_install() {
 #!$(pkg_path_for core/bash)/bin/bash
 set -e
 
-export PATH="$(pkg_path_for ${ruby_pkg})/bin:/sbin:/usr/sbin:/usr/local/sbin:/usr/local/bin:/usr/bin:/bin:$pkg_prefix/vendor/bin:\$PATH"
+# Determine Ruby version for user gem path
+RUBY_ABI_VERSION=\$($(pkg_path_for ${ruby_pkg})/bin/ruby -e 'puts RbConfig::CONFIG["ruby_version"]')
+USER_GEM_HOME="\${HOME}/.chef/ruby/\${RUBY_ABI_VERSION}/gems"
+
+# Create user gem directory if it does not exist
+mkdir -p "\${USER_GEM_HOME}"
+
+export PATH="$(pkg_path_for ${ruby_pkg})/bin:/sbin:/usr/sbin:/usr/local/sbin:/usr/local/bin:/usr/bin:/bin:\${USER_GEM_HOME}/bin:$pkg_prefix/vendor/bin:\$PATH"
 export LD_LIBRARY_PATH="$(pkg_path_for core/libarchive)/lib:\$LD_LIBRARY_PATH"
-export GEM_HOME="$pkg_prefix/vendor"
-export GEM_PATH="$pkg_prefix/vendor"
+export GEM_HOME="\${USER_GEM_HOME}"
+export GEM_PATH="\${USER_GEM_HOME}:$pkg_prefix/vendor"
 
 exec $(pkg_path_for ${ruby_pkg})/bin/ruby $pkg_prefix/libexec/chef-cli "\$@"
 EOF

lib/chef-cli/command/gem.rb

@@ -1,5 +1,4 @@
-#
-#  Copyright (c) 2019-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved.
+# Copyright:: (c) 2019-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved.
 # License:: Apache License, Version 2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -20,28 +19,54 @@
 require "rubygems" unless defined?(Gem)
 require "rubygems/gem_runner"
 require "rubygems/exceptions"
+require "fileutils" unless defined?(FileUtils)
 
 module ChefCLI
   module Command
-
     # Forwards all commands to rubygems.
     class GemForwarder < ChefCLI::Command::Base
       banner "Usage: #{ChefCLI::Dist::EXEC} gem GEM_COMMANDS_AND_OPTIONS"
 
       def run(params)
-        retval = Gem::GemRunner.new.run( params.clone )
+        setup_gem_environment if habitat_gem_home_enabled?
+        retval = Gem::GemRunner.new.run(params.clone)
         retval.nil? || retval
       rescue Gem::SystemExitException => e
-        exit( e.exit_code )
+        exit(e.exit_code)
       end
 
       # Lazy solution: By automatically returning false, we force ChefCLI::Base to
       # call this class' run method, so that Gem::GemRunner can handle the -v flag
       # appropriately (showing the gem version, or installing a specific version
       # of a gem).
-      def needs_version?(params)
+      def needs_version?(_params)
         false
       end
+
+      private
+
+      # Detects whether the user gem home feature is enabled.
+      # This is set via CHEF_GEM_HOME_ENABLED in the Habitat plan's
+      # do_setup_environment/Invoke-SetupEnvironment, or falls back to
+      # habitat_install? detection.
+      def habitat_gem_home_enabled?
+        ENV["CHEF_GEM_HOME_ENABLED"] == "true" || habitat_install?
+      end
+
+      # Sets up GEM_HOME and GEM_PATH to use ~/.chef/ruby/<ruby_version>/gems
+      # when running inside a Habitat-based environment. This ensures gems
+      # persist across Workstation upgrades since the Habitat package path
+      # changes on each upgrade.
+      def setup_gem_environment
+        gem_dir = habitat_user_gem_dir
+        FileUtils.mkdir_p(gem_dir) unless Dir.exist?(gem_dir)
+
+        ENV["GEM_HOME"] = gem_dir
+        # Include existing GEM_PATH so vendor gems remain accessible
+        existing_gem_path = ENV["GEM_PATH"]
+        ENV["GEM_PATH"] = [gem_dir, existing_gem_path].reject { |p| p.nil? || p.empty? }.join(File::PATH_SEPARATOR)
+        Gem.clear_paths
+      end
     end
   end
 end

lib/chef-cli/helpers.rb

@@ -173,20 +173,25 @@ def habitat_env(show_warning: false)
         raise "Error: Could not determine the vendor package prefix. Ensure #{ChefCLI::Dist::HAB_PKG_NAME} is installed and CHEF_CLI_VERSION is set correctly." unless vendor_pkg_prefix
 
         vendor_dir = File.join(vendor_pkg_prefix, "vendor")
+
+        # User gem directory for persistent gem storage across upgrades
+        user_gem_dir = habitat_user_gem_dir
+
         # Construct PATH including Ruby bin directory for chef-cli exec command
         ruby_bin_dir = File.dirname(RbConfig.ruby)
         path = [
           File.join(bin_pkg_prefix, "bin"),
           File.join(vendor_dir, "bin"),
+          File.join(user_gem_dir, "bin"),
           ruby_bin_dir, # Add Ruby bin directory so exec can find gem etc.
           ENV["PATH"].split(File::PATH_SEPARATOR), # Preserve existing PATH
         ].flatten.uniq
 
         {
         "PATH" => path.join(File::PATH_SEPARATOR),
         "GEM_ROOT" => Gem.default_dir, # Default directory for gems
-        "GEM_HOME" => vendor_dir,      # Set only if vendor_dir exists
-        "GEM_PATH" => vendor_dir,      # Set only if vendor_dir exists
+        "GEM_HOME" => user_gem_dir,    # User-local gem dir for persistence
+        "GEM_PATH" => [user_gem_dir, vendor_dir].join(File::PATH_SEPARATOR),
         }
       end
     end
@@ -216,6 +221,14 @@ def get_pkg_prefix(pkg_name)
       path if !path.empty? && Dir.exist?(path) # Return path only if it exists
     end
 
+    # Returns the user-local gem directory for the current Ruby version
+    # under ~/.chef/ruby/<MAJOR.MINOR.0>/gems
+    # This path persists across Habitat package upgrades.
+    def habitat_user_gem_dir
+      ruby_version = RbConfig::CONFIG["ruby_version"] # e.g., "3.1.0"
+      File.expand_path(File.join("~", ".chef", "ruby", ruby_version, "gems"))
+    end
+
     def omnibus_expand_path(*paths)
       dir = File.expand_path(File.join(paths))
       raise OmnibusInstallNotFound.new unless dir && File.directory?(dir)

spec/unit/command/gem_spec.rb

@@ -0,0 +1,175 @@
+#
+# Copyright:: (c) 2019-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "spec_helper"
+require "chef-cli/command/gem"
+
+describe ChefCLI::Command::GemForwarder do
+  let(:command_instance) { described_class.new }
+  let(:gem_runner) { instance_double(Gem::GemRunner) }
+  let(:ruby_version) { RbConfig::CONFIG["ruby_version"] }
+  let(:expected_gem_dir) { File.expand_path("~/.chef/ruby/#{ruby_version}/gems") }
+
+  before do
+    allow(Gem::GemRunner).to receive(:new).and_return(gem_runner)
+  end
+
+  it "has a usage banner" do
+    expect(command_instance.banner).to eq("Usage: chef gem GEM_COMMANDS_AND_OPTIONS")
+  end
+
+  describe "#needs_version?" do
+    it "returns false to let GemRunner handle version flag" do
+      expect(command_instance.needs_version?([])).to be(false)
+    end
+
+    it "returns false even with -v parameter" do
+      expect(command_instance.needs_version?(["-v"])).to be(false)
+    end
+  end
+
+  describe "#run" do
+    context "when NOT in a Habitat environment" do
+      before do
+        allow(command_instance).to receive(:habitat_install?).and_return(false)
+        allow(ENV).to receive(:[]).with("CHEF_GEM_HOME_ENABLED").and_return(nil)
+      end
+
+      it "forwards params to Gem::GemRunner" do
+        expect(gem_runner).to receive(:run).with(%w(install knife)).and_return(true)
+        expect(command_instance.run(%w(install knife))).to eq(true)
+      end
+
+      it "does not modify GEM_HOME" do
+        expect(gem_runner).to receive(:run).with(%w(list)).and_return(true)
+        expect(ENV).not_to receive(:[]=).with("GEM_HOME", anything)
+        command_instance.run(%w(list))
+      end
+
+      it "returns true when GemRunner returns nil" do
+        expect(gem_runner).to receive(:run).with(%w(list)).and_return(nil)
+        expect(command_instance.run(%w(list))).to eq(true)
+      end
+    end
+
+    context "when in a Habitat environment" do
+      let(:vendor_dir) { "/hab/pkgs/chef/chef-cli/1.0.0/123/vendor" }
+      let(:existing_gem_path) { "#{expected_gem_dir}#{File::PATH_SEPARATOR}#{vendor_dir}" }
+
+      before do
+        allow(command_instance).to receive(:habitat_install?).and_return(true)
+        allow(ENV).to receive(:[]).with("CHEF_GEM_HOME_ENABLED").and_return("true")
+        allow(command_instance).to receive(:habitat_user_gem_dir).and_return(expected_gem_dir)
+        allow(ENV).to receive(:[]).with("GEM_PATH").and_return(existing_gem_path)
+        allow(Dir).to receive(:exist?).with(expected_gem_dir).and_return(true)
+        allow(Gem).to receive(:clear_paths)
+      end
+
+      it "sets GEM_HOME to user gem directory" do
+        expect(gem_runner).to receive(:run).with(%w(install knife)).and_return(true)
+        expect(ENV).to receive(:[]=).with("GEM_HOME", expected_gem_dir)
+        allow(ENV).to receive(:[]=).with("GEM_PATH", anything)
+        command_instance.run(%w(install knife))
+      end
+
+      it "sets GEM_PATH to include both user gem dir and existing GEM_PATH" do
+        expect(gem_runner).to receive(:run).with(%w(install knife)).and_return(true)
+        allow(ENV).to receive(:[]=).with("GEM_HOME", expected_gem_dir)
+        expected_gem_path = "#{expected_gem_dir}#{File::PATH_SEPARATOR}#{existing_gem_path}"
+        expect(ENV).to receive(:[]=).with("GEM_PATH", expected_gem_path)
+        command_instance.run(%w(install knife))
+      end
+
+      it "clears Gem paths after setting environment" do
+        expect(gem_runner).to receive(:run).with(%w(install knife)).and_return(true)
+        allow(ENV).to receive(:[]=)
+        expect(Gem).to receive(:clear_paths)
+        command_instance.run(%w(install knife))
+      end
+
+      it "creates the gem directory if it doesn't exist" do
+        allow(Dir).to receive(:exist?).with(expected_gem_dir).and_return(false)
+        expect(FileUtils).to receive(:mkdir_p).with(expected_gem_dir)
+        allow(ENV).to receive(:[]=)
+        expect(gem_runner).to receive(:run).with(%w(install knife)).and_return(true)
+        command_instance.run(%w(install knife))
+      end
+
+      it "does not create the gem directory if it already exists" do
+        allow(Dir).to receive(:exist?).with(expected_gem_dir).and_return(true)
+        expect(FileUtils).not_to receive(:mkdir_p)
+        allow(ENV).to receive(:[]=)
+        expect(gem_runner).to receive(:run).with(%w(install knife)).and_return(true)
+        command_instance.run(%w(install knife))
+      end
+
+      it "forwards all gem subcommands correctly" do
+        allow(ENV).to receive(:[]=)
+        %w(install list uninstall source search update).each do |subcmd|
+          expect(gem_runner).to receive(:run).with([subcmd]).and_return(true)
+          expect(command_instance.run([subcmd])).to eq(true)
+        end
+      end
+    end
+
+    context "when CHEF_GEM_HOME_ENABLED is set but habitat_install? is false" do
+      before do
+        allow(command_instance).to receive(:habitat_install?).and_return(false)
+        allow(ENV).to receive(:[]).with("CHEF_GEM_HOME_ENABLED").and_return("true")
+        allow(command_instance).to receive(:habitat_user_gem_dir).and_return(expected_gem_dir)
+        allow(ENV).to receive(:[]).with("GEM_PATH").and_return(nil)
+        allow(Dir).to receive(:exist?).with(expected_gem_dir).and_return(true)
+        allow(Gem).to receive(:clear_paths)
+      end
+
+      it "still sets up gem environment via env var detection" do
+        expect(gem_runner).to receive(:run).with(%w(install knife)).and_return(true)
+        expect(ENV).to receive(:[]=).with("GEM_HOME", expected_gem_dir)
+        allow(ENV).to receive(:[]=).with("GEM_PATH", anything)
+        command_instance.run(%w(install knife))
+      end
+    end
+
+    context "when GemRunner raises Gem::SystemExitException" do
+      before do
+        allow(command_instance).to receive(:habitat_install?).and_return(false)
+        allow(ENV).to receive(:[]).with("CHEF_GEM_HOME_ENABLED").and_return(nil)
+      end
+
+      it "exits with the exception's exit code" do
+        exception = Gem::SystemExitException.new(1)
+        allow(gem_runner).to receive(:run).and_raise(exception)
+        expect { command_instance.run(%w(install bad_gem)) }.to raise_error(SystemExit) { |e|
+          expect(e.status).to eq(1)
+        }
+      end
+    end
+  end
+
+  describe "#habitat_user_gem_dir" do
+    it "returns ~/.chef/ruby/<version>/gems path" do
+      expect(command_instance.send(:habitat_user_gem_dir)).to eq(expected_gem_dir)
+    end
+
+    it "uses the ruby_version from RbConfig" do
+      allow(RbConfig::CONFIG).to receive(:[]).with("ruby_version").and_return("3.3.0")
+      expect(command_instance.send(:habitat_user_gem_dir)).to eq(
+        File.expand_path("~/.chef/ruby/3.3.0/gems")
+      )
+    end
+  end
+end

spec/unit/command/shell_init_spec.rb

@@ -346,6 +346,8 @@
 
   context "habitat standalone shell-init on bash" do
     let(:cli_hab_path) { "/hab/pkgs/chef/chef-cli/1.0.0/123" }
+    let(:ruby_version) { RbConfig::CONFIG["ruby_version"] }
+    let(:user_gem_dir) { File.expand_path("~/.chef/ruby/#{ruby_version}/gems") }
 
     let(:argv) { ["bash"] }
 
@@ -359,8 +361,8 @@
 
       command_instance.run(argv)
       expect(stdout_io.string).to include("export PATH=\"#{cli_hab_path}/bin")
-      expect(stdout_io.string).to include("export GEM_HOME=\"#{cli_hab_path}/vendor")
-      expect(stdout_io.string).to include("export GEM_PATH=\"#{cli_hab_path}/vendor")
+      expect(stdout_io.string).to include("export GEM_HOME=\"#{user_gem_dir}")
+      expect(stdout_io.string).to include("export GEM_PATH=\"#{user_gem_dir}#{File::PATH_SEPARATOR}#{cli_hab_path}/vendor")
     end
   end
 
@@ -380,10 +382,13 @@
       expect(command_instance).to receive(:get_pkg_prefix).with("chef/chef-workstation").and_return(chef_dke_path)
       expect(command_instance).to receive(:get_pkg_prefix).with("chef/chef-cli").and_return(cli_hab_path)
 
+      ruby_version = RbConfig::CONFIG["ruby_version"]
+      user_gem_dir = File.expand_path("~/.chef/ruby/#{ruby_version}/gems")
+
       command_instance.run(argv)
       expect(stdout_io.string).to include("export PATH=\"#{chef_dke_path}/bin")
-      expect(stdout_io.string).to include("export GEM_HOME=\"#{cli_hab_path}/vendor")
-      expect(stdout_io.string).to include("export GEM_PATH=\"#{cli_hab_path}/vendor")
+      expect(stdout_io.string).to include("export GEM_HOME=\"#{user_gem_dir}")
+      expect(stdout_io.string).to include("export GEM_PATH=\"#{user_gem_dir}#{File::PATH_SEPARATOR}#{cli_hab_path}/vendor")
     end
 
     describe "autocompletion" do