wrap calls to aws configure set in a file lock to prevent multiprocess file trashing

Author: snickell

lib/aws/google/cached_credentials.rb

@@ -28,7 +28,7 @@ def refresh_if_near_expiration
         @mutex.synchronize do
           if near_expiration?(SYNC_EXPIRATION_LENGTH)
             refresh
-            write_credentials
+            with_write_lock { write_credentials }
           end
         end
       end
@@ -51,6 +51,39 @@ def write_credentials
           system("aws configure set #{key} #{value} --profile #{@session_profile}")
         end
       end
+
+      private
+
+      def now_monotonic
+        Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
+
+      # Lock file ~/.aws/aws-google.lock to prevent multiple simultaneous calls from
+      # different processes from corrupting ~/.aws/config and ~/.aws/credentials
+      def with_write_lock
+        lock_path = aws_google_lock_path
+        start_time = now_monotonic
+
+        # Open aws-google.lock for read/write
+        File.open(lock_path, File::RDWR | File::CREAT) do |lock|
+          # Keep trying to exclusively file lock it every 0.1s until we succeed
+          until lock.flock(File::LOCK_EX | File::LOCK_NB)
+            raise "Timed out after 60s waiting for: #{lock_path}" if now_monotonic - start_time >= 60
+            sleep 0.1
+          end
+
+          yield
+        end
+      end
+
+      def aws_google_lock_path
+        dot_aws_dir = File.dirname(
+          Aws.shared_config.config_path ||
+          Aws.shared_config.credentials_path ||
+          File.expand_path('~/.aws/config')
+        )
+        File.join(dot_aws_dir, 'aws-google.lock')
+      end
     end
   end
 end

test/aws/google_test.rb

@@ -22,10 +22,15 @@
     ENV.stubs(:[]).returns(nil)
   end
 
+  after do
+    lock_path = File.expand_path(File.join(__dir__, 'fixtures', 'aws-google.lock'))
+    File.delete(lock_path) if File.exist?(lock_path)
+  end
+
   describe 'not configured' do
     it 'does nothing' do
       Aws::Google.expects(:new).never
-      Aws::STS::Client.new
+      Aws::STS::Client.new(region: 'us-east-1')
     end
   end
 
@@ -36,7 +41,7 @@
         client_id: 'client_id',
         client_secret: 'client_secret',
         profile: 'cdo',
-        client: Aws::STS::Client.new(stub_responses: true)
+        client: Aws::STS::Client.new(region: 'us-east-1', stub_responses: true)
       }
     end
 
@@ -130,12 +135,56 @@
       Aws::Google.stubs(:config).returns(config)
       @oauth_default.once
       system.times(5)
-      c = Aws::STS::Client.new.config.credentials
+      c = Aws::STS::Client.new(region: 'us-east-1').config.credentials
       _(c.credentials.access_key_id).must_equal credentials[:access_key_id]
       _(c.credentials.secret_access_key).must_equal credentials[:secret_access_key]
       _(c.credentials.session_token).must_equal credentials[:session_token]
     end
 
+    describe 'write lock' do
+      let :provider do
+        Aws::Google.allocate.tap do |google|
+          google.instance_variable_set(:@credentials, Aws::Credentials.new('x', 'y', 'z'))
+          google.instance_variable_set(:@expiration, 123)
+          google.instance_variable_set(:@session_profile, 'cdo_session')
+        end
+      end
+
+      let(:lock_path) { File.expand_path(File.join(__dir__, 'fixtures', 'aws-google.lock')) }
+      let(:lock) { mock }
+
+      it 'writes credentials while holding the lock' do
+        File.expects(:open).with(lock_path, File::RDWR | File::CREAT).yields(lock)
+        lock.expects(:flock).with(File::LOCK_EX | File::LOCK_NB).returns(true)
+        system.times(5)
+
+        provider.send(:with_write_lock) { provider.send(:write_credentials) }
+      end
+
+      it 'retries until the lock is available' do
+        File.expects(:open).with(lock_path, File::RDWR | File::CREAT).yields(lock)
+        lock.expects(:flock).with(File::LOCK_EX | File::LOCK_NB).times(3).returns(false, false, true)
+        Process.stubs(:clock_gettime).with(Process::CLOCK_MONOTONIC).returns(0, 10, 20)
+        provider.expects(:sleep).with(0.1).twice
+        system.times(5)
+
+        provider.send(:with_write_lock) { provider.send(:write_credentials) }
+      end
+
+      it 'raises when the lock times out' do
+        File.expects(:open).with(lock_path, File::RDWR | File::CREAT).yields(lock)
+        lock.expects(:flock).with(File::LOCK_EX | File::LOCK_NB).returns(false)
+        Process.stubs(:clock_gettime).with(Process::CLOCK_MONOTONIC).returns(0, 60)
+        provider.expects(:system).never
+
+        err = assert_raises(RuntimeError) do
+          provider.send(:with_write_lock) { provider.send(:write_credentials) }
+        end
+
+        _(err.message).must_equal "Timed out after 60s waiting for: #{lock_path}"
+      end
+    end
+
     describe 'valid Google auth, no AWS permissions' do
       before do
         config[:client].stub_responses(
@@ -210,8 +259,8 @@
       before do
         Aws.shared_config.fresh(
           config_enabled: true,
-          credentials_path: nil,
-          config_path: nil
+          credentials_path: File.expand_path(File.join(__dir__, 'fixtures', 'missing_aws_credentials')),
+          config_path: File.expand_path(File.join(__dir__, 'fixtures', 'missing_aws_config'))
         )
       end