temporal-sdk-migration [2/x]: temporal: support protobuf sharing with the official Ruby SDK

.github/workflows/tests.yml

@@ -26,6 +26,20 @@ jobs:
         run: |
           bundle exec rspec
 
+  test_temporalio_coexistence:
+    runs-on: ubuntu-latest
+    env:
+      BUNDLE_GEMFILE: gemfiles/temporalio.gemfile
+
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '3.4'
+          bundler-cache: true
+      - name: Test protobuf coexistence
+        run: bundle exec rspec spec/unit/lib/temporal/protobuf_loading_spec.rb
+
   test_examples:
     runs-on: ubuntu-latest
 

.gitignore

@@ -1,6 +1,7 @@
 .bundle/
 .yardoc
 Gemfile.lock
+/gemfiles/*.lock
 _yardoc/
 
 /coverage/

Makefile

@@ -5,5 +5,6 @@ PROTO_OUT := lib/gen
 
 proto:
 	$(foreach PROTO_DIR,$(PROTO_DIRS),bundle exec grpc_tools_ruby_protoc -Iproto --ruby_out=$(PROTO_OUT) --grpc_out=$(PROTO_OUT) $(PROTO_DIR)*.proto;)
+	sed -i "/^require 'temporal\// s/$$/ unless ENV['COINBASE_TEMPORAL_RUBY_DISABLE_PROTO_LOAD'] == '1'/" $(PROTO_OUT)/temporal/api/workflowservice/v1/service_services_pb.rb $(PROTO_OUT)/temporal/api/operatorservice/v1/service_services_pb.rb
 
 .PHONY: proto

docs/temporalio-coexistence.md

@@ -0,0 +1,32 @@
+_Written with AI_
+
+# Load both Ruby SDKs in one process
+
+The two SDKs define the same Temporal API protobuf messages. Load the official
+messages first. Then disable this SDK's message loading before requiring it:
+
+```ruby
+require 'temporalio/client'
+ENV['COINBASE_TEMPORAL_RUBY_DISABLE_PROTO_LOAD'] = '1'
+require 'temporal-ruby'
+```
+
+The option is process-wide. Set it before the first Coinbase require. Do not
+change it after boot. The Coinbase gRPC service stubs still load, but use the
+official SDK's message classes. Without the option, Coinbase loading stays
+unchanged. The `temporalio` gem is not a runtime dependency of this gem.
+
+This follows the approach in [Coinbase PR 335](https://github.com/coinbase/temporal-ruby/pull/335).
+It does not make the two Workflow runtimes' histories compatible. Each Task
+Queue must have only one Workflow runtime, even if a process loads both SDKs.
+Do not create official runtimes or clients before a process forks.
+
+Run the protobuf loading and serialization probes with:
+
+```sh
+BUNDLE_GEMFILE=gemfiles/temporalio.gemfile bundle install
+BUNDLE_GEMFILE=gemfiles/temporalio.gemfile bundle exec rspec spec/unit/lib/temporal/protobuf_loading_spec.rb
+```
+
+These probes do not connect to Temporal. Application boot, fork, polling, and
+payload compatibility need separate environment tests.

gemfiles/temporalio.gemfile

@@ -0,0 +1,3 @@
+eval_gemfile '../Gemfile'
+
+gem 'temporalio', '1.10.1', require: false

lib/temporal.rb

@@ -1,5 +1,7 @@
-# Protoc wants all of its generated files on the LOAD_PATH
-$LOAD_PATH << File.expand_path('./gen', __dir__)
+# The official SDK owns API messages when both SDKs share a process.
+unless ENV['COINBASE_TEMPORAL_RUBY_DISABLE_PROTO_LOAD'] == '1'
+  $LOAD_PATH << File.expand_path('./gen', __dir__)
+end
 
 require 'securerandom'
 require 'forwardable'

lib/temporal/connection/grpc.rb

@@ -3,11 +3,15 @@
 require 'google/protobuf/well_known_types'
 require 'securerandom'
 require 'json'
-require 'gen/temporal/api/filter/v1/message_pb'
+unless ENV['COINBASE_TEMPORAL_RUBY_DISABLE_PROTO_LOAD'] == '1'
+  require 'gen/temporal/api/filter/v1/message_pb'
+  require 'gen/temporal/api/enums/v1/workflow_pb'
+  require 'gen/temporal/api/enums/v1/common_pb'
+end
+
+# Coinbase still uses Ruby gRPC stubs with the official SDK's message classes.
 require 'gen/temporal/api/workflowservice/v1/service_services_pb'
 require 'gen/temporal/api/operatorservice/v1/service_services_pb'
-require 'gen/temporal/api/enums/v1/workflow_pb'
-require 'gen/temporal/api/enums/v1/common_pb'
 require 'temporal/connection/errors'
 require 'temporal/connection/interceptors/client_name_version_interceptor'
 require 'temporal/connection/serializer'

lib/temporal/connection/serializer/base.rb

@@ -1,6 +1,8 @@
 require 'oj'
-require 'gen/temporal/api/common/v1/message_pb'
-require 'gen/temporal/api/command/v1/message_pb'
+unless ENV['COINBASE_TEMPORAL_RUBY_DISABLE_PROTO_LOAD'] == '1'
+  require 'gen/temporal/api/common/v1/message_pb'
+  require 'gen/temporal/api/command/v1/message_pb'
+end
 
 module Temporal
   module Connection

lib/temporal/testing/replay_tester.rb

@@ -1,4 +1,6 @@
-require "gen/temporal/api/history/v1/message_pb"
+unless ENV['COINBASE_TEMPORAL_RUBY_DISABLE_PROTO_LOAD'] == '1'
+  require "gen/temporal/api/history/v1/message_pb"
+end
 require "json"
 require "temporal/errors"
 require "temporal/metadata/workflow_task"

spec/unit/lib/temporal/protobuf_loading_spec.rb

@@ -0,0 +1,67 @@
+require 'open3'
+require 'rbconfig'
+
+describe 'Temporal protobuf loading' do
+  let(:lib_dir) { File.expand_path('../../../../lib', __dir__) }
+
+  def run_probe(code, disable_proto_load: nil)
+    Open3.capture3(
+      { 'COINBASE_TEMPORAL_RUBY_DISABLE_PROTO_LOAD' => disable_proto_load },
+      RbConfig.ruby, '-I', lib_dir, '-e', code
+    )
+  end
+
+  let(:exercise_coinbase) do
+    <<~'RUBY'
+      require 'temporal-ruby'
+      require 'temporal/worker'
+      require 'temporal/testing/replay_tester'
+
+      payload_converter = Temporal::Connection::Converter::Payload::ProtoJSON.new
+      execution = Temporalio::Api::Common::V1::WorkflowExecution.new(workflow_id: 'test-id')
+      payload = payload_converter.to_payload(execution)
+      raise 'protobuf payload did not round trip' unless payload_converter.from_payload(payload) == execution
+
+      converter = Temporal::ConverterWrapper.new(
+        Temporal::Configuration::DEFAULT_CONVERTER,
+        Temporal::Configuration::DEFAULT_PAYLOAD_CODEC
+      )
+      command = Temporal::Workflow::Command::CompleteWorkflow.new(result: 'test-result')
+      proto = Temporal::Connection::Serializer.serialize(command, converter)
+      raise 'wrong command type' unless proto.command_type == :COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION
+      result = converter.from_result_payloads(proto.complete_workflow_execution_command_attributes.result)
+      raise 'result did not round trip' unless result == 'test-result'
+
+      workflow_stub = Temporalio::Api::WorkflowService::V1::WorkflowService::Stub
+      operator_stub = Temporalio::Api::OperatorService::V1::OperatorService::Stub
+      raise 'missing Ruby gRPC stubs' unless workflow_stub < GRPC::ClientStub && operator_stub < GRPC::ClientStub
+    RUBY
+  end
+
+  it 'loads its own messages and stubs without the opt-in' do
+    stdout, stderr, status = run_probe(exercise_coinbase + <<~'RUBY')
+      raise 'legacy messages were not loaded' unless $LOADED_FEATURES.any? do |path|
+        path.include?('/gen/temporal/api/common/v1/message_pb.rb')
+      end
+    RUBY
+    expect(status.success?).to eq(true), "#{stdout}\n#{stderr}"
+  end
+
+  it 'uses official messages with Coinbase serializers, worker definitions, and gRPC stubs' do
+    skip 'Run with BUNDLE_GEMFILE=gemfiles/temporalio.gemfile' unless ENV['BUNDLE_GEMFILE']&.end_with?('temporalio.gemfile')
+
+    stdout, stderr, status = run_probe(
+      "require 'temporalio/client'\nrequire 'temporalio/worker'\n" + exercise_coinbase + <<~'RUBY',
+        message_loads = $LOADED_FEATURES.grep(%r{/gen/temporal/api/.*_pb\.rb$}).reject do |path|
+          path.end_with?('_services_pb.rb')
+        end
+        raise "Coinbase messages loaded: #{message_loads}" unless message_loads.empty?
+        raise 'Coinbase gen directory entered the load path' if $LOAD_PATH.any? { |path| path.end_with?('/lib/gen') }
+        raise 'official client not loaded' unless defined?(Temporalio::Client)
+        raise 'official worker not loaded' unless defined?(Temporalio::Worker)
+      RUBY
+      disable_proto_load: '1'
+    )
+    expect(status.success?).to eq(true), "#{stdout}\n#{stderr}"
+  end
+end