cre-4958: manual shard assignments

.github/workflows/cre-system-tests.yaml

@@ -110,6 +110,12 @@ jobs:
             "Test_CRE_V2_Sharding": [
                {"topology":"workflow-gateway-sharded","configs":"configs/workflow-gateway-sharded-don.toml"}
              ],
+            "Test_CRE_V2_ShardManualAssignment": [
+               {"topology":"workflow-gateway-sharded-manual","configs":"configs/workflow-gateway-sharded-manual.toml"}
+             ],
+            "Test_CRE_V2_ShardRingOCROverrides": [
+               {"topology":"workflow-gateway-sharded-ringocr-overrides","configs":"configs/workflow-gateway-sharded-ringocr-overrides.toml"}
+             ],
             "Test_CRE_V2_Module_Cache": [
                {"topology":"workflow-gateway-cache-test","configs":"configs/workflow-gateway-don-cache-test.toml"}
              ],

core/config/docs/core.toml

@@ -1045,6 +1045,9 @@ ShardOrchestratorPort = 50051 # Default
 # ShardOrchestratorAddress is the URL that the shard orchestration client will try to connect to.
 # Required when ShardingEnabled=true and ShardIndex > 0.
 ShardOrchestratorAddress = '' # Default
+# ShardAssignmentMode controls how workflows are assigned to shards.
+# One of: "ringocr-only" (default), "manual-only", "ringocr-with-overrides".
+ShardAssignmentMode = 'ringocr-only' # Default
 
 # Local Out-Of-Process Plugins
 [LOOPP]

core/config/sharding_config.go

@@ -13,4 +13,5 @@ type Sharding interface {
 	ShardIndex() uint16
 	ShardOrchestratorPort() uint16
 	ShardOrchestratorAddress() *url.URL
+	ShardAssignmentMode() string
 }

core/config/toml/types.go

@@ -3363,8 +3363,15 @@ type Sharding struct {
 	ShardIndex               *uint16
 	ShardOrchestratorPort    *uint16
 	ShardOrchestratorAddress *commonconfig.URL
+	ShardAssignmentMode      *string
 }
 
+const (
+	ShardAssignmentModeRingOCROnly      = "ringocr-only"
+	ShardAssignmentModeManualOnly       = "manual-only"
+	ShardAssignmentModeRingOCROverrides = "ringocr-with-overrides"
+)
+
 func (s *Sharding) setFrom(f *Sharding) {
 	if f.ShardingEnabled != nil {
 		s.ShardingEnabled = f.ShardingEnabled
@@ -3393,18 +3400,27 @@ func (s *Sharding) setFrom(f *Sharding) {
 	if f.ShardOrchestratorAddress != nil {
 		s.ShardOrchestratorAddress = f.ShardOrchestratorAddress
 	}
+
+	if f.ShardAssignmentMode != nil {
+		s.ShardAssignmentMode = f.ShardAssignmentMode
+	}
 }
 
 func (s *Sharding) ValidateConfig() (err error) {
-	// If sharding is enabled and ShardIndex > 0, ShardOrchestratorAddress must be specified
 	if s.ShardingEnabled != nil && *s.ShardingEnabled {
 		if s.ShardIndex != nil && *s.ShardIndex > 0 {
-			if s.ShardOrchestratorAddress == nil || s.ShardOrchestratorAddress.URL() == nil {
-				err = errors.Join(err, configutils.ErrInvalid{
-					Name:  "ShardOrchestratorAddress",
-					Value: s.ShardOrchestratorAddress,
-					Msg:   "must be specified when ShardingEnabled is true and ShardIndex > 0",
-				})
+			mode := ShardAssignmentModeRingOCROnly
+			if s.ShardAssignmentMode != nil {
+				mode = *s.ShardAssignmentMode
+			}
+			if mode != ShardAssignmentModeManualOnly {
+				if s.ShardOrchestratorAddress == nil || s.ShardOrchestratorAddress.URL() == nil {
+					err = errors.Join(err, configutils.ErrInvalid{
+						Name:  "ShardOrchestratorAddress",
+						Value: s.ShardOrchestratorAddress,
+						Msg:   "must be specified when ShardingEnabled is true and ShardIndex > 0",
+					})
+				}
 			}
 		}
 	}

core/scripts/cre/environment/configs/workflow-gateway-sharded-manual.toml

@@ -0,0 +1,131 @@
+
+[chip_router]
+  image = "local-cre-chip-router:v1.0.1"
+
+[[blockchains]]
+  type = "anvil"
+  chain_id = "1337"
+  container_name = "anvil-1337"
+  docker_cmd_params = ["-b", "0.5", "--mixed-mining"]
+
+[[blockchains]]
+  type = "anvil"
+  chain_id = "2337"
+  container_name = "anvil-2337"
+  port = "8546"
+  docker_cmd_params = ["-b", "0.5", "--mixed-mining"]
+
+[jd]
+  csa_encryption_key = "d1093c0060d50a3c89c189b2e485da5a3ce57f3dcb38ab7e2c0d5f0bb2314a44"
+  image = "job-distributor:0.28.0"
+
+[fake]
+  port = 8171
+
+[fake_http]
+  port = 8666
+
+[infra]
+  type = "docker"
+
+[[nodesets]]
+  nodes = 4
+  name = "shard0"
+  don_family = "test-don-family"
+  don_types = ["workflow", "shard"]
+  shard_index = 0
+  override_mode = "each"
+  http_port_range_start = 10100
+
+  env_vars = { CL_EVM_CMD = "", OTEL_SERVICE_NAME = "chainlink-node", CL_CRE_SETTINGS = '{"global":{"PerOrg":{"BaseTriggerRetransmitEnabled":"true"}}}' }
+  capabilities = ["vault", "cron", "http-action", "http-trigger", "consensus", "don-time", "evm-1337", "evm-2337"]
+  registry_based_launch_allowlist = ["cron-trigger@1.0.0"]
+
+  [nodesets.db]
+    image = "postgres:12.0"
+    port = 13000
+
+  [[nodesets.node_specs]]
+    roles = ["plugin"]
+    [nodesets.node_specs.node]
+      docker_ctx = "../../../.."
+      docker_file = "core/chainlink.Dockerfile"
+      docker_build_args = { "CL_IS_PROD_BUILD" = "false" }
+      custom_ports = ["19876:9876"]
+      user_config_overrides = "[Sharding]\nShardAssignmentMode = \"manual-only\"\n"
+
+  [[nodesets.node_specs]]
+    roles = ["plugin"]
+    [nodesets.node_specs.node]
+      docker_ctx = "../../../.."
+      docker_file = "core/chainlink.Dockerfile"
+      docker_build_args = { "CL_IS_PROD_BUILD" = "false" }
+      custom_ports = ["19877:9876"]
+      user_config_overrides = "[Sharding]\nShardAssignmentMode = \"manual-only\"\n"
+
+  [[nodesets.node_specs]]
+    roles = ["plugin"]
+    [nodesets.node_specs.node]
+      docker_ctx = "../../../.."
+      docker_file = "core/chainlink.Dockerfile"
+      docker_build_args = { "CL_IS_PROD_BUILD" = "false" }
+      custom_ports = ["19878:9876"]
+      user_config_overrides = "[Sharding]\nShardAssignmentMode = \"manual-only\"\n"
+
+  [[nodesets.node_specs]]
+    roles = ["plugin"]
+    [nodesets.node_specs.node]
+      docker_ctx = "../../../.."
+      docker_file = "core/chainlink.Dockerfile"
+      docker_build_args = { "CL_IS_PROD_BUILD" = "false" }
+      custom_ports = ["19879:9876"]
+      user_config_overrides = "[Sharding]\nShardAssignmentMode = \"manual-only\"\n"
+
+[[nodesets]]
+  nodes = 4
+  name = "shard1"
+  don_family = "test-don-family"
+  don_types = ["workflow", "shard"]
+  shard_index = 1
+  override_mode = "all"
+  http_port_range_start = 10400
+
+  env_vars = { CL_EVM_CMD = "", OTEL_SERVICE_NAME = "chainlink-node", CL_CRE_SETTINGS = '{"global":{"PerOrg":{"BaseTriggerRetransmitEnabled":"true"}}}' }
+  capabilities = ["vault", "cron", "http-action", "http-trigger", "consensus", "don-time", "evm-1337", "evm-2337"]
+  registry_based_launch_allowlist = ["cron-trigger@1.0.0"]
+
+  [nodesets.db]
+    image = "postgres:12.0"
+    port = 13400
+
+[[nodesets.node_specs]]
+    roles = ["plugin"]
+    [nodesets.node_specs.node]
+      docker_ctx = "../../../.."
+      docker_file = "core/chainlink.Dockerfile"
+      docker_build_args = { "CL_IS_PROD_BUILD" = "false" }
+      user_config_overrides = "[Sharding]\nShardAssignmentMode = \"manual-only\"\n"
+
+[[nodesets]]
+  nodes = 1
+  name = "bootstrap-gateway"
+  don_family = "test-don-family"
+  don_types = ["bootstrap", "gateway"]
+  override_mode = "each"
+  http_port_range_start = 10300
+
+  env_vars = { CL_EVM_CMD = "", OTEL_SERVICE_NAME = "chainlink-node", CL_CRE_SETTINGS = '{"global":{"PerOrg":{"BaseTriggerRetransmitEnabled":"true"}}}' }
+  supported_evm_chains = [1337, 2337]
+
+  [nodesets.db]
+    image = "postgres:12.0"
+    port = 13200
+
+  [[nodesets.node_specs]]
+    roles = ["bootstrap", "gateway"]
+    [nodesets.node_specs.node]
+      docker_ctx = "../../../.."
+      docker_file = "core/chainlink.Dockerfile"
+      docker_build_args = { "CL_IS_PROD_BUILD" = "false" }
+      custom_ports = ["5002:5002","15002:15002"]
+      user_config_overrides = ""

core/scripts/cre/environment/configs/workflow-gateway-sharded-ringocr-overrides.toml

@@ -0,0 +1,131 @@
+
+[chip_router]
+  image = "local-cre-chip-router:v1.0.1"
+
+[[blockchains]]
+  type = "anvil"
+  chain_id = "1337"
+  container_name = "anvil-1337"
+  docker_cmd_params = ["-b", "0.5", "--mixed-mining"]
+
+[[blockchains]]
+  type = "anvil"
+  chain_id = "2337"
+  container_name = "anvil-2337"
+  port = "8546"
+  docker_cmd_params = ["-b", "0.5", "--mixed-mining"]
+
+[jd]
+  csa_encryption_key = "d1093c0060d50a3c89c189b2e485da5a3ce57f3dcb38ab7e2c0d5f0bb2314a44"
+  image = "job-distributor:0.28.0"
+
+[fake]
+  port = 8171
+
+[fake_http]
+  port = 8666
+
+[infra]
+  type = "docker"
+
+[[nodesets]]
+  nodes = 4
+  name = "shard0"
+  don_family = "test-don-family"
+  don_types = ["workflow", "shard"]
+  shard_index = 0
+  override_mode = "each"
+  http_port_range_start = 10100
+
+  env_vars = { CL_EVM_CMD = "", OTEL_SERVICE_NAME = "chainlink-node", CL_CRE_SETTINGS = '{"global":{"PerOrg":{"BaseTriggerRetransmitEnabled":"true"}}}' }
+  capabilities = ["vault", "cron", "http-action", "http-trigger", "consensus", "don-time", "evm-1337", "evm-2337"]
+  registry_based_launch_allowlist = ["cron-trigger@1.0.0"]
+
+  [nodesets.db]
+    image = "postgres:12.0"
+    port = 13000
+
+  [[nodesets.node_specs]]
+    roles = ["plugin"]
+    [nodesets.node_specs.node]
+      docker_ctx = "../../../.."
+      docker_file = "core/chainlink.Dockerfile"
+      docker_build_args = { "CL_IS_PROD_BUILD" = "false" }
+      custom_ports = ["60051:50051", "19876:9876"]
+      user_config_overrides = "[Sharding]\nShardAssignmentMode = \"ringocr-with-overrides\"\n"
+
+  [[nodesets.node_specs]]
+    roles = ["plugin"]
+    [nodesets.node_specs.node]
+      docker_ctx = "../../../.."
+      docker_file = "core/chainlink.Dockerfile"
+      docker_build_args = { "CL_IS_PROD_BUILD" = "false" }
+      custom_ports = ["19877:9876"]
+      user_config_overrides = "[Sharding]\nShardAssignmentMode = \"ringocr-with-overrides\"\n"
+
+  [[nodesets.node_specs]]
+    roles = ["plugin"]
+    [nodesets.node_specs.node]
+      docker_ctx = "../../../.."
+      docker_file = "core/chainlink.Dockerfile"
+      docker_build_args = { "CL_IS_PROD_BUILD" = "false" }
+      custom_ports = ["19878:9876"]
+      user_config_overrides = "[Sharding]\nShardAssignmentMode = \"ringocr-with-overrides\"\n"
+
+  [[nodesets.node_specs]]
+    roles = ["plugin"]
+    [nodesets.node_specs.node]
+      docker_ctx = "../../../.."
+      docker_file = "core/chainlink.Dockerfile"
+      docker_build_args = { "CL_IS_PROD_BUILD" = "false" }
+      custom_ports = ["19879:9876"]
+      user_config_overrides = "[Sharding]\nShardAssignmentMode = \"ringocr-with-overrides\"\n"
+
+[[nodesets]]
+  nodes = 4
+  name = "shard1"
+  don_family = "test-don-family"
+  don_types = ["workflow", "shard"]
+  shard_index = 1
+  override_mode = "all"
+  http_port_range_start = 10400
+
+  env_vars = { CL_EVM_CMD = "", OTEL_SERVICE_NAME = "chainlink-node", CL_CRE_SETTINGS = '{"global":{"PerOrg":{"BaseTriggerRetransmitEnabled":"true"}}}' }
+  capabilities = ["vault", "cron", "http-action", "http-trigger", "consensus", "don-time", "evm-1337", "evm-2337"]
+  registry_based_launch_allowlist = ["cron-trigger@1.0.0"]
+
+  [nodesets.db]
+    image = "postgres:12.0"
+    port = 13400
+
+[[nodesets.node_specs]]
+    roles = ["plugin"]
+    [nodesets.node_specs.node]
+      docker_ctx = "../../../.."
+      docker_file = "core/chainlink.Dockerfile"
+      docker_build_args = { "CL_IS_PROD_BUILD" = "false" }
+      user_config_overrides = "[Sharding]\nShardAssignmentMode = \"ringocr-with-overrides\"\n"
+
+[[nodesets]]
+  nodes = 1
+  name = "bootstrap-gateway"
+  don_family = "test-don-family"
+  don_types = ["bootstrap", "gateway"]
+  override_mode = "each"
+  http_port_range_start = 10300
+
+  env_vars = { CL_EVM_CMD = "", OTEL_SERVICE_NAME = "chainlink-node", CL_CRE_SETTINGS = '{"global":{"PerOrg":{"BaseTriggerRetransmitEnabled":"true"}}}' }
+  supported_evm_chains = [1337, 2337]
+
+  [nodesets.db]
+    image = "postgres:12.0"
+    port = 13200
+
+  [[nodesets.node_specs]]
+    roles = ["bootstrap", "gateway"]
+    [nodesets.node_specs.node]
+      docker_ctx = "../../../.."
+      docker_file = "core/chainlink.Dockerfile"
+      docker_build_args = { "CL_IS_PROD_BUILD" = "false" }
+      custom_ports = ["5002:5002","15002:15002"]
+      user_config_overrides = ""