diff --git a/.github/workflows/miner-tests.yml b/.github/workflows/miner-tests.yml index 1ad0b4de46e..577398de667 100644 --- a/.github/workflows/miner-tests.yml +++ b/.github/workflows/miner-tests.yml @@ -19,6 +19,7 @@ env: CI: 1 CACHE_TIMEOUT_MINUTES: 5 SCRIPT_TIMEOUT_MINUTES: 30 + MINER_TEST_TIMEOUT_MINUTES: 240 FIL_PROOFS_PARAMETER_CACHE: /var/tmp/filecoin-proof-parameters SHELL_IMAGE: busybox @@ -84,7 +85,7 @@ jobs: fail-fast: false matrix: # The `forest-dev devnet miner --group` values. Add groups as more Lotus itests are ported. - group: [onboarding, wdpost] + group: [onboarding, wdpost, terminate] env: # We use a custom Dockerfile for CI to speed up the build process. FOREST_DOCKERFILE_OVERRIDE: scripts/devnet/forest_ci.dockerfile @@ -127,7 +128,7 @@ jobs: source ./scripts/devnet/test_harness.sh devnet_test_env_init forest-dev devnet miner --group "$MINER_TEST_GROUP" - timeout-minutes: 120 + timeout-minutes: ${{ fromJSON(env.MINER_TEST_TIMEOUT_MINUTES) }} - name: Dump docker logs if: always() uses: jwalton/gh-docker-logs@v2 diff --git a/src/dev/subcommands/devnet_cmd/miner.rs b/src/dev/subcommands/devnet_cmd/miner.rs index afabf8eabfb..b30cfdd3a51 100644 --- a/src/dev/subcommands/devnet_cmd/miner.rs +++ b/src/dev/subcommands/devnet_cmd/miner.rs @@ -7,6 +7,7 @@ mod api; mod devnet; mod onboarding; +mod terminate; mod wdpost; use crate::dev::subcommands::tests_cmd::helpers::harness_env; @@ -33,6 +34,7 @@ pub struct MinerTestCommand { enum Group { Onboarding, Wdpost, + Terminate, } impl MinerTestCommand { @@ -59,6 +61,7 @@ fn tests(group: Group) -> Vec { match group { Group::Onboarding => onboarding::tests(), Group::Wdpost => wdpost::tests(), + Group::Terminate => terminate::tests(), } } diff --git a/src/dev/subcommands/devnet_cmd/miner/api.rs b/src/dev/subcommands/devnet_cmd/miner/api.rs index 8094adf30ed..17449779499 100644 --- a/src/dev/subcommands/devnet_cmd/miner/api.rs +++ b/src/dev/subcommands/devnet_cmd/miner/api.rs @@ -5,10 +5,12 @@ //! Lotus Miner JSON-RPC API, which Forest's RPC does not define. use crate::dev::subcommands::tests_cmd::helpers::{harness_env, poll}; +use crate::lotus_json::LotusJson; use crate::rpc::{ApiPaths, Client, Request}; use crate::shim::address::Address; use crate::utils::UrlFromMultiAddr; -use anyhow::{Context as _, bail}; +use anyhow::{Context as _, bail, ensure}; +use cid::Cid; use itertools::Itertools as _; use serde::Deserialize; use serde::de::{DeserializeOwned, IgnoredAny}; @@ -139,6 +141,110 @@ impl MinerApi { status.state ))) } + + /// Queues `sector` for early termination. + pub async fn terminate_sector(&self, sector: u64) -> anyhow::Result<()> { + self.call::("Filecoin.SectorTerminate", json!([sector])) + .await?; + Ok(()) + } + + /// Flushes the terminate batch; returns the message CID when one was sent. + pub async fn terminate_flush(&self) -> anyhow::Result> { + Ok(self + .call::>>("Filecoin.SectorTerminateFlush", json!([])) + .await? + .map(|c| c.into_inner())) + } + + /// Sectors waiting in the terminate batch. + pub async fn terminate_pending(&self) -> anyhow::Result> { + let pending: Vec = self + .call("Filecoin.SectorTerminatePending", json!([])) + .await?; + Ok(pending.into_iter().map(|s| s.number).collect()) + } + + /// After [`Self::terminate_sector`], waits until `sector` is `TerminateWait`, + /// `TerminateFinality`, or `Removed`, flushing the terminate batch once while + /// `Terminating`; a failed terminate exit fails at once. Returns the + /// `TerminateSectors` message CID. + pub async fn wait_until_terminated(&self, sector: u64) -> anyhow::Result { + let last_state = AtomicU8::new(u8::MAX); + // Lotus `msgTriggerred`: once-only flush latch via the captured CID. + let terminate_cid = parking_lot::Mutex::new(None::); + let waited = poll(&format!("sector {sector} terminate success"), || async { + let status = self.sectors_status(sector).await?; + if last_state.swap(status.state as u8, Ordering::Relaxed) != status.state as u8 { + eprintln!("sector {sector} state: {:?}", status.state); + } + match status.state { + SectorState::Terminating => { + if terminate_cid.lock().is_none() { + let pending = self.terminate_pending().await?; + if pending.is_empty() { + return Ok(None); + } + ensure!( + pending.as_slice() == [sector], + "SectorTerminatePending is {pending:?}, expected exactly [{sector}]" + ); + if let Some(cid) = self.terminate_flush().await? { + eprintln!("terminate message: {cid}"); + *terminate_cid.lock() = Some(cid); + let pending = self.terminate_pending().await?; + ensure!( + pending.is_empty(), + "SectorTerminatePending not empty after flush: {pending:?}" + ); + } + } + Ok(None) + } + SectorState::TerminateWait + | SectorState::TerminateFinality + | SectorState::Removed => { + let cid = terminate_cid.lock().ok_or_else(|| { + anyhow::anyhow!( + "sector {sector} reached {:?} without a terminate message CID", + status.state + ) + })?; + Ok(Some(cid)) + } + SectorState::TerminateFailed + | SectorState::Removing + | SectorState::RemoveFailed + | SectorState::FailedUnrecoverable => bail!( + "sector {sector} reached {:?}, which is not a successful terminate exit", + status.state + ), + _ => Ok(None), + } + }) + .await; + let Err(e) = waited else { + return waited; + }; + let Ok(status) = self.sectors_status(sector).await else { + return Err(e); + }; + let log = status + .log + .iter() + .map(|l| { + if l.trace.is_empty() { + format!("{} {}", l.kind, l.message) + } else { + format!("{} {}\n{}", l.kind, l.message, l.trace) + } + }) + .join("\n"); + Err(e.context(format!( + "sector {sector} is in {:?}\nsector log:\n{log}", + status.state + ))) + } } /// Lotus `abi.SectorID` diff --git a/src/dev/subcommands/devnet_cmd/miner/terminate.rs b/src/dev/subcommands/devnet_cmd/miner/terminate.rs new file mode 100644 index 00000000000..35832343374 --- /dev/null +++ b/src/dev/subcommands/devnet_cmd/miner/terminate.rs @@ -0,0 +1,163 @@ +// Copyright 2019-2026 ChainSafe Systems +// SPDX-License-Identifier: Apache-2.0, MIT + +//! Early termination of a pledged CC sector through Forest's RPC and VM. +//! +//! lotus-miner locates the sector via Forest, then sends `TerminateSectors`. +//! +//! Mirrors Lotus +//! [`TestTerminate`](https://github.com/filecoin-project/lotus/blob/v1.39.1/itests/sector_terminate_test.go#L35). +//! Terminates a pledged sector on the genesis miner (not a pre-seal), so block +//! production stays intact. + +use super::api::TestMiner; +use super::devnet::{At, Devnet}; +use crate::dev::subcommands::tests_cmd::helpers::{ + block_on, poll, poll_until_message_executed, trial, +}; +use crate::rpc::prelude::*; +use crate::shim::address::Address; +use crate::shim::sector::SectorSize; +use anyhow::{Context as _, ensure}; +use libtest_mimic::Trial; +use num::BigInt; + +pub(super) fn tests() -> Vec { + vec![trial("terminate::pledged_sector", || { + block_on(pledged_sector()) + })] +} + +async fn pledged_sector() -> anyhow::Result<()> { + let net = Devnet::connect()?; + let miner = TestMiner::genesis()?; + + let at = net.synced().await?; + let before = at + .same(|tsk| StateMinerPower::request((miner.actor, tsk))) + .await?; + let info = at + .same(|tsk| StateMinerInfo::request((miner.actor, tsk))) + .await?; + ensure!( + info.sector_size == SectorSize::_2KiB, + "expected a 2 KiB miner, got {:?}", + info.sector_size + ); + let sector_power = BigInt::from(SectorSize::_2KiB as u64); + + let sector = miner.api.pledge_sector().await?; + miner.api.wait_until_proving(sector).await?; + + let location = net + .synced() + .await? + .same(|tsk| StateSectorPartition::request((miner.actor, sector, tsk))) + .await?; + let at = wait_for_sector_power( + &net, + miner.actor, + sector, + &before.miner_power.raw_byte_power, + ) + .await?; + let powered = at + .same(|tsk| StateMinerPower::request((miner.actor, tsk))) + .await?; + let expected_powered = &before.miner_power.raw_byte_power + §or_power; + ensure!( + powered.miner_power.raw_byte_power == expected_powered, + "after proving sector {sector}, raw power is {}, expected {expected_powered}", + powered.miner_power.raw_byte_power + ); + + miner.api.terminate_sector(sector).await?; + let terminate_cid = miner.api.wait_until_terminated(sector).await?; + let lookup = poll_until_message_executed(&net.forest, terminate_cid).await?; + ensure!( + lookup.receipt.exit_code().is_success(), + "TerminateSectors {terminate_cid} exited {}", + lookup.receipt.exit_code() + ); + let at = net.synced().await?; + let after = at + .same(|tsk| StateMinerPower::request((miner.actor, tsk))) + .await?; + ensure!( + after.miner_power.raw_byte_power == before.miner_power.raw_byte_power, + "after terminating sector {sector}, raw power is {}, expected the pre-pledge {}", + after.miner_power.raw_byte_power, + before.miner_power.raw_byte_power + ); + + let partitions = at + .same(|tsk| StateMinerPartitions::request((miner.actor, location.deadline, tsk))) + .await?; + let partition = partitions + .get(usize::try_from(location.partition)?) + .with_context(|| { + format!( + "deadline {} lost partition {} after terminate", + location.deadline, location.partition + ) + })?; + ensure!( + partition.all_sectors.get(sector), + "sector {sector} missing from AllSectors after terminate" + ); + ensure!( + !partition.live_sectors.get(sector), + "sector {sector} still in LiveSectors after terminate" + ); + ensure!( + !partition.active_sectors.get(sector), + "sector {sector} still in ActiveSectors after terminate" + ); + Ok(()) +} + +/// Waits until `miner`'s raw power is `baseline` plus one 2 KiB sector, waiting through the +/// sector's next deadline close when prove-commit has not added power yet. +async fn wait_for_sector_power<'a>( + net: &'a Devnet, + miner: Address, + sector: u64, + baseline: &BigInt, +) -> anyhow::Result> { + let target = baseline + BigInt::from(SectorSize::_2KiB as u64); + let at = net.synced().await?; + let power = at + .same(|tsk| StateMinerPower::request((miner, tsk))) + .await?; + if power.miner_power.raw_byte_power >= target { + return Ok(at); + } + + let location = at + .same(|tsk| StateSectorPartition::request((miner, sector, tsk))) + .await?; + let current = at + .same(|tsk| StateMinerProvingDeadline::request((miner, tsk))) + .await? + .0; + let deadline = current.for_index(location.deadline).next_not_elapsed(); + eprintln!( + "raw power still {}; waiting for deadline {} to close at epoch {} so WindowPoSt can claim sector {sector}", + power.miner_power.raw_byte_power, location.deadline, deadline.close + ); + net.wait_for_epoch(deadline.close).await?; + poll( + &format!( + "raw power {target} after deadline {} for sector {sector}", + location.deadline + ), + || async { + let at = net.synced().await?; + let power = at + .same(|tsk| StateMinerPower::request((miner, tsk))) + .await?; + Ok((power.miner_power.raw_byte_power >= target).then_some(at)) + }, + ) + .await +} diff --git a/src/rpc/types/mod.rs b/src/rpc/types/mod.rs index 5a9be22fcc1..8dc0dc4d471 100644 --- a/src/rpc/types/mod.rs +++ b/src/rpc/types/mod.rs @@ -393,7 +393,7 @@ lotus_json_with_self!(MinerSectors); pub struct MinerPartitions { #[schemars(with = "LotusJson")] #[serde(with = "crate::lotus_json")] - all_sectors: BitField, + pub all_sectors: BitField, #[schemars(with = "LotusJson")] #[serde(with = "crate::lotus_json")] faulty_sectors: BitField, @@ -405,7 +405,7 @@ pub struct MinerPartitions { pub live_sectors: BitField, #[schemars(with = "LotusJson")] #[serde(with = "crate::lotus_json")] - active_sectors: BitField, + pub active_sectors: BitField, } lotus_json_with_self!(MinerPartitions);