Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions .github/workflows/miner-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ env:
CI: 1
CACHE_TIMEOUT_MINUTES: 5
SCRIPT_TIMEOUT_MINUTES: 30
MINER_TEST_TIMEOUT_MINUTES: 240
FIL_PROOFS_PARAMETER_CACHE: /var/tmp/filecoin-proof-parameters
SHELL_IMAGE: busybox

Expand Down Expand Up @@ -84,7 +85,7 @@ jobs:
fail-fast: false
matrix:
# The `forest-dev devnet miner --group` values. Add groups as more Lotus itests are ported.
group: [onboarding, wdpost]
group: [onboarding, wdpost, terminate]
env:
# We use a custom Dockerfile for CI to speed up the build process.
FOREST_DOCKERFILE_OVERRIDE: scripts/devnet/forest_ci.dockerfile
Expand Down Expand Up @@ -127,7 +128,7 @@ jobs:
source ./scripts/devnet/test_harness.sh
devnet_test_env_init
forest-dev devnet miner --group "$MINER_TEST_GROUP"
timeout-minutes: 120
timeout-minutes: ${{ fromJSON(env.MINER_TEST_TIMEOUT_MINUTES) }}
- name: Dump docker logs
if: always()
uses: jwalton/gh-docker-logs@v2
Expand Down
3 changes: 3 additions & 0 deletions src/dev/subcommands/devnet_cmd/miner.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
mod api;
mod devnet;
mod onboarding;
mod terminate;
mod wdpost;

use crate::dev::subcommands::tests_cmd::helpers::harness_env;
Expand All @@ -33,6 +34,7 @@ pub struct MinerTestCommand {
enum Group {
Onboarding,
Wdpost,
Terminate,
}

impl MinerTestCommand {
Expand All @@ -59,6 +61,7 @@ fn tests(group: Group) -> Vec<Trial> {
match group {
Group::Onboarding => onboarding::tests(),
Group::Wdpost => wdpost::tests(),
Group::Terminate => terminate::tests(),
}
}

Expand Down
108 changes: 107 additions & 1 deletion src/dev/subcommands/devnet_cmd/miner/api.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,12 @@
//! Lotus Miner JSON-RPC API, which Forest's RPC does not define.

use crate::dev::subcommands::tests_cmd::helpers::{harness_env, poll};
use crate::lotus_json::LotusJson;
use crate::rpc::{ApiPaths, Client, Request};
use crate::shim::address::Address;
use crate::utils::UrlFromMultiAddr;
use anyhow::{Context as _, bail};
use anyhow::{Context as _, bail, ensure};
use cid::Cid;
use itertools::Itertools as _;
use serde::Deserialize;
use serde::de::{DeserializeOwned, IgnoredAny};
Expand Down Expand Up @@ -139,6 +141,110 @@ impl MinerApi {
status.state
)))
}

/// Queues `sector` for early termination.
pub async fn terminate_sector(&self, sector: u64) -> anyhow::Result<()> {
self.call::<IgnoredAny>("Filecoin.SectorTerminate", json!([sector]))
.await?;
Ok(())
}

/// Flushes the terminate batch; returns the message CID when one was sent.
pub async fn terminate_flush(&self) -> anyhow::Result<Option<Cid>> {
Ok(self
.call::<Option<LotusJson<Cid>>>("Filecoin.SectorTerminateFlush", json!([]))
.await?
.map(|c| c.into_inner()))
}

/// Sectors waiting in the terminate batch.
pub async fn terminate_pending(&self) -> anyhow::Result<Vec<u64>> {
let pending: Vec<SectorId> = self
.call("Filecoin.SectorTerminatePending", json!([]))
.await?;
Ok(pending.into_iter().map(|s| s.number).collect())
}

/// After [`Self::terminate_sector`], waits until `sector` is `TerminateWait`,
/// `TerminateFinality`, or `Removed`, flushing the terminate batch once while
/// `Terminating`; a failed terminate exit fails at once. Returns the
/// `TerminateSectors` message CID.
pub async fn wait_until_terminated(&self, sector: u64) -> anyhow::Result<Cid> {
let last_state = AtomicU8::new(u8::MAX);
// Lotus `msgTriggerred`: once-only flush latch via the captured CID.
let terminate_cid = parking_lot::Mutex::new(None::<Cid>);
let waited = poll(&format!("sector {sector} terminate success"), || async {
let status = self.sectors_status(sector).await?;
if last_state.swap(status.state as u8, Ordering::Relaxed) != status.state as u8 {
eprintln!("sector {sector} state: {:?}", status.state);
}
match status.state {
SectorState::Terminating => {
if terminate_cid.lock().is_none() {
let pending = self.terminate_pending().await?;
if pending.is_empty() {
return Ok(None);
}
ensure!(
pending.as_slice() == [sector],
"SectorTerminatePending is {pending:?}, expected exactly [{sector}]"
);
if let Some(cid) = self.terminate_flush().await? {
eprintln!("terminate message: {cid}");
*terminate_cid.lock() = Some(cid);
let pending = self.terminate_pending().await?;
ensure!(
pending.is_empty(),
"SectorTerminatePending not empty after flush: {pending:?}"
);
Comment thread
sudo-shashank marked this conversation as resolved.
Resolved
}
}
Ok(None)
}
SectorState::TerminateWait
| SectorState::TerminateFinality
| SectorState::Removed => {
let cid = terminate_cid.lock().ok_or_else(|| {
anyhow::anyhow!(
"sector {sector} reached {:?} without a terminate message CID",
status.state
)
})?;
Ok(Some(cid))
}
SectorState::TerminateFailed
| SectorState::Removing
| SectorState::RemoveFailed
| SectorState::FailedUnrecoverable => bail!(
"sector {sector} reached {:?}, which is not a successful terminate exit",
status.state
),
_ => Ok(None),
}
})
.await;
let Err(e) = waited else {
return waited;
};
let Ok(status) = self.sectors_status(sector).await else {
return Err(e);
};
let log = status
.log
.iter()
.map(|l| {
if l.trace.is_empty() {
format!("{} {}", l.kind, l.message)
} else {
format!("{} {}\n{}", l.kind, l.message, l.trace)
}
})
.join("\n");
Err(e.context(format!(
"sector {sector} is in {:?}\nsector log:\n{log}",
status.state
)))
}
}

/// Lotus `abi.SectorID`
Expand Down
163 changes: 163 additions & 0 deletions src/dev/subcommands/devnet_cmd/miner/terminate.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,163 @@
// Copyright 2019-2026 ChainSafe Systems
// SPDX-License-Identifier: Apache-2.0, MIT

//! Early termination of a pledged CC sector through Forest's RPC and VM.
//!
//! lotus-miner locates the sector via Forest, then sends `TerminateSectors`.
//!
//! Mirrors Lotus
//! [`TestTerminate`](https://github.com/filecoin-project/lotus/blob/v1.39.1/itests/sector_terminate_test.go#L35).
//! Terminates a pledged sector on the genesis miner (not a pre-seal), so block
//! production stays intact.

use super::api::TestMiner;
use super::devnet::{At, Devnet};
use crate::dev::subcommands::tests_cmd::helpers::{
block_on, poll, poll_until_message_executed, trial,
};
use crate::rpc::prelude::*;
use crate::shim::address::Address;
use crate::shim::sector::SectorSize;
use anyhow::{Context as _, ensure};
use libtest_mimic::Trial;
use num::BigInt;

pub(super) fn tests() -> Vec<Trial> {
vec![trial("terminate::pledged_sector", || {
block_on(pledged_sector())
})]
}

async fn pledged_sector() -> anyhow::Result<()> {
let net = Devnet::connect()?;
let miner = TestMiner::genesis()?;

let at = net.synced().await?;
let before = at
.same(|tsk| StateMinerPower::request((miner.actor, tsk)))
.await?;
let info = at
.same(|tsk| StateMinerInfo::request((miner.actor, tsk)))
.await?;
ensure!(
info.sector_size == SectorSize::_2KiB,
"expected a 2 KiB miner, got {:?}",
info.sector_size
);
let sector_power = BigInt::from(SectorSize::_2KiB as u64);

let sector = miner.api.pledge_sector().await?;
miner.api.wait_until_proving(sector).await?;

let location = net
.synced()
.await?
.same(|tsk| StateSectorPartition::request((miner.actor, sector, tsk)))
.await?;
let at = wait_for_sector_power(
&net,
miner.actor,
sector,
&before.miner_power.raw_byte_power,
)
.await?;
let powered = at
.same(|tsk| StateMinerPower::request((miner.actor, tsk)))
.await?;
let expected_powered = &before.miner_power.raw_byte_power + &sector_power;
ensure!(
powered.miner_power.raw_byte_power == expected_powered,
"after proving sector {sector}, raw power is {}, expected {expected_powered}",
powered.miner_power.raw_byte_power
);

miner.api.terminate_sector(sector).await?;
let terminate_cid = miner.api.wait_until_terminated(sector).await?;
Comment thread
sudo-shashank marked this conversation as resolved.
Resolved
let lookup = poll_until_message_executed(&net.forest, terminate_cid).await?;
ensure!(
lookup.receipt.exit_code().is_success(),
"TerminateSectors {terminate_cid} exited {}",
lookup.receipt.exit_code()
);
let at = net.synced().await?;
let after = at
.same(|tsk| StateMinerPower::request((miner.actor, tsk)))
.await?;
ensure!(
after.miner_power.raw_byte_power == before.miner_power.raw_byte_power,
"after terminating sector {sector}, raw power is {}, expected the pre-pledge {}",
after.miner_power.raw_byte_power,
before.miner_power.raw_byte_power
);

let partitions = at
.same(|tsk| StateMinerPartitions::request((miner.actor, location.deadline, tsk)))
.await?;
let partition = partitions
.get(usize::try_from(location.partition)?)
.with_context(|| {
format!(
"deadline {} lost partition {} after terminate",
location.deadline, location.partition
)
})?;
ensure!(
partition.all_sectors.get(sector),
"sector {sector} missing from AllSectors after terminate"
);
ensure!(
!partition.live_sectors.get(sector),
"sector {sector} still in LiveSectors after terminate"
);
ensure!(
!partition.active_sectors.get(sector),
"sector {sector} still in ActiveSectors after terminate"
);
Ok(())
}

/// Waits until `miner`'s raw power is `baseline` plus one 2 KiB sector, waiting through the
/// sector's next deadline close when prove-commit has not added power yet.
async fn wait_for_sector_power<'a>(
net: &'a Devnet,
miner: Address,
sector: u64,
baseline: &BigInt,
) -> anyhow::Result<At<'a>> {
let target = baseline + BigInt::from(SectorSize::_2KiB as u64);
let at = net.synced().await?;
let power = at
.same(|tsk| StateMinerPower::request((miner, tsk)))
.await?;
if power.miner_power.raw_byte_power >= target {
return Ok(at);
}

let location = at
.same(|tsk| StateSectorPartition::request((miner, sector, tsk)))
.await?;
let current = at
.same(|tsk| StateMinerProvingDeadline::request((miner, tsk)))
.await?
.0;
let deadline = current.for_index(location.deadline).next_not_elapsed();
eprintln!(
"raw power still {}; waiting for deadline {} to close at epoch {} so WindowPoSt can claim sector {sector}",
power.miner_power.raw_byte_power, location.deadline, deadline.close
);
net.wait_for_epoch(deadline.close).await?;
poll(
&format!(
"raw power {target} after deadline {} for sector {sector}",
location.deadline
),
|| async {
let at = net.synced().await?;
let power = at
.same(|tsk| StateMinerPower::request((miner, tsk)))
.await?;
Ok((power.miner_power.raw_byte_power >= target).then_some(at))
},
)
.await
}
4 changes: 2 additions & 2 deletions src/rpc/types/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -393,7 +393,7 @@ lotus_json_with_self!(MinerSectors);
pub struct MinerPartitions {
#[schemars(with = "LotusJson<BitField>")]
#[serde(with = "crate::lotus_json")]
all_sectors: BitField,
pub all_sectors: BitField,
#[schemars(with = "LotusJson<BitField>")]
#[serde(with = "crate::lotus_json")]
faulty_sectors: BitField,
Expand All @@ -405,7 +405,7 @@ pub struct MinerPartitions {
pub live_sectors: BitField,
#[schemars(with = "LotusJson<BitField>")]
#[serde(with = "crate::lotus_json")]
active_sectors: BitField,
pub active_sectors: BitField,
}
lotus_json_with_self!(MinerPartitions);

Expand Down
Loading