Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions ddprof-lib/src/main/cpp/counters.h
Original file line number Diff line number Diff line change
Expand Up @@ -277,6 +277,12 @@
* release is retried on a later call rather than proceeding, but this \
* counts how often that retry path is taken. */ \
X(REFERENCE_CHAIN_TAG_RELEASE_FAILED, "reference_chain_tag_release_failed") \
/* ReferenceChainTracker::runPassManualWalk()'s root/stack-ref enumeration \
* (IterateOverReachableObjects) failed with a JVMTI error - the attempt \
* learned nothing about roots and _root_enum_incomplete holds the terminal \
* ladder; MAX_CONSECUTIVE_ENUM_ERRORS consecutive failures abandon the \
* search with SearchAbandonReason::ROOT_ENUM_ERROR. */ \
X(REFERENCE_CHAIN_ROOT_ENUM_ERRORS, "reference_chain_root_enum_errors") \
/* The profiler-side reference-chain writer could not acquire a \
* sample-record lock within its bounded retry budget and dropped the \
* already-dequeued datadog.ReferenceChain event for this dump - not \
Expand Down
16 changes: 13 additions & 3 deletions ddprof-lib/src/main/cpp/flightRecorder.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2666,9 +2666,19 @@ void Recording::recordReferenceChainAbandoned(
buf->putVar64(event->_start_time);
// SearchAbandonReason (the engine's reason enum) - kept as a small fixed table
// here rather than a T_XXX enum type, mirroring NativeSocketEvent's
// _operation -> kOpNames string mapping above.
static const char *const kReasons[] = {"none", "frontier_cap", "ttl", "canary_stuck"};
buf->putUtf8(event->_reason < 4 ? kReasons[event->_reason] : "unknown");
// _operation -> kOpNames string mapping above. Indexed directly by the
// SearchAbandonReason value; the "unknown" slot at RECORDING_END (4) exists
// only to keep ROOT_ENUM_ERROR (5) at its enum index - RECORDING_END is never
// enqueued as an abandonment event (see its own comment), so that slot's
// string is never actually written. Without its entry, ROOT_ENUM_ERROR - a
// genuinely distinct, engine-reported abandonment - would serialize as
// "unknown" and misattribute the cause in the JFR event.
static const char *const kReasons[] = {"none", "frontier_cap", "ttl",
"canary_stuck", "unknown",
"root_enum_error"};
buf->putUtf8(event->_reason < (u8)(sizeof(kReasons) / sizeof(kReasons[0]))
? kReasons[event->_reason]
: "unknown");
buf->putVar32(event->_passes_run);
buf->putVar32(event->_frontier_size);
buf->putVar32(event->_hop_cap);
Expand Down
11 changes: 11 additions & 0 deletions ddprof-lib/src/main/cpp/javaApi.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1228,6 +1228,13 @@ Java_com_datadoghq_profiler_JavaProfiler_runReferenceChainPass0(
return ReferenceChainTracker::instance()->runPassSerialized(jvmti, env);
}

extern "C" DLLEXPORT void JNICALL
Java_com_datadoghq_profiler_JavaProfiler_setRootEnumErrorInjection0(
JNIEnv *env, jclass unused, jboolean enabled) {
ReferenceChainTracker::instance()->setRootEnumErrorInjectionForTest(
enabled == JNI_TRUE);
}

extern "C" DLLEXPORT void JNICALL
Java_com_datadoghq_profiler_JavaProfiler_pollReferenceChainTargets0(
JNIEnv *env, jclass unused) {
Expand Down Expand Up @@ -1586,6 +1593,10 @@ Java_com_datadoghq_profiler_JavaProfiler_runReferenceChainPass0(
return JNI_FALSE;
}

extern "C" DLLEXPORT void JNICALL
Java_com_datadoghq_profiler_JavaProfiler_setRootEnumErrorInjection0(
JNIEnv *env, jclass unused, jboolean enabled) {}

extern "C" DLLEXPORT void JNICALL
Java_com_datadoghq_profiler_JavaProfiler_pollReferenceChainTargets0(
JNIEnv *env, jclass unused) {}
Expand Down
13 changes: 12 additions & 1 deletion ddprof-lib/src/main/cpp/referenceChainTraversal.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -210,7 +210,17 @@ void ReferenceChainTracker::runPassManualWalk(jvmtiEnv *jvmti, JNIEnv *jni,
ctx.frontier_cap_hit = false;

u64 root_enum_start_ns = OS::nanotime();
jvmtiError root_err = jvmti->IterateOverReachableObjects(
jvmtiError root_err;
#ifdef DEBUG
// Debug-build-only error injection (setRootEnumErrorInjectionForTest()): forces this attempt
// to fail with JVMTI_ERROR_OUT_OF_MEMORY without touching the real JVMTI call - a failed
// call makes no progress and leaves user_data unmodified, so skipping it is faithful to the
// failure being simulated. Release builds never set the flag.
if (_root_enum_error_injection.load(std::memory_order_acquire)) {
root_err = JVMTI_ERROR_OUT_OF_MEMORY;
} else
#endif
root_err = jvmti->IterateOverReachableObjects(
heapRootCallback, stackRefCallback, /*object_ref_callback=*/nullptr,
&ctx);
u64 root_enum_safepoint_ns = OS::nanotime() - root_enum_start_ns;
Expand All @@ -233,6 +243,7 @@ void ReferenceChainTracker::runPassManualWalk(jvmtiEnv *jvmti, JNIEnv *jni,
// count feeds the ladder's ROOT_ENUM_ERROR abandon; any successful attempt resets it.
_root_enum_incomplete = true;
_consecutive_enum_errors++;
Counters::increment(REFERENCE_CHAIN_ROOT_ENUM_ERRORS);
*truncated = true;
*frontier_cap_hit = false;
return;
Expand Down
14 changes: 14 additions & 0 deletions ddprof-lib/src/main/cpp/referenceChains.h
Original file line number Diff line number Diff line change
Expand Up @@ -788,6 +788,13 @@ class ReferenceChainTracker {
// ladder abandons the search (ROOT_ENUM_ERROR) - the only bound when ttl=0.
u32 _consecutive_enum_errors = 0;

// Debug-build-only test seam flag: when true, runPassManualWalk()'s root/stack-ref enumeration
// attempt fails with JVMTI_ERROR_OUT_OF_MEMORY without touching the real JVMTI call (see
// setRootEnumErrorInjectionForTest()). Written by a JNI test thread, read on the BFS thread, so
// atomic with release/acquire ordering - a torn/relaxed read could let one pass run without the
// injection the test counted on.
std::atomic<bool> _root_enum_error_injection{false};

ReferenceChainTracker()
: _enabled(false),
_frontier(nullptr), _configured_frontier_cap(0),
Expand Down Expand Up @@ -1298,6 +1305,13 @@ class ReferenceChainTracker {
: OS::nanotime() - (u64)ms_since_last_enum * 1000000ULL;
_last_root_enum_safepoint_ms = last_enum_cost_ms;
}
// Debug-build-only test seam (javaApi.cpp's setRootEnumErrorInjection0()): see
// _root_enum_error_injection's own comment. No engine lock: the flag is atomic; seeding it
// while a driven pass is in flight only shifts which pass sees the injection, which no test
// depends on.
void setRootEnumErrorInjectionForTest(bool enabled) {
_root_enum_error_injection.store(enabled, std::memory_order_release);
}
bool shouldRunRootEnumForTest(u64 now_ns, bool manual_first_pass) const {
MutexLocker engine_guard(_engine_lock);
return shouldRunRootEnum(now_ns, manual_first_pass);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@
* default temp directory as the scratch where the bundled library will be exploded before
* linking.
*/
public static JavaProfiler getInstance() throws IOException {

Check warning on line 82 in ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java

View workflow job for this annotation

GitHub Actions / check-javadoc

no @throws for java.io.IOException

Check warning on line 82 in ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java

View workflow job for this annotation

GitHub Actions / check-javadoc

Check warning on line 82 in ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java

View workflow job for this annotation

GitHub Actions / check-javadoc

no @throws for java.io.IOException

Check warning on line 82 in ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java

View workflow job for this annotation

GitHub Actions / check-javadoc

return getInstance(null, null);
}

Expand All @@ -89,7 +89,7 @@
*
* @param scratchDir directory where the bundled library will be exploded before linking
*/
public static JavaProfiler getInstance(String scratchDir) throws IOException {

Check warning on line 92 in ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java

View workflow job for this annotation

GitHub Actions / check-javadoc

no @throws for java.io.IOException

Check warning on line 92 in ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java

View workflow job for this annotation

GitHub Actions / check-javadoc

Check warning on line 92 in ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java

View workflow job for this annotation

GitHub Actions / check-javadoc

no @throws for java.io.IOException

Check warning on line 92 in ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java

View workflow job for this annotation

GitHub Actions / check-javadoc

return getInstance(null, scratchDir);
}

Expand All @@ -101,7 +101,7 @@
* @param scratchDir directory where the bundled library will be exploded before linking; ignored
* when 'libLocation' is {@literal null}
*/
public static synchronized JavaProfiler getInstance(String libLocation, String scratchDir)

Check warning on line 104 in ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java

View workflow job for this annotation

GitHub Actions / check-javadoc

no @throws for java.io.IOException

Check warning on line 104 in ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java

View workflow job for this annotation

GitHub Actions / check-javadoc

Check warning on line 104 in ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java

View workflow job for this annotation

GitHub Actions / check-javadoc

no @throws for java.io.IOException

Check warning on line 104 in ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java

View workflow job for this annotation

GitHub Actions / check-javadoc

throws IOException {
if (instance != null) {
return instance;
Expand Down Expand Up @@ -197,7 +197,7 @@
}

/** Records the completion of the trace root */
public boolean recordTraceRoot(

Check warning on line 200 in ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java

View workflow job for this annotation

GitHub Actions / check-javadoc

no @PARAM for operation

Check warning on line 200 in ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java

View workflow job for this annotation

GitHub Actions / check-javadoc

no @PARAM for endpoint

Check warning on line 200 in ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java

View workflow job for this annotation

GitHub Actions / check-javadoc

no @PARAM for rootSpanId

Check warning on line 200 in ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java

View workflow job for this annotation

GitHub Actions / check-javadoc

no @PARAM for operation

Check warning on line 200 in ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java

View workflow job for this annotation

GitHub Actions / check-javadoc

no @PARAM for endpoint

Check warning on line 200 in ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java

View workflow job for this annotation

GitHub Actions / check-javadoc

no @PARAM for rootSpanId
long rootSpanId, String endpoint, String operation, int sizeLimit) {
return recordTrace0(rootSpanId, endpoint, operation, sizeLimit);
}
Expand Down Expand Up @@ -667,6 +667,18 @@
*/
static native boolean runReferenceChainPass0();

/**
* Test seam (debug native builds only): when {@code enabled}, forces every reference-chain
* root/stack-ref enumeration attempt ({@code IterateOverReachableObjects}) to fail with a JVMTI
* error, without touching the real JVMTI call. With the injection active the tracker counts
* consecutive failed attempts and abandons with the {@code ROOT_ENUM_ERROR} reason after {@code
* MAX_CONSECUTIVE_ENUM_ERRORS} attempts, so a test can drive the {@code
* datadog.ReferenceChainAbandoned} event's {@code root_enum_error} reason deterministically.
* Restore with {@code enabled = false} before the next test in this shared JVM: the flag is
* process-wide singleton state.
*/
static native void setRootEnumErrorInjection0(boolean enabled);

/**
* Test seam (debug native builds only): runs one poll of ReferenceChainTracker's
* LivenessTracker-to-chain-reconstruction bridging step synchronously - for each current leak
Expand Down
73 changes: 73 additions & 0 deletions ddprof-lib/src/test/cpp/referenceChainJfrRoundtrip_ut.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@
#include "engine.h"
#include "flightRecorder.h"
#include "jfrMetadata.h"
#include "referenceChainFrontier.h"
#include "profiler.h"
#include "tsc.h"
#include "vmEntry.h"
Expand Down Expand Up @@ -535,3 +536,75 @@ TEST_F(ReferenceChainJfrRoundtripTest, DefaultLivenessEventHasZeroLeakTag) {
EXPECT_EQ(0, event.leak_tag);
EXPECT_EQ(0u, event.leak_tag_incarnation);
}

// recordReferenceChainAbandoned()'s kReasons[] table must map every
// SearchAbandonReason value to a distinct string, indexed directly by the
// enum value - a stale table silently serializes a genuinely-distinct
// abandonment as "unknown", misattributing the cause in the
// datadog.ReferenceChainAbandoned event. ROOT_ENUM_ERROR (5) pins the
// table's tail - without its entry, the engine's enum-error abandonment
// (referenceChainTraversal.cpp's error branch) serializes as "unknown";
// an out-of-range value pins the bounds check.
TEST_F(ReferenceChainJfrRoundtripTest,
AbandonedEventSerializesRootEnumErrorReason) {
Arguments args;
ASSERT_FALSE(args.parse("referencechains=true:hops=64:budget=1000"));
args._jfr_options = JFR_SYNC_OPTS;

// Own file, NOT the companion JMC round-trip path - this test's events use
// reason values only (no class dictionary entries), and leaving them on
// the companion path would make the Java parser test resolve unrelated
// state.
const std::string path = chainRoundtripJfrPath() + ".abandoned";
int fd = open(path.c_str(), O_CREAT | O_RDWR | O_TRUNC, 0644);
ASSERT_GE(fd, 0) << "could not open " << path << " for writing";

Recording rec(fd, args);
Buffer *buf = rec.buffer(/*lock_index=*/0);
rec.flushIfNeeded(buf, /*limit=*/0);

// Both events carry _start_time = 0 (single-byte var64) so the reason
// field's offset is computable without decoding the timestamp.
struct ReasonCase {
u8 reason;
const char *expected;
};
const ReasonCase cases[] = {
{SearchAbandonReason::ROOT_ENUM_ERROR, "root_enum_error"},
{250, "unknown"}, // out of table range - bounds check
};
for (const ReasonCase &c : cases) {
ReferenceChainAbandonedEvent event;
event._reason = c.reason;
event._start_time = 0;

int offset_before = buf->offset();
rec.recordReferenceChainAbandoned(buf, &event);
int offset_after = buf->offset();
EXPECT_LE(offset_after, RECORDING_BUFFER_LIMIT);
EXPECT_GT(offset_after, offset_before);

// Field order (recordReferenceChainAbandoned): 1-byte size prefix
// (writeEventSizePrefix patches one byte - this event is always <
// 128 bytes), var64 type id, var64 startTime, then the reason putUtf8
// = 1 encoding-tag byte + var32 length + payload. _start_time = 0
// makes the timestamp var64 exactly one byte.
const char *data = buf->data() + offset_before;
int off = 1; // size prefix
off += sizeOfVar64((u64)T_REFERENCE_CHAIN_ABANDONED);
off += sizeOfVar64(0);
EXPECT_EQ(3, data[off])
<< "reason putUtf8's encoding-tag byte must precede its length";
off += 1;
u32 len = decodeVar32(data, &off);
u32 expected_len = (u32)strlen(c.expected);
ASSERT_EQ(expected_len, len)
<< "reason " << (int)c.reason << " must serialize as "
<< c.expected;
EXPECT_EQ(0, memcmp(data + off, c.expected, expected_len))
<< "reason " << (int)c.reason << " must serialize as "
<< c.expected;
}

unlink(path.c_str());
}
16 changes: 16 additions & 0 deletions ddprof-lib/src/test/cpp/referenceChainsBfsTests.inc
Original file line number Diff line number Diff line change
Expand Up @@ -351,11 +351,20 @@ TEST_F(ReferenceChainsBfsTest, ConsecutiveEnumErrorsAbandonSearch) {
std::pair<u64, u64> saved = ReferenceChainsTestAccessor::setRootEnumRawStateForTest(0, 0);
active_fixture->ior_error = JVMTI_ERROR_OUT_OF_MEMORY;

// Pin the errored-attempt accounting: each failed root enumeration must bump the exported
// reference_chain_root_enum_errors counter (runPassManualWalk's error path). Delta-based:
// the counter is process-global and earlier tests in this binary may have incremented it.
long long enum_errors_before = Counters::getCounter(REFERENCE_CHAIN_ROOT_ENUM_ERRORS);

// Pass 1 (manual): the enumeration attempt errors - incomplete, counter 1, still RUNNING.
bool truncated = true;
ASSERT_TRUE(tracker->runPass(&mock_jvmti, &mock_jni, &truncated));
EXPECT_TRUE(truncated);
EXPECT_EQ(SearchState::RUNNING, tracker->searchState());
EXPECT_EQ(enum_errors_before + 1,
Counters::getCounter(REFERENCE_CHAIN_ROOT_ENUM_ERRORS))
<< "the first errored enumeration attempt must increment "
"reference_chain_root_enum_errors";

// Passes 2..MAX: backdate the floor so each pass runs a real enumeration attempt that errors
// again; the ladder must abandon when the counter reaches MAX_CONSECUTIVE_ENUM_ERRORS.
Expand All @@ -373,6 +382,13 @@ TEST_F(ReferenceChainsBfsTest, ConsecutiveEnumErrorsAbandonSearch) {
ASSERT_TRUE(tracker->runPass(&mock_jvmti, &mock_jni, &truncated));
EXPECT_EQ(SearchState::ABANDONED, tracker->searchState());
EXPECT_EQ(SearchAbandonReason::ROOT_ENUM_ERROR, ReferenceChainsTestAccessor::searchAbandonReasonForTest());
// Every one of the MAX_CONSECUTIVE_ENUM_ERRORS errored attempts - including this cap-reaching
// one - must have been counted, so removing or retargeting the increment in the error path
// fails here.
EXPECT_EQ(enum_errors_before + (long long)ReferenceChainTracker::MAX_CONSECUTIVE_ENUM_ERRORS,
Counters::getCounter(REFERENCE_CHAIN_ROOT_ENUM_ERRORS))
<< "each of the " << ReferenceChainTracker::MAX_CONSECUTIVE_ENUM_ERRORS
<< " consecutive errored attempts must increment reference_chain_root_enum_errors";

ReferenceChainsTestAccessor::setRootEnumRawStateForTest(saved.first, saved.second);
tracker->stop();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,10 @@ public static boolean runReferenceChainPass0() {
return JavaProfiler.runReferenceChainPass0();
}

public static void setRootEnumErrorInjection0(boolean enabled) {
JavaProfiler.setRootEnumErrorInjection0(enabled);
}

public static void setReferenceChainPassSpacingForTest0(long msSinceLastPass, long cadenceMs) {
JavaProfiler.setReferenceChainPassSpacingForTest0(msSinceLastPass, cadenceMs);
}
Expand Down
Loading
Loading