Skip to content

Add timestamps to management, server and agent .err logs #12967

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
dheeraj12347 wants to merge 5 commits into
base: main
Choose a base branch
from
Open

Add timestamps to management, server and agent .err logs #12967

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
45c3bbe
Add timestamps to management, server and agent .err logs
dheeraj12347 Apr 6, 2026
36060f0
Add timestamps to stderr console logs for agent, management and API s…
dheeraj12347 Apr 14, 2026
9315f98
usage: add timestamped stderr console logging
dheeraj12347 Apr 15, 2026
15c2f36
WIP: console proxy timeout changes
dheeraj12347 Apr 18, 2026
bc9fb41
Clean up whitespace in log4j config files
dheeraj12347 Apr 21, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 8 additions & 8 deletions agent/conf/log4j-cloud.xml.in
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,13 @@ under the License.
<!-- ============================== -->

<Console name="CONSOLE" target="SYSTEM_OUT">
<ThresholdFilter level="OFF" onMatch="ACCEPT" onMismatch="DENY"/>
<PatternLayout pattern="%-5p [%c{3}] (%t:%x) (logid:%X{logcontextid}) %m%ex%n"/>
<ThresholdFilter level="ERROR" onMatch="DENY" onMismatch="ACCEPT"/>
<PatternLayout pattern="%d{DEFAULT} %-5p [%c{3}] (%t:%x) (logid:%X{logcontextid}) %m%ex%n"/>
</Console>

<Console name="CONSOLE_ERR" target="SYSTEM_ERR">
<ThresholdFilter level="ERROR" onMatch="ACCEPT" onMismatch="DENY"/>
<PatternLayout pattern="%d{DEFAULT} %-5p [%c{3}] (%t:%x) (logid:%X{logcontextid}) %m%ex%n"/>
</Console>
Comment thread
dheeraj12347 marked this conversation as resolved.
Comment on lines 40 to 48
</Appenders>

Expand All @@ -50,17 +55,11 @@ under the License.
<!-- ================ -->

<Logger name="com.cloud" level="INFO"/>

<Logger name="org.apache" level="INFO"/>

<Logger name="org" level="INFO"/>

<Logger name="net" level="INFO"/>

<Logger name="com.amazonaws" level="INFO"/>

<Logger name="httpclient.wire" level="INFO"/>

<Logger name="org.apache.http.wire" level="INFO"/>

<!-- ======================= -->
Expand All @@ -69,6 +68,7 @@ under the License.

<Root level="INFO">
<AppenderRef ref="CONSOLE"/>
<AppenderRef ref="CONSOLE_ERR"/>
<AppenderRef ref="FILE"/>
</Root>

Expand Down
18 changes: 12 additions & 6 deletions client/conf/log4j-cloud.xml.in
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ under the License.
<Appenders>

<properties>
<property name="filters">net.sf.cglib.proxy</property>
<property name="filters">net.sf.cglib.proxy</property>
</properties>

<!-- ================================= -->
Expand All @@ -37,7 +37,6 @@ under the License.
<PatternLayout pattern="%d{DEFAULT} %-5p [%c{1.}] (%t:%x) (logid:%X{logcontextid}) %m%ex{filters(${filters})}%n"/>
</RollingFile>


<RollingFile name="APISERVER" append="true" fileName="@APISERVERLOG@" filePattern="@APISERVERLOG@.%d{yyyy-MM-dd}.gz">
<ThresholdFilter level="DEBUG" onMatch="ACCEPT" onMismatch="DENY"/>
<Policies>
Expand Down Expand Up @@ -69,16 +68,21 @@ under the License.
<!-- ============================== -->

<Console name="CONSOLE" target="SYSTEM_OUT">
<ThresholdFilter level="OFF" onMatch="ACCEPT" onMismatch="DENY"/>
<PatternLayout pattern="%-5p [%c{1.}] (%t:%x) (logid:%X{logcontextid}) %m%ex{filters(${filters})}%n"/>
<ThresholdFilter level="ERROR" onMatch="DENY" onMismatch="ACCEPT"/>
<PatternLayout pattern="%d{DEFAULT} %-5p [%c{1.}] (%t:%x) (logid:%X{logcontextid}) %m%ex{filters(${filters})}%n"/>
</Console>

<Console name="CONSOLE_ERR" target="SYSTEM_ERR">
<ThresholdFilter level="ERROR" onMatch="ACCEPT" onMismatch="DENY"/>
<PatternLayout pattern="%d{DEFAULT} %-5p [%c{1.}] (%t:%x) (logid:%X{logcontextid}) %m%ex{filters(${filters})}%n"/>
</Console>
Comment thread
dheeraj12347 marked this conversation as resolved.
Comment on lines 70 to +78

<!-- ============================== -->
<!-- send alert warnings+ as the SNMP trap if it is configured! -->
<!-- ============================== -->

<SnmpTrapAppender name="SNMP" SnmpManagerIpAddresses="" SnmpManagerPorts="" SnmpManagerCommunities="">
<ThresholdFilter level="WARN" onMatch="ACCEPT" onMismatch="DENY"/>
<ThresholdFilter level="WARN" onMatch="ACCEPT" onMismatch="DENY"/>
</SnmpTrapAppender>
</Appenders>

Expand All @@ -102,7 +106,7 @@ under the License.

<Logger name="net" level="INFO"/>

<Logger name="apiserver.com.cloud" level="DEBUG"/>
<Logger name="apiserver.com.cloud" level="DEBUG"/>

<Logger name="apiserver.com.cloud" level="DEBUG" additivity="false">
<AppenderRef ref="APISERVER"/>
Expand All @@ -115,6 +119,7 @@ under the License.
<Logger name="org.apache.cloudstack.alert" additivity="false" level="WARN">
<AppenderRef ref="SYSLOG"/>
<AppenderRef ref="CONSOLE"/>
<AppenderRef ref="CONSOLE_ERR"/>
<AppenderRef ref="FILE"/>
<AppenderRef ref="SNMP"/>
<AppenderRef ref="ALERTSYSLOG"/>
Expand All @@ -127,6 +132,7 @@ under the License.
<Root level="INFO">
<AppenderRef ref="SYSLOG"/>
<AppenderRef ref="CONSOLE"/>
<AppenderRef ref="CONSOLE_ERR"/>
<AppenderRef ref="FILE"/>
</Root>

Expand Down
15 changes: 9 additions & 6 deletions server/conf/log4j-cloud.xml.in
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@ under the License.
<!-- Preserve messages in a local file -->
<!-- ================================= -->


<!-- A regular appender -->
<RollingFile name="FILE" append="true" fileName="@MSLOG@" filePattern="@MSLOG@.%d{yyyy-MM-dd}.gz">
<ThresholdFilter level="TRACE" onMatch="ACCEPT" onMismatch="DENY"/>
Expand All @@ -34,7 +33,6 @@ under the License.
<PatternLayout pattern="%d{DEFAULT} %-5p [%c{1.}] (%t:%x) %m%ex%n"/>
</RollingFile>


<RollingFile name="INFO-FILE" append="true" fileName="/var/log/cloudstack/management/management-server-info.log" filePattern="/var/log/cloudstack/management/management-server-info.log.%d{yyyy-MM-dd}.gz">
<ThresholdFilter level="INFO" onMatch="ACCEPT" onMismatch="DENY"/>
<Policies>
Expand All @@ -43,7 +41,6 @@ under the License.
<PatternLayout pattern="%d{DEFAULT} %-5p [%c{1.}] (%t:%x) (logid:%X{logcontextid}) %m%ex%n"/>
</RollingFile>


<RollingFile name="APISERVER" append="true" fileName="@APISERVERLOG@" filePattern="@APISERVERLOG@.%d{yyyy-MM-dd}.gz">
<ThresholdFilter level="DEBUG" onMatch="ACCEPT" onMismatch="DENY"/>
<Policies>
Expand All @@ -56,7 +53,6 @@ under the License.
<!-- Append warnings+ to the syslog if it is listening on UDP port -->
<!-- ============================== -->


<Syslog name="SYSLOG" host="localhost" facility="LOCAL6">
<ThresholdFilter level="WARN" onMatch="ACCEPT" onMismatch="DENY"/>
<PatternLayout pattern="%-5p [%c{1.}] (%t:%x) %m%ex%n"/>
Expand All @@ -67,8 +63,13 @@ under the License.
<!-- ============================== -->

<Console name="CONSOLE" target="SYSTEM_OUT">
<ThresholdFilter level="OFF" onMatch="ACCEPT" onMismatch="DENY"/>
<PatternLayout pattern="%-5p [%c{1.}] (%t:%x) %m%ex%n"/>
<ThresholdFilter level="ERROR" onMatch="DENY" onMismatch="ACCEPT"/>
<PatternLayout pattern="%d{DEFAULT} %-5p [%c{1.}] (%t:%x) %m%ex%n"/>
</Console>

<Console name="CONSOLE_ERR" target="SYSTEM_ERR">
<ThresholdFilter level="ERROR" onMatch="ACCEPT" onMismatch="DENY"/>
<PatternLayout pattern="%d{DEFAULT} %-5p [%c{1.}] (%t:%x) %m%ex%n"/>
Comment on lines 65 to +72
Copy link
Copy Markdown
Contributor

@DaanHoogland DaanHoogland Apr 13, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we need to double check this, @dheeraj12347 . I looked in an env and the .out file already contains the date stamps for each line. The .err file doesn’t .

Copy link
Copy Markdown
Contributor Author

@dheeraj12347 dheeraj12347 Apr 14, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we need to double check this, @dheeraj12347 . I looked in an env and the .out file already contains the date stamps for each line. The .err file doesn’t .

You’re right, Daan – .out already had timestamps while .err didn’t. I’ve now added a CONSOLE_ERR appender using SYSTEM_ERR with the same timestamped pattern, so stderr (and thus the .err files) will get timestamps as well.

Comment on lines +69 to +72
Copy link
Copy Markdown
Contributor Author

@dheeraj12347 dheeraj12347 Apr 15, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I’ve now updated usage/conf/log4j-cloud_usage.xml.in to add a CONSOLE_ERR appender targeting SYSTEM_ERR with a timestamped pattern, so usage.err will also get timestamps in line with issue #12887.

Comment on lines 65 to +72
</Console>
Comment on lines 65 to 73
Copy link

Copilot AI Apr 8, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The PR description says the .err logs are fed from console/stderr, but this Console appender targets SYSTEM_OUT. If systemd is capturing stderr into .err, changing the stdout pattern won’t affect .err. Consider switching the console target to SYSTEM_ERR (or update the systemd unit routing) so the timestamped console output actually lands in .err.

Copilot uses AI. Check for mistakes.
</Appenders>

Expand Down Expand Up @@ -101,6 +102,7 @@ under the License.
<Logger name="org.apache.cloudstack.alerts" additivity="false" level="WARN">
<AppenderRef ref="SYSLOG"/>
<AppenderRef ref="CONSOLE"/>
<AppenderRef ref="CONSOLE_ERR"/>
<AppenderRef ref="FILE"/>
<AppenderRef ref="SNMP"/>
<AppenderRef ref="ALERTSYSLOG"/>
Expand All @@ -113,6 +115,7 @@ under the License.
<Root level="INFO">
<AppenderRef ref="SYSLOG"/>
<AppenderRef ref="CONSOLE"/>
<AppenderRef ref="CONSOLE_ERR"/>
<AppenderRef ref="FILE"/>
<AppenderRef ref="INFO-FILE"/>
</Root>
Expand Down
77 changes: 58 additions & 19 deletions services/console-proxy/server/src/main/java/com/cloud/consoleproxy/ConsoleProxy.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,11 @@ public class ConsoleProxy {
static String factoryClzName;
static boolean standaloneStart = false;

/**
* Session timeout in milliseconds, default 300000 (5 minutes).
*/
public static int sessionTimeoutMillis = 300000;

static String encryptorPassword = "Dummy";
static final String[] skipProperties = new String[]{"certificate", "cacertificate", "keystore_password", "privatekey"};

Expand All @@ -92,11 +97,13 @@ public static void addAllowedSession(String sessionUuid) {
private static void configLog4j() {
final ClassLoader loader = Thread.currentThread().getContextClassLoader();
URL configUrl = loader.getResource("/conf/log4j-cloud.xml");
if (configUrl == null)
if (configUrl == null) {
configUrl = ClassLoader.getSystemResource("log4j-cloud.xml");
}

if (configUrl == null)
if (configUrl == null) {
configUrl = ClassLoader.getSystemResource("conf/log4j-cloud.xml");
}

if (configUrl != null) {
try {
Expand All @@ -121,9 +128,8 @@ private static void configLog4j() {
private static void configProxy(Properties conf) {
LOGGER.info("Configure console proxy...");
for (Object key : conf.keySet()) {
LOGGER.info("Property " + (String)key + ": " + conf.getProperty((String)key));
if (!ArrayUtils.contains(skipProperties, key)) {
LOGGER.info("Property " + (String)key + ": " + conf.getProperty((String)key));
LOGGER.info("Property " + (String) key + ": " + conf.getProperty((String) key));
}
}

Expand Down Expand Up @@ -165,13 +171,31 @@ private static void configProxy(Properties conf) {
defaultBufferSize = Integer.parseInt(s);
LOGGER.info("Setting defaultBufferSize=" + defaultBufferSize);
}

// Read consoleproxy.session.timeout in milliseconds.
s = conf.getProperty("consoleproxy.session.timeout");
if (s != null) {
try {
int parsedTimeout = Integer.parseInt(s);
if (parsedTimeout < 1000) {
LOGGER.warn("Invalid value for consoleproxy.session.timeout: " + s
+ " ms, must be >= 1000 ms, keeping default " + sessionTimeoutMillis + " ms");
} else {
sessionTimeoutMillis = parsedTimeout;
LOGGER.info("Setting consoleproxy.session.timeout=" + sessionTimeoutMillis + " ms");
}
} catch (NumberFormatException e) {
LOGGER.warn("Invalid value for consoleproxy.session.timeout: " + s
+ ", keeping default " + sessionTimeoutMillis + " ms", e);
}
}
}

public static ConsoleProxyServerFactory getHttpServerFactory() {
try {
Class<?> clz = Class.forName(factoryClzName);
try {
ConsoleProxyServerFactory factory = (ConsoleProxyServerFactory)clz.newInstance();
ConsoleProxyServerFactory factory = (ConsoleProxyServerFactory) clz.newInstance();
factory.init(ConsoleProxy.ksBits, ConsoleProxy.ksPassword);
return factory;
} catch (InstantiationException e) {
Expand Down Expand Up @@ -243,7 +267,7 @@ public static ConsoleProxyAuthenticationResult authenticateConsoleAccess(Console
}

if (result != null && result instanceof String) {
authResult = new Gson().fromJson((String)result, ConsoleProxyAuthenticationResult.class);
authResult = new Gson().fromJson((String) result, ConsoleProxyAuthenticationResult.class);
} else {
LOGGER.error("Invalid authentication return object " + result + " for vm: " + param.getClientTag() + ", decline the access");
authResult.setSuccess(false);
Expand Down Expand Up @@ -318,19 +342,25 @@ public static void startWithContext(Properties conf, Object context, byte[] ksBi
LOGGER.error("Unable to setup private channel due to ClassNotFoundException", e);
}

// ensure we have a Properties object before merging defaults
if (conf == null) {
conf = new Properties();
}

// merge properties from conf file
InputStream confs = ConsoleProxy.class.getResourceAsStream("/conf/consoleproxy.properties");
Properties props = new Properties();
if (confs == null) {
final File file = PropertiesUtil.findConfigFile("consoleproxy.properties");
if (file == null)
if (file == null) {
LOGGER.info("Can't load consoleproxy.properties from classpath, will use default configuration");
else
} else {
try {
confs = new FileInputStream(file);
} catch (FileNotFoundException e) {
LOGGER.info("Ignoring file not found exception and using defaults");
}
}
}
if (confs != null) {
try {
Expand All @@ -339,15 +369,18 @@ public static void startWithContext(Properties conf, Object context, byte[] ksBi
for (Object key : props.keySet()) {
// give properties passed via context high priority, treat properties from consoleproxy.properties
// as default values
if (conf.get(key) == null)
if (conf.get(key) == null) {
conf.put(key, props.get(key));
}
}
} catch (Exception e) {
LOGGER.error(e.toString(), e);
}
}
try {
confs.close();
if (confs != null) {
confs.close();
}
} catch (IOException e) {
LOGGER.error("Failed to close consolepropxy.properties : " + e.toString(), e);
}
Expand Down Expand Up @@ -481,8 +514,9 @@ public static ConsoleProxyClient getVncViewer(ConsoleProxyClientParam param) thr
ConsoleProxyClientStatsCollector statsCollector = getStatsCollector();
String loadInfo = statsCollector.getStatsReport();
reportLoadInfo(loadInfo);
if (LOGGER.isDebugEnabled())
if (LOGGER.isDebugEnabled()) {
LOGGER.debug("Report load change : " + loadInfo);
}
}

return viewer;
Expand All @@ -506,13 +540,15 @@ public static ConsoleProxyClient getAjaxVncViewer(ConsoleProxyClientParam param,
// protected against malicious attack by modifying URL content
if (ajaxSession != null) {
long ajaxSessionIdFromUrl = Long.parseLong(ajaxSession);
if (ajaxSessionIdFromUrl != viewer.getAjaxSessionId())
if (ajaxSessionIdFromUrl != viewer.getAjaxSessionId()) {
throw new AuthenticationException("Cannot use the existing viewer " + viewer + ": modified AJAX session id");
}
}

if (param.getClientHostPassword() == null || param.getClientHostPassword().isEmpty() ||
!param.getClientHostPassword().equals(viewer.getClientHostPassword()))
!param.getClientHostPassword().equals(viewer.getClientHostPassword())) {
throw new AuthenticationException("Cannot use the existing viewer " + viewer + ": bad sid");
}

if (!viewer.isFrontEndAlive()) {

Expand All @@ -526,8 +562,9 @@ public static ConsoleProxyClient getAjaxVncViewer(ConsoleProxyClientParam param,
ConsoleProxyClientStatsCollector statsCollector = getStatsCollector();
String loadInfo = statsCollector.getStatsReport();
reportLoadInfo(loadInfo);
if (LOGGER.isDebugEnabled())
if (LOGGER.isDebugEnabled()) {
LOGGER.debug("Report load change : " + loadInfo);
}
}
return viewer;
}
Expand Down Expand Up @@ -593,7 +630,7 @@ public void execute(Runnable r) {
}

public static ConsoleProxyNoVncClient getNoVncViewer(ConsoleProxyClientParam param, String ajaxSession,
Session session) throws AuthenticationException {
Session session) throws AuthenticationException {
boolean reportLoadChange = false;
String clientKey = param.getClientMapKey();
LOGGER.debug("Getting NoVNC viewer for {}. Session requires new viewer: {}, client tag: {}. session UUID: {}",
Expand All @@ -609,8 +646,9 @@ public static ConsoleProxyNoVncClient getNoVncViewer(ConsoleProxyClientParam par
reportLoadChange = true;
} else {
if (param.getClientHostPassword() == null || param.getClientHostPassword().isEmpty() ||
!param.getClientHostPassword().equals(viewer.getClientHostPassword()))
!param.getClientHostPassword().equals(viewer.getClientHostPassword())) {
throw new AuthenticationException("Cannot use the existing viewer " + viewer + ": bad sid");
}

try {
authenticationExternally(param);
Expand All @@ -620,7 +658,7 @@ public static ConsoleProxyNoVncClient getNoVncViewer(ConsoleProxyClientParam par
}
LOGGER.info("Initializing new novnc client and disconnecting existing session");
try {
((ConsoleProxyNoVncClient)viewer).getSession().disconnect();
((ConsoleProxyNoVncClient) viewer).getSession().disconnect();
} catch (IOException e) {
LOGGER.error("Exception while disconnect session of novnc viewer object: " + viewer, e);
}
Expand All @@ -635,10 +673,11 @@ public static ConsoleProxyNoVncClient getNoVncViewer(ConsoleProxyClientParam par
ConsoleProxyClientStatsCollector statsCollector = getStatsCollector();
String loadInfo = statsCollector.getStatsReport();
reportLoadInfo(loadInfo);
if (LOGGER.isDebugEnabled())
if (LOGGER.isDebugEnabled()) {
LOGGER.debug("Report load change : " + loadInfo);
}
}
return (ConsoleProxyNoVncClient)viewer;
return (ConsoleProxyNoVncClient) viewer;
}
}
}
Loading