Mqtt

.github/workflows/cdb-base-mqtt.yml

@@ -0,0 +1,45 @@
+name: CDB Base MQTT
+
+on:
+  push:
+    paths:
+      - "tools/developer_tools/cdb-base-mqtt/**"
+      - ".github/workflows/cdb-base-mqtt.yml"
+  pull_request:
+    paths:
+      - "tools/developer_tools/cdb-base-mqtt/**"
+      - ".github/workflows/cdb-base-mqtt.yml"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: tools/developer_tools/cdb-base-mqtt
+
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Set up Java 8 and Maven cache
+        uses: actions/setup-java@v4
+        with:
+          distribution: temurin
+          java-version: "8"
+          cache: maven
+          cache-dependency-path: tools/developer_tools/cdb-base-mqtt/pom.xml
+
+      - name: Run tests
+        run: mvn --batch-mode --no-transfer-progress verify
+
+      - name: Publish test results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: cdb-base-mqtt-test-results
+          path: tools/developer_tools/cdb-base-mqtt/target/surefire-reports/
+          if-no-files-found: warn

Makefile

@@ -9,6 +9,7 @@ TOP = .
 SUBDIRS = src
 
 .PHONY: support support-mysql dev-config prepare-release release-python-client
+.PHONY: mqtt-configuration configure-mqtt mqtt-configuration-dev configure-mqtt-dev
 .PHONY: db backup db-dev deploy-web-portal undeploy-web-portal deploy-web-service undeploy-web-service
 .PHONY: db-dev backup-dev deploy-web-portal-dev undeploy-web-portal-dev deploy-web-service-dev undeploy-web-service-dev
 
@@ -28,6 +29,12 @@ release-python-client:
 configuration:
 	$(TOP)/sbin/cdb_create_configuration.sh
 
+mqtt-configuration:
+	$(TOP)/sbin/cdb_create_mqtt_configuration.sh cdb
+
+configure-mqtt:
+	$(TOP)/sbin/cdb_configure_mqtt_service.sh cdb
+
 support:
 	$(TOP)/sbin/cdb_install_support.sh
 
@@ -79,6 +86,12 @@ undeploy-web-service:
 configuration-dev:
 	$(TOP)/sbin/cdb_create_configuration.sh cdb_dev
 
+mqtt-configuration-dev:
+	$(TOP)/sbin/cdb_create_mqtt_configuration.sh cdb_dev
+
+configure-mqtt-dev:
+	$(TOP)/sbin/cdb_configure_mqtt_service.sh cdb_dev
+
 db-dev:
 	$(TOP)/sbin/cdb_create_db.sh cdb_dev
 

sbin/cdb_configure_mqtt_service.sh

@@ -0,0 +1,170 @@
+#!/bin/bash
+
+# Copyright (c) UChicago Argonne, LLC. All rights reserved.
+# See LICENSE file.
+
+
+#
+# Script used for configuring MQTT connector for CDB webapp
+# Deploys MQTT Resource Adapter and creates connection pool/resource
+#
+# Usage:
+#
+# $0 [CDB_DB_NAME] [mqtt_config_file]
+#
+# If no config file is specified, defaults to $CDB_INSTALL_DIR/etc/<CDB_DB_NAME>.mqtt.conf
+#
+# Sample MQTT configuration file contents (mqtt.conf):
+# MQTT_HOST=localhost              # MQTT broker hostname (default: localhost)
+# MQTT_PORT=1883                   # MQTT broker port (default: 1883)
+# MQTT_USERNAME=admin              # MQTT username (optional)
+# MQTT_PASSWORD=admin              # MQTT password (optional)
+# MQTT_CLEAN_SESSION=true          # Clean session flag (optional)
+# MQTT_QOS=1                       # Quality of Service level (optional)
+# MQTT_KEEP_ALIVE_INTERVAL=60      # Keep alive interval in seconds (optional)
+# MQTT_CONNECTION_TIMEOUT=30       # Connection timeout in seconds (optional)
+# MQTT_MAX_INFLIGHT=10             # Maximum number of messages in flight (optional)
+# MQTT_AUTOMATIC_RECONNECT=true    # Automatic reconnection on disconnect (optional)
+# MQTT_FILE_PERSISTANCE=false      # Enable file-based message persistence (optional)
+# MQTT_PERSISTENCE_DIRECTORY=.     # Directory for persistent message storage (optional)
+# MQTT_TOPIC_FILTER                # MQTT topic filter for subscriptions (optional)
+
+MY_DIR=`dirname $0` && cd $MY_DIR && MY_DIR=`pwd`
+if [ -z "${CDB_ROOT_DIR}" ]; then
+    CDB_ROOT_DIR=$MY_DIR/..
+fi
+CDB_ENV_FILE=${CDB_ROOT_DIR}/setup.sh
+if [ ! -f ${CDB_ENV_FILE} ]; then
+    echo "Environment file ${CDB_ENV_FILE} does not exist."
+    exit 2
+fi
+. ${CDB_ENV_FILE} > /dev/null
+
+# Constants
+MQTT_POOL_NAME="cdb/MQTT/pool"
+MQTT_RESOURCE_NAME="cdb/MQTT/resource"
+MQTT_RAR_NAME="mqtt-rar-0.8.0"
+MQTT_CONNECTOR_RAR_DEPLOYMENT_NAME="cdb-mqtt-rar-deployment"
+MQTT_RAR_PATH=$CDB_ROOT_DIR/src/lib/${MQTT_RAR_NAME}.rar
+
+# Look for the deployment-specific MQTT configuration file
+CDB_DB_NAME=${1:-${CDB_DB_NAME:-cdb}}
+if [ ! -z "$2" ]; then
+    mqttConfigFile=$2
+else
+    mqttConfigFile=$CDB_INSTALL_DIR/etc/${CDB_DB_NAME}.mqtt.conf
+fi
+
+if [ -f $mqttConfigFile ]; then
+    echo "Using MQTT config file: $mqttConfigFile"
+    . $mqttConfigFile
+else
+    echo "Error: MQTT config file $mqttConfigFile not found."
+    echo "You can create one using cdb_create_mqtt_configuration.sh ${CDB_DB_NAME}"
+    exit 1
+fi
+
+CDB_HOST_ARCH=$(uname -sm | tr -s '[:upper:][:blank:]' '[:lower:][\-]')
+GLASSFISH_DIR=$CDB_SUPPORT_DIR/payara/$CDB_HOST_ARCH
+
+ASADMIN_CMD=$GLASSFISH_DIR/bin/asadmin
+
+# MQTT Configuration defaults
+MQTT_HOST=${MQTT_HOST:=localhost}
+MQTT_PORT=${MQTT_PORT:=1883}
+
+# Build properties string for connection pool
+PROPERTIES="serverURIs=tcp\\://${MQTT_HOST}\\:${MQTT_PORT}"
+
+if [ ! -z "$MQTT_USERNAME" ]; then
+    PROPERTIES="${PROPERTIES}:userName=${MQTT_USERNAME}"
+fi
+
+if [ ! -z "$MQTT_PASSWORD" ]; then
+    PROPERTIES="${PROPERTIES}:password=${MQTT_PASSWORD}"
+fi
+
+if [ ! -z "$MQTT_CLEAN_SESSION" ]; then
+    PROPERTIES="${PROPERTIES}:cleanSession=${MQTT_CLEAN_SESSION}"
+fi
+
+if [ ! -z "$MQTT_KEEP_ALIVE_INTERVAL" ]; then
+    PROPERTIES="${PROPERTIES}:keepAliveInterval=${MQTT_KEEP_ALIVE_INTERVAL}"
+fi
+
+if [ ! -z "$MQTT_CONNECTION_TIMEOUT" ]; then
+    PROPERTIES="${PROPERTIES}:connectionTimeout=${MQTT_CONNECTION_TIMEOUT}"
+fi
+
+if [ ! -z "$MQTT_MAX_INFLIGHT" ]; then
+    PROPERTIES="${PROPERTIES}:maxInflight=${MQTT_MAX_INFLIGHT}"
+fi
+
+if [ ! -z "$MQTT_AUTOMATIC_RECONNECT" ]; then
+    PROPERTIES="${PROPERTIES}:automaticReconnect=${MQTT_AUTOMATIC_RECONNECT}"
+fi
+
+if [ ! -z "$MQTT_FILE_PERSISTANCE" ]; then
+    PROPERTIES="${PROPERTIES}:filePersistance=${MQTT_FILE_PERSISTANCE}"
+fi
+
+if [ ! -z "$MQTT_PERSISTENCE_DIRECTORY" ]; then
+    PROPERTIES="${PROPERTIES}:persistenceDirectory=${MQTT_PERSISTENCE_DIRECTORY}"
+fi
+
+if [ ! -z "$MQTT_QOS" ]; then
+    PROPERTIES="${PROPERTIES}:qos=${MQTT_QOS}"
+fi
+
+if [ ! -z "$MQTT_TOPIC_FILTER" ]; then
+    PROPERTIES="${PROPERTIES}:topicFilter=${MQTT_TOPIC_FILTER}"
+fi
+
+# Deploy MQTT RAR
+echo "Deploying MQTT RAR"
+if [ -f "$MQTT_RAR_PATH" ]; then
+    # Check if already deployed and undeploy if needed
+    $ASADMIN_CMD list-applications | grep -q ${MQTT_CONNECTOR_RAR_DEPLOYMENT_NAME} && {
+        echo "Undeploying existing MQTT RAR"
+        # Check if resource exists and delete it
+        $ASADMIN_CMD list-connector-resources | grep -q ${MQTT_RESOURCE_NAME} && {
+            echo "Deleting existing MQTT resource"
+            $ASADMIN_CMD delete-connector-resource ${MQTT_RESOURCE_NAME} || exit 1
+        }
+
+        # Check if connection pool exists and delete it
+        $ASADMIN_CMD list-connector-connection-pools | grep -q ${MQTT_POOL_NAME} && {
+            echo "Deleting existing MQTT connection pool"
+            $ASADMIN_CMD delete-connector-connection-pool ${MQTT_POOL_NAME} || exit 1
+        }
+
+        echo "Undeploying existing MQTT RAR"
+        $ASADMIN_CMD undeploy ${MQTT_CONNECTOR_RAR_DEPLOYMENT_NAME} || exit 1
+    }
+    $ASADMIN_CMD deploy --name ${MQTT_CONNECTOR_RAR_DEPLOYMENT_NAME} $MQTT_RAR_PATH || exit 1
+else
+    echo "Warning: MQTT RAR file not found at $MQTT_RAR_PATH"
+    exit 1
+fi
+
+# Create MQTT connection pool
+echo "Creating MQTT connection pool ${MQTT_POOL_NAME}"
+$ASADMIN_CMD create-connector-connection-pool \
+    --raname ${MQTT_CONNECTOR_RAR_DEPLOYMENT_NAME} \
+    --connectiondefinition fish.payara.cloud.connectors.mqtt.api.MQTTConnectionFactory \
+    --property "${PROPERTIES}" \
+    ${MQTT_POOL_NAME} || exit 1
+# Create MQTT resource
+echo "Creating MQTT resource ${MQTT_RESOURCE_NAME}"
+$ASADMIN_CMD create-connector-resource \
+    --poolname ${MQTT_POOL_NAME} \
+    ${MQTT_RESOURCE_NAME} || exit 1
+
+# The connector logs allocation failures with a full stack trace before returning null.
+$ASADMIN_CMD set-log-levels fish.payara.cloud.connectors.mqtt.api.outbound.MQTTConnectionFactoryImpl=OFF || exit 1
+
+# Test MQTT connection pool
+echo "Testing MQTT connection pool"
+$ASADMIN_CMD ping-connection-pool ${MQTT_POOL_NAME} || { echo "Warning: MQTT connection pool ping failed"; exit 1; }
+
+echo "Restart or redeploy CDB."

sbin/cdb_create_mqtt_configuration.sh

@@ -0,0 +1,135 @@
+#!/bin/bash
+
+# Script to create MQTT configuration file for CDB service
+# Usage: $0 [CDB_DB_NAME]
+
+MY_DIR=`dirname $0` && cd $MY_DIR && MY_DIR=`pwd`
+if [ -z "${CDB_ROOT_DIR}" ]; then
+    CDB_ROOT_DIR=$MY_DIR/..
+fi
+CDB_ENV_FILE=${CDB_ROOT_DIR}/setup.sh
+if [ ! -f ${CDB_ENV_FILE} ]; then
+    echo "Environment file ${CDB_ENV_FILE} does not exist."
+    exit 2
+fi
+. ${CDB_ENV_FILE} > /dev/null
+
+# Default configuration file location
+CDB_DB_NAME=${1:-${CDB_DB_NAME:-cdb}}
+MQTT_CONFIG_FILE=${CDB_INSTALL_DIR}/etc/${CDB_DB_NAME}.mqtt.conf
+
+echo "==================================="
+echo "MQTT Configuration Setup for CDB"
+echo "==================================="
+echo ""
+echo "This script will help you create an MQTT configuration file."
+echo "Configuration will be saved to: $MQTT_CONFIG_FILE"
+echo ""
+
+# Check if config file already exists
+if [ -f "$MQTT_CONFIG_FILE" ]; then
+    read -p "Configuration file already exists. Overwrite? (y/n): " -n 1 -r
+    echo
+    if [[ ! $REPLY =~ ^[Yy]$ ]]; then
+        echo "Exiting without changes."
+        exit 0
+    fi
+fi
+
+echo ""
+echo "Configuration Details:"
+echo "- cleanSession: Whether client and server should remember state across reconnects"
+echo "- automaticReconnect: Whether client will automatically reconnect if connection is lost"
+echo "- filePersistance: Whether to use file persistence for un-acknowledged messages"
+echo "- persistenceDirectory: Directory to use for file persistence"
+echo "- connectionTimeout: Connection timeout value in seconds"
+echo "- maxInflight: Maximum messages that can be sent without acknowledgements"
+echo "- keepAliveInterval: Keep alive interval in seconds"
+echo "- userName/password: Authentication credentials"
+# Disable MDB only variables.
+# echo "- topicFilter: Topic Filter (For MDBs only)"
+# echo "- qos: Quality of Service for the subscription (For MDBs only)"
+echo ""
+
+# Ensure directory exists
+mkdir -p $(dirname "$MQTT_CONFIG_FILE")
+
+# Prompt for configuration values
+read -p "MQTT Host [localhost]: " MQTT_HOST
+MQTT_HOST=${MQTT_HOST:-localhost}
+
+read -p "MQTT Port [1883]: " MQTT_PORT
+MQTT_PORT=${MQTT_PORT:-1883}
+
+read -p "MQTT Username (leave empty for no auth): " MQTT_USERNAME
+
+if [ ! -z "$MQTT_USERNAME" ]; then
+    read -s -p "MQTT Password: " MQTT_PASSWORD
+    echo
+fi
+
+read -p "Clean Session (true/false) [false]: " CLEAN_SESSION
+CLEAN_SESSION=${CLEAN_SESSION:-false}
+
+read -p "Automatic Reconnect (true/false) [true]: " AUTOMATIC_RECONNECT
+AUTOMATIC_RECONNECT=${AUTOMATIC_RECONNECT:-true}
+
+read -p "File Persistance (true/false) [false]: " FILE_PERSISTANCE
+FILE_PERSISTANCE=${FILE_PERSISTANCE:-false}
+
+read -p "Persistence Directory [.]: " PERSISTENCE_DIRECTORY
+PERSISTENCE_DIRECTORY=${PERSISTENCE_DIRECTORY:-.}
+
+read -p "Connection Timeout (seconds) [30]: " CONNECTION_TIMEOUT
+CONNECTION_TIMEOUT=${CONNECTION_TIMEOUT:-30}
+
+read -p "Max Inflight [10]: " MAX_INFLIGHT
+MAX_INFLIGHT=${MAX_INFLIGHT:-10}
+
+read -p "Keep Alive Interval (seconds) [60]: " KEEP_ALIVE_INTERVAL
+KEEP_ALIVE_INTERVAL=${KEEP_ALIVE_INTERVAL:-60}
+
+# MDB only variables 
+# read -p "Topic Filter (leave empty if not using MDB): " TOPIC_FILTER
+# read -p "QoS (0/1/2) [0]: " QOS
+# QOS=${QOS:-0}
+
+# Write configuration file
+cat > "$MQTT_CONFIG_FILE" << EOF
+# MQTT Configuration for CDB Service
+# Generated on $(date)
+
+MQTT_HOST=$MQTT_HOST
+MQTT_PORT=$MQTT_PORT
+EOF
+
+if [ ! -z "$MQTT_USERNAME" ]; then
+    echo "MQTT_USERNAME=$MQTT_USERNAME" >> "$MQTT_CONFIG_FILE"
+fi
+
+if [ ! -z "$MQTT_PASSWORD" ]; then
+    echo "MQTT_PASSWORD=$MQTT_PASSWORD" >> "$MQTT_CONFIG_FILE"
+fi
+
+cat >> "$MQTT_CONFIG_FILE" << EOF
+MQTT_CLEAN_SESSION=$CLEAN_SESSION
+MQTT_AUTOMATIC_RECONNECT=$AUTOMATIC_RECONNECT
+MQTT_FILE_PERSISTANCE=$FILE_PERSISTANCE
+MQTT_PERSISTENCE_DIRECTORY=$PERSISTENCE_DIRECTORY
+MQTT_CONNECTION_TIMEOUT=$CONNECTION_TIMEOUT
+MQTT_MAX_INFLIGHT=$MAX_INFLIGHT
+MQTT_KEEP_ALIVE_INTERVAL=$KEEP_ALIVE_INTERVAL
+EOF
+
+if [ ! -z "$TOPIC_FILTER" ]; then
+    echo "MQTT_TOPIC_FILTER=$TOPIC_FILTER" >> "$MQTT_CONFIG_FILE"
+fi
+
+if [ ! -z "$QOS" ]; then
+    echo "MQTT_QOS=$QOS" >> "$MQTT_CONFIG_FILE"
+fi
+
+echo ""
+echo "Configuration file created successfully at: $MQTT_CONFIG_FILE"
+echo ""
+echo "You can now run cdb_configure_mqtt_service.sh ${CDB_DB_NAME} to apply this configuration."