Feat/fauxsq

.github/workflows/ci.common.yml

@@ -8,6 +8,10 @@ on:
       node_version:
         required: true
         type: string
+      queue_backend:
+        required: false
+        type: string
+        default: 'fauxqs'
 
 jobs:
   build:
@@ -32,9 +36,17 @@ jobs:
 
       - name: Docker start
         run: npm run docker:start:ci -- --filter=${{ inputs.package_name }}
+        env:
+          QUEUE_BACKEND: ${{ inputs.queue_backend }}
 
       - name: Run Tests
-        run: npm run test:ci -- --filter=${{ inputs.package_name }}
+        run: |
+          echo "::notice::Running ${{ inputs.package_name }} with QUEUE_BACKEND=${{ inputs.queue_backend }}"
+          npm run test:ci -- --filter=${{ inputs.package_name }}
+        env:
+          QUEUE_BACKEND: ${{ inputs.queue_backend }}
 
       - name: Docker stop
         run: npm run docker:stop:ci -- --filter=${{ inputs.package_name }}
+        env:
+          QUEUE_BACKEND: ${{ inputs.queue_backend }}

.github/workflows/ci.yml

@@ -12,6 +12,7 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       packages: ${{ steps.detect.outputs.packages }}
+      aws_packages: ${{ steps.detect.outputs.aws_packages }}
     steps:
       - name: Get changed files
         id: changed-files
@@ -39,22 +40,45 @@ jobs:
             ["packages/sqs"]="@message-queue-toolkit/sqs"
           )
 
+          AWS_PACKAGE_PATHS=("packages/sqs" "packages/sns")
+
           PACKAGES=()
+          AWS_PACKAGES=()
           for path in "${!PATH_TO_NAME[@]}"; do
             if echo "$ALL_CHANGED_FILES" | grep -q "$path/"; then
-              PACKAGES+=("\"${PATH_TO_NAME[$path]}\"")
+              is_aws=false
+              for aws_path in "${AWS_PACKAGE_PATHS[@]}"; do
+                if [ "$path" = "$aws_path" ]; then
+                  is_aws=true
+                  break
+                fi
+              done
+              if [ "$is_aws" = true ]; then
+                AWS_PACKAGES+=("\"${PATH_TO_NAME[$path]}\"")
+              else
+                PACKAGES+=("\"${PATH_TO_NAME[$path]}\"")
+              fi
             fi
           done
 
           if [ ${#PACKAGES[@]} -eq 0 ]; then
             echo 'packages=[]' >> $GITHUB_OUTPUT
-            echo "No packages changed"
+            echo "No non-AWS packages changed"
           else
             JSON="[$(IFS=,; echo "${PACKAGES[*]}")]"
             echo "packages=$JSON" >> $GITHUB_OUTPUT
             echo "Changed packages: $JSON"
           fi
 
+          if [ ${#AWS_PACKAGES[@]} -eq 0 ]; then
+            echo 'aws_packages=[]' >> $GITHUB_OUTPUT
+            echo "No AWS packages changed"
+          else
+            AWS_JSON="[$(IFS=,; echo "${AWS_PACKAGES[*]}")]"
+            echo "aws_packages=$AWS_JSON" >> $GITHUB_OUTPUT
+            echo "Changed AWS packages: $AWS_JSON"
+          fi
+
   general:
     needs: [changed-files-job]
     if: needs.changed-files-job.outputs.packages != '[]'
@@ -67,9 +91,23 @@ jobs:
       node_version: ${{ matrix.node-version }}
       package_name: ${{ matrix.package-name }}
 
+  aws-packages:
+    needs: [changed-files-job]
+    if: needs.changed-files-job.outputs.aws_packages != '[]'
+    strategy:
+      matrix:
+        node-version: [22.x, 24.x]
+        package-name: ${{ fromJson(needs.changed-files-job.outputs.aws_packages) }}
+        queue-backend: [fauxqs, localstack]
+    uses: ./.github/workflows/ci.common.yml
+    with:
+      node_version: ${{ matrix.node-version }}
+      package_name: ${{ matrix.package-name }}
+      queue_backend: ${{ matrix.queue-backend }}
+
   automerge:
-    needs: [general]
-    if: always() && (needs.general.result == 'success' || needs.general.result == 'skipped')
+    needs: [general, aws-packages]
+    if: always() && (needs.general.result == 'success' || needs.general.result == 'skipped') && (needs.aws-packages.result == 'success' || needs.aws-packages.result == 'skipped')
     runs-on: ubuntu-latest
     permissions:
       pull-requests: write

docker-compose.yml

@@ -9,25 +9,6 @@ services:
       - rabbit_data:/var/lib/rabbitmq
     restart: on-failure
 
-  localstack:
-    image: localstack/localstack:4.13.1
-    network_mode: bridge
-    hostname: localstack
-    ports:
-      - '127.0.0.1:4566:4566' # LocalStack Gateway
-      - '127.0.0.1:4510-4559:4510-4559' # external services port range
-    environment:
-      - SERVICES=sns,sqs,s3,sts
-      - DEBUG=0
-      - DATA_DIR=${DATA_DIR-}
-      - DOCKER_HOST=unix:///var/run/docker.sock
-      - LOCALSTACK_HOST=localstack
-    #      - LOCALSTACK_API_KEY=someDummyKey
-    volumes:
-      - '${TMPDIR:-/tmp}/localstack:/var/log/localstack'
-      - '/var/run/docker.sock:/var/run/docker.sock'
-    restart: on-failure
-
   redis:
     image: redis:6.2.7-alpine
     command: redis-server --requirepass sOmE_sEcUrE_pAsS
@@ -95,6 +76,22 @@ services:
       retries: 10
       start_period: 10s
 
+  localstack:
+    image: localstack/localstack:4.13.1
+    network_mode: bridge
+    hostname: localstack
+    ports:
+      - '127.0.0.1:4566:4566'
+      - '127.0.0.1:4510-4559:4510-4559'
+    environment:
+      - SERVICES=sns,sqs,s3,sts
+      - DEBUG=0
+      - LOCALSTACK_HOST=localstack
+    volumes:
+      - '${TMPDIR:-/tmp}/localstack:/var/log/localstack'
+      - '/var/run/docker.sock:/var/run/docker.sock'
+    restart: on-failure
+
 volumes:
   rabbit_data:
     driver: local

package.json

@@ -15,7 +15,7 @@
     },
     "dependencies": {},
     "devDependencies": {
-        "turbo": "^2.6.2"
+        "turbo": "^2.8.10"
     },
     "packageManager": "npm@10.7.0"
 }

packages/gcp-pubsub/test/consumers/PubSubPermissionConsumer.payloadOffloading.spec.ts

@@ -9,7 +9,7 @@ import { PUBSUB_MESSAGE_MAX_SIZE } from '../../lib/pubsub/AbstractPubSubService.
 import { OFFLOADED_PAYLOAD_SIZE_ATTRIBUTE } from '../../lib/utils/messageUtils.ts'
 import { PubSubPermissionPublisher } from '../publishers/PubSubPermissionPublisher.ts'
 import { deletePubSubTopicAndSubscription } from '../utils/cleanupPubSub.ts'
-import { assertBucket, emptyBucket } from '../utils/gcsUtils.ts'
+import { assertBucket, emptyBuckets } from '../utils/gcsUtils.ts'
 import type { Dependencies } from '../utils/testContext.ts'
 import { registerDependencies } from '../utils/testContext.ts'
 import { PubSubPermissionConsumer } from './PubSubPermissionConsumer.ts'
@@ -75,7 +75,7 @@ describe('PubSubPermissionConsumer - Payload Offloading', () => {
     })
 
     afterAll(async () => {
-      await emptyBucket(gcsStorage, gcsBucketName)
+      await emptyBuckets(gcsStorage, gcsBucketName)
 
       const { awilixManager } = diContainer.cradle
       await awilixManager.executeDispose()
@@ -259,7 +259,7 @@ describe('PubSubPermissionConsumer - Payload Offloading', () => {
     })
 
     afterAll(async () => {
-      await emptyBucket(gcsStorage, gcsBucketName)
+      await emptyBuckets(gcsStorage, gcsBucketName)
 
       const { awilixManager } = diContainer.cradle
       await awilixManager.executeDispose()

packages/gcp-pubsub/test/utils/gcsUtils.ts

@@ -11,14 +11,16 @@ export async function assertBucket(storage: Storage, bucketName: string) {
   return bucket
 }
 
-export async function emptyBucket(storage: Storage, bucketName: string) {
-  const bucket = storage.bucket(bucketName)
-  const [exists] = await bucket.exists()
+export async function emptyBuckets(storage: Storage, ...bucketNames: string[]) {
+  for (const bucketName of bucketNames) {
+    const bucket = storage.bucket(bucketName)
+    const [exists] = await bucket.exists()
 
-  if (!exists) {
-    return
-  }
+    if (!exists) {
+      continue
+    }
 
-  const [files] = await bucket.getFiles()
-  await Promise.all(files.map((file) => file.delete({ ignoreNotFound: true })))
+    const [files] = await bucket.getFiles()
+    await Promise.all(files.map((file) => file.delete({ ignoreNotFound: true })))
+  }
 }

packages/s3-payload-store/package.json

@@ -27,8 +27,6 @@
         "test:coverage": "npm run test -- --coverage",
         "lint": "biome check . && tsc",
         "lint:fix": "biome check --write .",
-        "docker:start": "docker compose up -d --quiet-pull localstack",
-        "docker:stop": "docker compose down",
         "prepublishOnly": "npm run lint && npm run build"
     },
     "dependencies": {},
@@ -43,8 +41,9 @@
         "@lokalise/tsconfig": "^3.0.0",
         "@types/node": "^25.0.2",
         "@vitest/coverage-v8": "^4.0.15",
+        "fauxqs": "^1.10.0",
         "rimraf": "^6.0.1",
-        "typescript": "^5.9.2",
+        "typescript": "^5.9.3",
         "vitest": "^4.0.15"
     },
     "homepage": "https://github.com/kibertoad/message-queue-toolkit",

packages/s3-payload-store/test/store/S3PayloadStore.spec.ts

@@ -6,7 +6,7 @@ import { beforeAll, beforeEach, describe, expect, it } from 'vitest'
 import { resolvePayloadStoreConfig, S3PayloadStore } from '../../lib/S3PayloadStore.ts'
 import { assertEmptyBucket, getObjectContent, objectExists } from '../utils/s3Utils.ts'
 import { streamToString } from '../utils/streamUtils.ts'
-import { TEST_AWS_CONFIG } from '../utils/testS3Config.ts'
+import { getTestS3Config } from '../utils/testS3Config.ts'
 
 const TEST_BUCKET = 'test-bucket'
 
@@ -15,7 +15,7 @@ describe('S3PayloadStore', () => {
   let store: S3PayloadStore
 
   beforeAll(() => {
-    s3 = new S3(TEST_AWS_CONFIG)
+    s3 = new S3(getTestS3Config())
     store = new S3PayloadStore({ s3 }, { bucketName: TEST_BUCKET })
   })
   beforeEach(async () => {

packages/s3-payload-store/test/utils/fauxqsInstance.ts

@@ -0,0 +1,19 @@
+/** biome-ignore-all lint/suspicious/noConsole: test **/
+import { type FauxqsServer, startFauxqs } from 'fauxqs'
+
+let server: FauxqsServer | undefined
+
+export async function ensureFauxqsServer(): Promise<void> {
+  if (server) return
+  server = await startFauxqs({ port: 0, logger: false, host: 'localstack' })
+  console.log(`[fauxqs] server started on port ${server.port}`)
+}
+
+export function getFauxqsServer(): FauxqsServer | undefined {
+  return server
+}
+
+export function getFauxqsPort(): number {
+  if (!server) throw new Error('fauxqs server not started — call ensureFauxqsServer() first')
+  return server.port
+}

packages/s3-payload-store/test/utils/testS3Config.ts

@@ -1,10 +1,17 @@
 import type { S3ClientConfig } from '@aws-sdk/client-s3'
+import { createLocalhostHandler } from 'fauxqs'
 
-export const TEST_AWS_CONFIG: S3ClientConfig = {
-  endpoint: 'http://s3.localhost.localstack.cloud:4566',
-  region: 'eu-west-1',
-  credentials: {
-    accessKeyId: 'access',
-    secretAccessKey: 'secret',
-  },
+import { getFauxqsPort } from './fauxqsInstance.ts'
+
+export function getTestS3Config(): S3ClientConfig {
+  const port = getFauxqsPort()
+  return {
+    endpoint: `http://s3.localhost:${port}`,
+    region: 'eu-west-1',
+    credentials: {
+      accessKeyId: 'access',
+      secretAccessKey: 'secret',
+    },
+    requestHandler: createLocalhostHandler(),
+  }
 }

packages/s3-payload-store/test/utils/vitest.setup.ts

@@ -0,0 +1,3 @@
+import { ensureFauxqsServer } from './fauxqsInstance.ts'
+
+await ensureFauxqsServer()

packages/s3-payload-store/vitest.config.ts

@@ -7,6 +7,8 @@ export default defineConfig({
     watch: false,
     mockReset: true,
     pool: 'threads',
+    maxWorkers: 1,
+    setupFiles: ['test/utils/vitest.setup.ts'],
     coverage: {
       provider: 'v8',
       include: ['lib/**/*.ts'],

packages/sns/package.json

@@ -23,7 +23,7 @@
         "test:coverage": "npm run test -- --coverage",
         "lint": "biome check . && tsc",
         "lint:fix": "biome check --write .",
-        "docker:start": "docker compose up -d localstack redis",
+        "docker:start": "docker compose up -d redis && if [ \"$QUEUE_BACKEND\" = \"localstack\" ]; then docker compose up -d --quiet-pull --wait localstack; fi",
         "docker:stop": "docker compose down",
         "prepublishOnly": "npm run lint && npm run build"
     },
@@ -53,8 +53,9 @@
         "@message-queue-toolkit/sqs": "*",
         "@types/node": "^25.0.2",
         "@vitest/coverage-v8": "^4.0.15",
-        "awilix": "^12.0.5",
+        "awilix": "^12.1.1",
         "awilix-manager": "^6.1.0",
+        "fauxqs": "^1.10.0",
         "ioredis": "^5.7.0",
         "rimraf": "^6.0.1",
         "typescript": "^5.9.3",

packages/sns/test/consumers/CreateLocateConfigMixConsumer.spec.ts

@@ -1,35 +1,25 @@
-import type { SNSClient } from '@aws-sdk/client-sns'
-import type { SQSClient } from '@aws-sdk/client-sqs'
-import type { STSClient } from '@aws-sdk/client-sts'
-import { deleteQueue } from '@message-queue-toolkit/sqs'
 import type { AwilixContainer } from 'awilix'
 import { beforeAll, beforeEach, describe, it } from 'vitest'
-import { assertTopic, deleteTopic } from '../../lib/utils/snsUtils.ts'
+import type { TestAwsResourceAdmin } from '../utils/testAdmin.ts'
 import { type Dependencies, registerDependencies } from '../utils/testContext.ts'
 import { CreateLocateConfigMixConsumer } from './CreateLocateConfigMixConsumer.ts'
 
 describe('CreateLocateConfigMixConsumer', () => {
   let diContainer: AwilixContainer<Dependencies>
-  let sqsClient: SQSClient
-  let snsClient: SNSClient
-  let stsClient: STSClient
+  let testAdmin: TestAwsResourceAdmin
 
   beforeAll(async () => {
     diContainer = await registerDependencies({}, false)
-    sqsClient = diContainer.cradle.sqsClient
-    snsClient = diContainer.cradle.snsClient
-    stsClient = diContainer.cradle.stsClient
+    testAdmin = diContainer.cradle.testAdmin
   })
 
   beforeEach(async () => {
-    await deleteQueue(sqsClient, CreateLocateConfigMixConsumer.CONSUMED_QUEUE_NAME)
-    await deleteTopic(snsClient, stsClient, CreateLocateConfigMixConsumer.SUBSCRIBED_TOPIC_NAME)
+    await testAdmin.deleteQueues(CreateLocateConfigMixConsumer.CONSUMED_QUEUE_NAME)
+    await testAdmin.deleteTopics(CreateLocateConfigMixConsumer.SUBSCRIBED_TOPIC_NAME)
   })
 
   it('accepts mixed config of create and locate', async () => {
-    await assertTopic(snsClient, stsClient, {
-      Name: CreateLocateConfigMixConsumer.SUBSCRIBED_TOPIC_NAME,
-    })
+    await testAdmin.createTopic(CreateLocateConfigMixConsumer.SUBSCRIBED_TOPIC_NAME)
     const consumer = new CreateLocateConfigMixConsumer(diContainer.cradle)
     await consumer.init()
   })