Migrate build system to esbuild and add environment validation (#1234)

* refactor: migrate build system to esbuild and add environment validation

Replace rollup with esbuild bundler for all client libraries, improve TypeScript configurations,
and introduce Zod-based environment variable validation for console webapp with separate
client/server schemas. Fix Next.js env var inlining in clientEnv by using explicit property
access instead of passing process.env object to Zod.

* functions-server and profiles - consolidate all in rotor package
* libs/core-functions move mongodb, isolated-vm and other heavy deps directly into rotor

Author: vklimontovich

.github/workflows/client-libraries-build.yaml

@@ -147,15 +147,15 @@ jobs:
 
           # Publish @jitsu/protocols
           echo "Publishing @jitsu/protocols..."
-          docker exec builder sh -c "cd types/protocols && npm publish --tag ${NPM_TAG} --access public"
+          docker exec builder sh -c "cd types/protocols && pnpm publish --tag ${NPM_TAG} --access public"
 
           # Publish @jitsu/js
           echo "Publishing @jitsu/js..."
-          docker exec builder sh -c "cd libs/jitsu-js && npm publish --tag ${NPM_TAG} --access public"
+          docker exec builder sh -c "cd libs/jitsu-js && pnpm publish --tag ${NPM_TAG} --access public"
 
           # Publish @jitsu/jitsu-react
           echo "Publishing @jitsu/jitsu-react..."
-          docker exec builder sh -c "cd libs/jitsu-react && npm publish --tag ${NPM_TAG} --access public"
+          docker exec builder sh -c "cd libs/jitsu-react && pnpm publish --tag ${NPM_TAG} --access public"
 
       - name: 🧹 Cleanup container
         if: always()

.github/workflows/lint.yml

@@ -40,8 +40,18 @@ jobs:
               - '**/playwright/**'
               - '**/*.spec.ts'
               - '**/playwright.config.ts'
+            nodejs:
+              - 'cli/**'
+              - 'libs/**'
+              - 'services/**'
+              - 'types/**'
+              - 'webapps/**'
+              - 'package.json'
+              - 'pnpm-lock.yaml'
+              - 'pnpm-workspace.yaml'
 
       - name: 🚀 Setup Turbo cache
+        if: steps.changes.outputs.nodejs == 'true'
         uses: actions/cache@v4
         with:
           path: |
@@ -52,39 +62,50 @@ jobs:
             ${{ runner.os }}-turbo-lint-
 
       - name: 🐳 Pull and start builder container
+        if: steps.changes.outputs.nodejs == 'true'
         run: |
           docker pull jitsucom/jitsu-builder:latest
           docker run -d --name builder -v ${{ github.workspace }}:/workspace -w /workspace -e CI=true jitsucom/jitsu-builder:latest tail -f /dev/null
 
       - name: 📦 Fetch dependencies from store
+        if: steps.changes.outputs.nodejs == 'true'
         run: |
           docker exec builder sh -c 'echo "pnpm store path: $(pnpm store path)"'
           docker exec builder pnpm fetch
 
       - name: 📦 Install dependencies
+        if: steps.changes.outputs.nodejs == 'true'
         run: docker exec builder pnpm install --frozen-lockfile --offline
 
       - name: ✨ Check code format
+        if: steps.changes.outputs.nodejs == 'true'
         run: docker exec builder pnpm format:check:all
 
       - name: 🔧 Run codegen
+        if: steps.changes.outputs.nodejs == 'true'
         run: docker exec builder pnpm codegen
 
       - name: 🧪️ Run Typecheck
+        if: steps.changes.outputs.nodejs == 'true'
         run: docker exec builder pnpm typecheck
 
       - name: 🧪 Run linter
+        if: steps.changes.outputs.nodejs == 'true'
         run: docker exec builder pnpm lint
 
       - name: 🧪 Run tests
+        if: steps.changes.outputs.nodejs == 'true'
         run: docker exec builder pnpm test
 
       - name: 🎭 Run Playwright tests
-        if: steps.changes.outputs.playwright == 'true'
-        run: docker exec builder pnpm test:playwright
+        if: steps.changes.outputs.nodejs == 'true' && steps.changes.outputs.playwright == 'true'
+        # Note: build of @jitsu/js and all it's dep is required for playwright tests to work
+        run: |
+          docker exec builder pnpm --filter jsondiffpatch --filter @jitsu/js build
+          docker exec builder pnpm test:playwright
 
       - name: 🧹 Cleanup container
-        if: always()
+        if: always() && steps.changes.outputs.nodejs == 'true'
         run: docker stop builder && docker rm builder
 
   bulker-test:
@@ -100,40 +121,52 @@ jobs:
           fetch-depth: 0
           submodules: true
 
+      - name: 🔍 Check changed files
+        id: changes
+        uses: dorny/paths-filter@v3
+        with:
+          filters: |
+            bulker:
+              - 'bulker/**'
+
       - name: Set up Go
-        uses: actions/setup-go@v5
+        if: steps.changes.outputs.bulker == 'true'
+        uses: actions/setup-go@v6
         with:
-          go-version: '1.24.9'
-          cache: false
+          go-version: '1.25'
+          cache-dependency-path: "bulker/**/*.sum"
+          check-latest: true
+#          cache: false
 
       - name: 🧪 Run Bulker tests
+        if: steps.changes.outputs.bulker == 'true'
         working-directory: ./bulker
-#        env:
-#          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-#          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-#          AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
-#          BULKER_TEST_BIGQUERY: ${{ secrets.BULKER_TEST_BIGQUERY }}
-#          BULKER_TEST_REDSHIFT_IAM: ${{ secrets.BULKER_TEST_REDSHIFT_IAM }}
-#          BULKER_TEST_S3: ${{ secrets.BULKER_TEST_S3 }}
-#          BULKER_TEST_MOTHERDUCK: ${{ secrets.BULKER_TEST_MOTHERDUCK }}
-#          BULKER_TEST_REDSHIFT_SERVERLESS: ${{ secrets.BULKER_TEST_REDSHIFT_SERVERLESS }}
-#          BULKER_TEST_SNOWFLAKE: ${{ secrets.BULKER_TEST_SNOWFLAKE }}
-#          BULKER_TEST_MILLION_ROWS_BATCHED: postgres
-#          TESTCONTAINERS_RYUK_DISABLED: true
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
+          BULKER_TEST_BIGQUERY: ${{ secrets.BULKER_TEST_BIGQUERY }}
+          BULKER_TEST_REDSHIFT_IAM: ${{ secrets.BULKER_TEST_REDSHIFT_IAM }}
+          BULKER_TEST_S3: ${{ secrets.BULKER_TEST_S3 }}
+          BULKER_TEST_MOTHERDUCK: ${{ secrets.BULKER_TEST_MOTHERDUCK }}
+          # BULKER_TEST_REDSHIFT: ${{ secrets.BULKER_TEST_REDSHIFT }}
+          BULKER_TEST_REDSHIFT_SERVERLESS: ${{ secrets.BULKER_TEST_REDSHIFT_SERVERLESS }}
+          BULKER_TEST_SNOWFLAKE: ${{ secrets.BULKER_TEST_SNOWFLAKE }}
+          BULKER_TEST_MILLION_ROWS_BATCHED: postgres
+          TESTCONTAINERS_RYUK_DISABLED: true
         run: |
-           exit 0
-#          go test -parallel=10 -timeout 60m -json ./jitsubase/... ./kafkabase/... ./eventslog/... ./bulkerlib/... ./bulkerapp/... > test_report.json
+           go test -parallel=10 -timeout 60m -json ./jitsubase/... ./kafkabase/... ./eventslog/... ./bulkerlib/... ./bulkerapp/... > test_report.json
 
 
       - name: 📊 Generate test report
-        if: always()
+        if: always() && steps.changes.outputs.bulker == 'true'
         working-directory: ./bulker
         run: |
           go install github.com/vakenbolt/go-test-report@latest
           cat test_report.json | ~/go/bin/go-test-report
 
       - name: 📤 Upload test artifacts
-        if: always()
+        if: always() && steps.changes.outputs.bulker == 'true'
         uses: actions/upload-artifact@v4
         with:
           name: bulker-test-report

.github/workflows/services-build.yaml

@@ -125,7 +125,7 @@ jobs:
           BRANCH: ${{ github.ref_name }}
           COMMIT_SHA: ${{ github.sha }}
         run: |
-          TARGETS="console rotor profiles"
+          TARGETS="console rotor"
           REGISTRY="${{ secrets.DOCKERHUB_USERNAME }}"
           SHORT_SHA=$(git rev-parse --short=7 HEAD)
 
@@ -217,7 +217,7 @@ jobs:
           BRANCH: ${{ github.ref_name }}
           COMMIT_SHA: ${{ github.sha }}
         run: |
-          TARGETS="bulker ingest sidecar syncctl ingmgr cfgkpr admin reprocessing-worker"
+          TARGETS="bulker ingest sidecar syncctl operator ingmgr cfgkpr admin reprocessing-worker"
           REGISTRY="${{ secrets.DOCKERHUB_USERNAME }}"
           SHORT_SHA=$(git rev-parse --short=7 HEAD)
           BUILD_TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
@@ -444,12 +444,16 @@ jobs:
 
           # Update CLI
           docker exec builder sh -c "cd cli/jitsu-cli && jq --arg version \"${VERSION}\" '.version = \$version' package.json > package.json.tmp && mv package.json.tmp package.json"
+          
+          # Update Types
+          docker exec builder sh -c "cd types/protocols && jq --arg version \"${VERSION}\" '.version = \$version' package.json > package.json.tmp && mv package.json.tmp package.json"
 
           # Update Functions Library
           docker exec builder sh -c "cd libs/functions && jq --arg version \"${VERSION}\" '.version = \$version' package.json > package.json.tmp && mv package.json.tmp package.json"
 
           echo "Updated versions:"
           docker exec builder sh -c "cd cli/jitsu-cli && cat package.json | grep version"
+          docker exec builder sh -c "cd types/protocols && cat package.json | grep version"
           docker exec builder sh -c "cd libs/functions && cat package.json | grep version"
 
       - name: 🏗️ Build packages
@@ -479,11 +483,15 @@ jobs:
 
           # Publish jitsu-cli
           echo "Publishing jitsu-cli..."
-          docker exec builder sh -c "cd cli/jitsu-cli && npm publish --tag ${NPM_TAG} --access public ${DRY_RUN_FLAG}"
-
+          docker exec builder sh -c "cd cli/jitsu-cli && pnpm publish --tag ${NPM_TAG} --access public ${DRY_RUN_FLAG}"
+          
+          # Publish @jitsu/protocols
+          echo "Publishing @jitsu/protocols..."
+          docker exec builder sh -c "cd types/protocols && pnpm publish --tag ${NPM_TAG} --access public ${DRY_RUN_FLAG}"
+          
           # Publish @jitsu/functions-lib
           echo "Publishing @jitsu/functions-lib..."
-          docker exec builder sh -c "cd libs/functions && npm publish --tag ${NPM_TAG} --access public ${DRY_RUN_FLAG}"
+          docker exec builder sh -c "cd libs/functions && pnpm publish --tag ${NPM_TAG} --access public ${DRY_RUN_FLAG}"
 
       - name: 🧹 Cleanup container
         if: always()

.gitignore

@@ -33,3 +33,5 @@ libs/jitsu-js/package/
 libs/jsondiffpatch/lib/
 .pnpm-store
 **/tsconfig.tsbuildinfo
+/services/functions-server/data/
+/services/functions-server/dist/

.npmrc

@@ -1,3 +1,2 @@
 auto-install-peers=true
 strict-peer-dependencies=false
-node-options=--max-old-space-size=16384

all.Dockerfile

@@ -28,7 +28,7 @@ WORKDIR /app
 # - procps: process management (ps, top, etc.)
 # - jq: JSON parsing for extracting package versions
 RUN apt-get update && \
-    apt-get install -y --no-install-recommends nano curl cron bash netcat-traditional procps jq && \
+    apt-get install -y --no-install-recommends ca-certificates nano curl cron bash netcat-traditional procps jq && \
     rm -rf /var/lib/apt/lists/*
 
 # ============================================================================
@@ -190,48 +190,13 @@ EXPOSE 3401
 # Copy compiled JavaScript from builder stage
 # The /dist folder contains the bundled Node.js application
 COPY --from=builder /app/services/rotor/dist .
+COPY --from=builder /app/services/rotor/entrypoint.sh .
 
 # Runtime environment configuration
 ENV NODE_ENV=production
 ENV JITSU_VERSION_COMMIT_SHA=${JITSU_BUILD_COMMIT_SHA}
 ENV JITSU_VERSION_DOCKER_TAG=${JITSU_BUILD_DOCKER_TAG}
 ENV JITSU_VERSION_STRING=${JITSU_BUILD_VERSION}
 
-# CMD provides flags to node binary (ENTRYPOINT would be "node" if specified)
-# Flags:
-#   --no-node-snapshot: Disable V8 snapshot (can cause issues in containers)
-#   --max-old-space-size=2048: Limit heap to 2GB (prevents OOM in constrained environments)
-#   main.js: The application entry point
-CMD ["--no-node-snapshot", "--max-old-space-size=2048", "main.js"]
-
-# ============================================================================
-# PROFILES STAGE - User profile management service
-# ============================================================================
-# Node.js service for managing user profiles and identity resolution
-FROM base AS profiles
-
-# Build arguments for version information
-ARG JITSU_BUILD_VERSION=dev,
-ARG JITSU_BUILD_DOCKER_TAG=dev,
-ARG JITSU_BUILD_COMMIT_SHA=unknown,
-
-WORKDIR /app
-
-# Create non-root user for security (same as rotor)
-RUN addgroup --system --gid 1001 runner
-RUN adduser --system --uid 1001 runner
-USER runner
-
-EXPOSE 3401
-
-# Copy compiled JavaScript from builder stage
-COPY --from=builder /app/services/profiles/dist .
-
-# Runtime environment configuration
-ENV NODE_ENV=production
-ENV JITSU_VERSION_COMMIT_SHA=${JITSU_BUILD_COMMIT_SHA}
-ENV JITSU_VERSION_DOCKER_TAG=${JITSU_BUILD_DOCKER_TAG}
-ENV JITSU_VERSION_STRING=${JITSU_BUILD_VERSION}
 
-# Node.js runtime flags (see rotor stage for detailed explanation)
-CMD ["--no-node-snapshot", "--max-old-space-size=2048", "main.js"]
+ENTRYPOINT ["/app/entrypoint.sh"]

builder.Dockerfile

@@ -1,53 +1,27 @@
-FROM debian:bookworm-slim
+FROM node:24-bookworm-slim
 
 # Install Node.js 24 manually from NodeSource + all runtime dependencies
 # This includes everything needed for building AND running the final images
 RUN apt-get update && \
-    apt-get install -y ca-certificates curl gnupg && \
-    mkdir -p /etc/apt/keyrings && \
-    curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \
-    echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_24.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list && \
-    apt-get update && \
-    apt-get install -y nodejs && \
-    apt-get install -y git curl telnet python3 g++ make jq nano cron bash netcat-traditional procps && \
+    apt-get install -y ca-certificates gnupg git curl telnet python3 g++ make jq nano cron bash netcat-traditional procps && \
     rm -rf /var/lib/apt/lists/* && \
     npm -g install pnpm@10 && \
     npm cache clean --force
 
+#print current user
+RUN whoami && echo "Current user is $(whoami)"
+
 # Set up pnpm global bin directory (for global package installs like Playwright)
 # Note: This does NOT affect the store location, which remains at /root/.local/share/pnpm/store
 ENV PNPM_HOME=/root/.local/share/pnpm
 ENV PATH="${PNPM_HOME}:${PATH}"
 # Override pnpm store location to avoid workspace-local stores
 ENV NPM_CONFIG_STORE_DIR=/pnpm-store
-ENV NODE_OPTIONS="--max-old-space-size=16384"
 
 # Copy only the files needed for dependency fetching and Playwright version extraction
 COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./
 COPY libs/jitsu-js/package.json ./libs/jitsu-js/package.json
 
-# Install minimal Chromium dependencies manually
-RUN apt-get update && \
-    apt-get install -y \
-    libnss3 \
-    libnspr4 \
-    libatk1.0-0 \
-    libatk-bridge2.0-0 \
-    libcups2 \
-    libdrm2 \
-    libdbus-1-3 \
-    libxkbcommon0 \
-    libxcomposite1 \
-    libxdamage1 \
-    libxfixes3 \
-    libxrandr2 \
-    libgbm1 \
-    libpango-1.0-0 \
-    libcairo2 \
-    libasound2 \
-    libatspi2.0-0 \
-    && rm -rf /var/lib/apt/lists/*
-
 # Extract Playwright version before cleanup
 RUN PLAYWRIGHT_VERSION=$(jq -r '.devDependencies["@playwright/test"]' ./libs/jitsu-js/package.json) && \
     echo "${PLAYWRIGHT_VERSION}" > /tmp/playwright-version.txt && \
@@ -64,7 +38,7 @@ RUN rm -rf /package.json /pnpm-lock.yaml /pnpm-workspace.yaml /libs
 RUN PLAYWRIGHT_VERSION=$(cat /tmp/playwright-version.txt) && \
     echo "Installing Playwright version: ${PLAYWRIGHT_VERSION}" && \
     pnpm add --global playwright@${PLAYWRIGHT_VERSION} && \
-    playwright install chromium
+    playwright install chromium  --with-deps --only-shell
 
 # Clean up any leftover node_modules
 RUN rm -rf /node_modules