Deployment Guide

Cloudflare Workers deployment, CI/CD configuration, and environment management

Prerequisites
Project Structure
Wrangler Configuration
Environment Setup
CI/CD Pipeline
Deployment Commands
Database Migrations
Secrets Management
Monitoring Setup
Rollback Procedures

Prerequisites

Required Tools

# Node.js 18+
node --version  # v18.0.0 or higher

# pnpm (recommended) or npm
pnpm --version  # 8.0.0 or higher

# Wrangler CLI
pnpm add -g wrangler
wrangler --version  # 3.0.0 or higher

# Authenticate with Cloudflare
wrangler login

Cloudflare Account Requirements

Workers Paid plan (for Workflows, Vectorize, higher limits)
D1 access enabled
R2 bucket created
Vectorize indexes created
Queues created

Project Structure

noozer/
├── api/
│   ├── src/
│   │   ├── workers/
│   │   │   ├── api-gateway/      # Public API routes
│   │   │   ├── admin-api/        # Admin endpoints
│   │   │   ├── crawler/          # Queue consumer: fetch
│   │   │   ├── extractor/        # Queue consumer: parse
│   │   │   ├── processor/        # Queue consumer: enrich
│   │   │   ├── classifier/       # Queue consumer: classify
│   │   │   ├── matcher/          # Queue consumer: match
│   │   │   └── notifier/         # Queue consumer: notify
│   │   ├── workflows/
│   │   │   ├── scheduled-crawl.ts
│   │   │   ├── story-recluster.ts
│   │   │   └── retention-cleanup.ts
│   │   ├── shared/
│   │   │   ├── db/
│   │   │   ├── services/
│   │   │   ├── utils/
│   │   │   └── types/
│   │   └── index.ts
│   ├── migrations/
│   │   ├── 0001_initial_schema.sql
│   │   ├── 0002_customer_tables.sql
│   │   └── ...
│   ├── wrangler.toml
│   ├── wrangler.dev.toml
│   ├── wrangler.staging.toml
│   └── package.json
├── docs/
└── README.md

Wrangler Configuration

Production: `wrangler.toml`

name = "noozer-api"
main = "src/index.ts"
compatibility_date = "2024-01-01"
compatibility_flags = ["nodejs_compat"]

# Account and zone (optional for workers.dev)
# account_id = "your-account-id"
# zone_id = "your-zone-id"

# Custom domain
# routes = [
#   { pattern = "api.noozer.io/*", zone_name = "noozer.io" }
# ]

# Worker limits
[limits]
cpu_ms = 50000

# ============================================================================
# D1 DATABASE
# ============================================================================
[[d1_databases]]
binding = "DB"
database_name = "noozer-production"
database_id = "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"

# ============================================================================
# R2 BUCKETS
# ============================================================================
[[r2_buckets]]
binding = "RAW_SNAPSHOTS"
bucket_name = "noozer-raw-snapshots-prod"

[[r2_buckets]]
binding = "EXPORTS"
bucket_name = "noozer-exports-prod"

# ============================================================================
# KV NAMESPACES
# ============================================================================
[[kv_namespaces]]
binding = "RATE_LIMITS"
id = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"

[[kv_namespaces]]
binding = "HOT_CACHE"
id = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"

[[kv_namespaces]]
binding = "FEATURE_FLAGS"
id = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"

# ============================================================================
# QUEUES
# ============================================================================
[[queues.producers]]
queue = "crawl-batch-prod"
binding = "CRAWL_BATCH_QUEUE"

[[queues.producers]]
queue = "article-extract-prod"
binding = "ARTICLE_EXTRACT_QUEUE"

[[queues.producers]]
queue = "article-enrich-prod"
binding = "ARTICLE_ENRICH_QUEUE"

[[queues.producers]]
queue = "article-process-prod"
binding = "ARTICLE_PROCESS_QUEUE"

[[queues.producers]]
queue = "article-classify-prod"
binding = "ARTICLE_CLASSIFY_QUEUE"

[[queues.producers]]
queue = "story-cluster-prod"
binding = "STORY_CLUSTER_QUEUE"

[[queues.producers]]
queue = "customer-match-prod"
binding = "CUSTOMER_MATCH_QUEUE"

[[queues.producers]]
queue = "notify-dispatch-prod"
binding = "NOTIFY_DISPATCH_QUEUE"

[[queues.producers]]
queue = "profile-rebuild-prod"
binding = "PROFILE_REBUILD_QUEUE"

# Queue consumers are defined in separate worker configs
# See queue-workers.toml

# ============================================================================
# VECTORIZE INDEXES
# ============================================================================
[[vectorize]]
binding = "VECTORIZE_ARTICLES"
index_name = "noozer-articles-prod"

[[vectorize]]
binding = "VECTORIZE_STORIES"
index_name = "noozer-stories-prod"

[[vectorize]]
binding = "VECTORIZE_PROFILES"
index_name = "noozer-profiles-prod"

[[vectorize]]
binding = "VECTORIZE_TAXONOMY"
index_name = "noozer-taxonomy-prod"

[[vectorize]]
binding = "VECTORIZE_ENTITIES"
index_name = "noozer-entities-prod"

[[vectorize]]
binding = "VECTORIZE_LOCATIONS"
index_name = "noozer-locations-prod"

[[vectorize]]
binding = "VECTORIZE_AUTHORS"
index_name = "noozer-authors-prod"

# ============================================================================
# WORKFLOWS
# ============================================================================
[[workflows]]
binding = "SCHEDULED_CRAWL"
name = "scheduled-crawl-prod"
class_name = "ScheduledCrawlWorkflow"

[[workflows]]
binding = "STORY_RECLUSTER"
name = "story-recluster-prod"
class_name = "StoryReclusterWorkflow"

[[workflows]]
binding = "RETENTION_CLEANUP"
name = "retention-cleanup-prod"
class_name = "RetentionCleanupWorkflow"

# ============================================================================
# CRON TRIGGERS
# ============================================================================
[triggers]
crons = [
  "*/15 * * * *",  # Every 15 minutes - high priority crawl
  "0 * * * *",     # Every hour - standard crawl
  "0 6 * * *",     # Daily at 6 AM - low priority crawl
  "0 */4 * * *",   # Every 4 hours - story recluster
  "0 2 * * *",     # Daily at 2 AM - retention cleanup
  "5 0 * * *",     # Daily at 00:05 - cost rollup
  "0 3 * * 0",     # Weekly Sunday 3 AM - authority recalc
]

# ============================================================================
# ENVIRONMENT VARIABLES (non-secret)
# ============================================================================
[vars]
ENVIRONMENT = "production"
LOG_LEVEL = "info"
API_VERSION = "v1"
DEFAULT_LANGUAGE = "en"
DEFAULT_REGION = "US"
MAX_ARTICLES_PER_CRAWL = "500"
ARTICLE_RETENTION_DAYS = "90"
COST_ALERT_EMAIL = "ops@noozer.io"

# ============================================================================
# OBSERVABILITY
# ============================================================================
[observability]
enabled = true
head_sampling_rate = 1

[observability.logs]
enabled = true
invocation_logs = true

Development: `wrangler.dev.toml`

name = "noozer-api-dev"
main = "src/index.ts"
compatibility_date = "2024-01-01"
compatibility_flags = ["nodejs_compat"]

# Use local mode for development
# wrangler dev --local

[[d1_databases]]
binding = "DB"
database_name = "noozer-dev"
database_id = "local"  # Uses local SQLite

[[r2_buckets]]
binding = "RAW_SNAPSHOTS"
bucket_name = "noozer-raw-snapshots-dev"

[[r2_buckets]]
binding = "EXPORTS"
bucket_name = "noozer-exports-dev"

[[kv_namespaces]]
binding = "RATE_LIMITS"
id = "dev-rate-limits"

[[kv_namespaces]]
binding = "HOT_CACHE"
id = "dev-hot-cache"

[[kv_namespaces]]
binding = "FEATURE_FLAGS"
id = "dev-feature-flags"

[vars]
ENVIRONMENT = "development"
LOG_LEVEL = "debug"
API_VERSION = "v1"

Staging: `wrangler.staging.toml`

name = "noozer-api-staging"
main = "src/index.ts"
compatibility_date = "2024-01-01"
compatibility_flags = ["nodejs_compat"]

routes = [
  { pattern = "api.staging.noozer.io/*", zone_name = "noozer.io" }
]

[[d1_databases]]
binding = "DB"
database_name = "noozer-staging"
database_id = "yyyyyyyy-yyyy-yyyy-yyyy-yyyyyyyyyyyy"

# ... similar to production but with -staging suffixes

[vars]
ENVIRONMENT = "staging"
LOG_LEVEL = "debug"
API_VERSION = "v1"

Environment Setup

Create Cloudflare Resources

# 1. Create D1 databases
wrangler d1 create noozer-production
wrangler d1 create noozer-staging

# 2. Create R2 buckets
wrangler r2 bucket create noozer-raw-snapshots-prod
wrangler r2 bucket create noozer-raw-snapshots-staging
wrangler r2 bucket create noozer-exports-prod
wrangler r2 bucket create noozer-exports-staging

# 3. Create KV namespaces
wrangler kv:namespace create RATE_LIMITS
wrangler kv:namespace create HOT_CACHE
wrangler kv:namespace create FEATURE_FLAGS

# 4. Create Queues
wrangler queues create crawl-batch-prod
wrangler queues create article-extract-prod
wrangler queues create article-enrich-prod
wrangler queues create article-process-prod
wrangler queues create article-classify-prod
wrangler queues create story-cluster-prod
wrangler queues create customer-match-prod
wrangler queues create notify-dispatch-prod
wrangler queues create profile-rebuild-prod

# 5. Create Vectorize indexes
wrangler vectorize create noozer-articles-prod --dimensions 1536 --metric cosine
wrangler vectorize create noozer-stories-prod --dimensions 1536 --metric cosine
wrangler vectorize create noozer-profiles-prod --dimensions 1536 --metric cosine
wrangler vectorize create noozer-taxonomy-prod --dimensions 1536 --metric cosine
wrangler vectorize create noozer-entities-prod --dimensions 1536 --metric cosine
wrangler vectorize create noozer-locations-prod --dimensions 1536 --metric cosine
wrangler vectorize create noozer-authors-prod --dimensions 1536 --metric cosine

Update wrangler.toml with IDs

After creating resources, update your wrangler.toml with the returned IDs:

# Get D1 database ID
wrangler d1 list

# Get KV namespace IDs
wrangler kv:namespace list

# Get Vectorize index info
wrangler vectorize list

CI/CD Pipeline

GitHub Actions: `.github/workflows/deploy.yml`

name: Deploy

on:
  push:
    branches: [main, staging]
  pull_request:
    branches: [main]

env:
  CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
  CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}

jobs:
  test:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      
      - uses: pnpm/action-setup@v2
        with:
          version: 8
          
      - uses: actions/setup-node@v4
        with:
          node-version: '20'
          cache: 'pnpm'
          
      - name: Install dependencies
        run: pnpm install --frozen-lockfile
        
      - name: Type check
        run: pnpm typecheck
        
      - name: Lint
        run: pnpm lint
        
      - name: Unit tests
        run: pnpm test:unit
        
      - name: Integration tests
        run: pnpm test:integration

  deploy-staging:
    needs: test
    runs-on: ubuntu-latest
    if: github.ref == 'refs/heads/staging'
    environment: staging
    
    steps:
      - uses: actions/checkout@v4
      
      - uses: pnpm/action-setup@v2
        with:
          version: 8
          
      - uses: actions/setup-node@v4
        with:
          node-version: '20'
          cache: 'pnpm'
          
      - name: Install dependencies
        run: pnpm install --frozen-lockfile
        
      - name: Build
        run: pnpm build
        
      - name: Run migrations
        run: |
          wrangler d1 migrations apply noozer-staging --config wrangler.staging.toml
          
      - name: Deploy to staging
        run: |
          wrangler deploy --config wrangler.staging.toml
          
      - name: Smoke tests
        run: pnpm test:smoke --env staging

  deploy-production:
    needs: test
    runs-on: ubuntu-latest
    if: github.ref == 'refs/heads/main'
    environment: production
    
    steps:
      - uses: actions/checkout@v4
      
      - uses: pnpm/action-setup@v2
        with:
          version: 8
          
      - uses: actions/setup-node@v4
        with:
          node-version: '20'
          cache: 'pnpm'
          
      - name: Install dependencies
        run: pnpm install --frozen-lockfile
        
      - name: Build
        run: pnpm build
        
      - name: Run migrations
        run: |
          wrangler d1 migrations apply noozer-production
          
      - name: Deploy to production
        run: |
          wrangler deploy
          
      - name: Smoke tests
        run: pnpm test:smoke --env production
        
      - name: Notify Slack
        if: always()
        uses: slackapi/slack-github-action@v1
        with:
          payload: |
            {
              "text": "Production deploy ${{ job.status }}: ${{ github.sha }}"
            }
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

Queue Worker Deployment

Queue consumers need separate deployments:

# .github/workflows/deploy-workers.yml
name: Deploy Queue Workers

on:
  push:
    branches: [main]
    paths:
      - 'api/src/workers/**'

jobs:
  deploy-workers:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        worker:
          - crawler
          - extractor
          - processor
          - classifier
          - matcher
          - notifier
    steps:
      - uses: actions/checkout@v4
      
      - name: Deploy ${{ matrix.worker }}
        run: |
          wrangler deploy \
            --config api/src/workers/${{ matrix.worker }}/wrangler.toml

Deployment Commands

Local Development

# Start local dev server
pnpm dev

# With local D1/R2/KV persistence
pnpm dev --persist

# Test specific worker
wrangler dev src/workers/crawler/index.ts --config wrangler.dev.toml

Staging Deployment

# Deploy all workers to staging
pnpm deploy:staging

# Deploy specific worker
wrangler deploy --config wrangler.staging.toml

# View staging logs
wrangler tail --config wrangler.staging.toml

Production Deployment

# Deploy to production
pnpm deploy:prod

# Or manual
wrangler deploy

# Deploy with specific version tag
wrangler deploy --tag v1.2.3

# View production logs
wrangler tail

Database Migrations

Migration File Format

Migrations are in api/migrations/ with format XXXX_description.sql:

-- 0001_initial_schema.sql
-- Migration: Initial schema
-- Created: 2024-01-15

CREATE TABLE IF NOT EXISTS sources (
    id TEXT PRIMARY KEY,
    name TEXT NOT NULL,
    -- ...
);

-- Always include down migration in comments for reference:
-- DOWN:
-- DROP TABLE IF EXISTS sources;

Running Migrations

# List pending migrations
wrangler d1 migrations list noozer-production

# Apply all pending migrations (production)
wrangler d1 migrations apply noozer-production

# Apply to staging
wrangler d1 migrations apply noozer-staging --config wrangler.staging.toml

# Apply to local dev
wrangler d1 migrations apply noozer-dev --local

# Create new migration
wrangler d1 migrations create noozer-production "add_new_feature"

Migration Best Practices

Always test locally first

wrangler d1 migrations apply noozer-dev --local

Make migrations idempotent

CREATE TABLE IF NOT EXISTS ...
CREATE INDEX IF NOT EXISTS ...

Avoid destructive operations in production
- Add columns as nullable first
- Deprecate before removing
- Use feature flags during transitions
Keep migrations small
- One logical change per migration
- Easier to rollback

Secrets Management

Setting Secrets

# Set individual secret
wrangler secret put ZENROWS_API_KEY
# (prompts for value)

# Set from file
cat secrets.txt | wrangler secret put ZENROWS_API_KEY

# Bulk set from .env
wrangler secret:bulk < .env.production

# List secrets (names only)
wrangler secret list

Required Secrets

# External API keys
wrangler secret put ZENROWS_API_KEY
wrangler secret put DATA4SEO_API_KEY
wrangler secret put SHAREDCOUNT_API_KEY
wrangler secret put OPENAI_API_KEY

# Auth secrets
wrangler secret put JWT_SECRET           # For admin JWT signing
wrangler secret put WEBHOOK_SIGNING_KEY  # For outbound webhooks
wrangler secret put ADMIN_API_KEY        # Initial admin bootstrap

# Notification services
wrangler secret put SENDGRID_API_KEY
wrangler secret put SLACK_BOT_TOKEN

# Monitoring
wrangler secret put SENTRY_DSN

Environment-Specific Secrets

# Staging secrets
wrangler secret put ZENROWS_API_KEY --config wrangler.staging.toml

# Use different keys for different environments
# Production: real API keys
# Staging: sandbox/test API keys
# Dev: mock keys or local stubs

Monitoring Setup

Cloudflare Analytics

Built-in analytics available in Cloudflare dashboard:

Request counts
Error rates
Latency percentiles
CPU time usage

Custom Metrics with Logpush

# wrangler.toml
[logpush]
enabled = true
destination = "https://your-analytics.example.com"

Alerting with Cloudflare Notifications

Configure in Cloudflare dashboard:

Go to Notifications
Create notification for:
- Worker errors > threshold
- Latency > threshold
- Request count anomalies

External Monitoring

// src/shared/monitoring.ts
import { Toucan } from 'toucan-js';

export function initSentry(env: Env, ctx: ExecutionContext) {
  return new Toucan({
    dsn: env.SENTRY_DSN,
    context: ctx,
    environment: env.ENVIRONMENT,
    release: env.VERSION,
  });
}

Health Check Endpoint

// src/workers/api-gateway/health.ts
export async function handleHealth(env: Env): Promise<Response> {
  const checks = await Promise.allSettled([
    checkD1(env.DB),
    checkKV(env.HOT_CACHE),
    checkVectorize(env.VECTORIZE_ARTICLES),
  ]);
  
  const healthy = checks.every(c => c.status === 'fulfilled');
  
  return Response.json({
    status: healthy ? 'healthy' : 'degraded',
    checks: checks.map((c, i) => ({
      name: ['d1', 'kv', 'vectorize'][i],
      status: c.status === 'fulfilled' ? 'ok' : 'error',
    })),
    timestamp: new Date().toISOString(),
  }, { status: healthy ? 200 : 503 });
}

Rollback Procedures

Immediate Rollback

# List recent deployments
wrangler deployments list

# Rollback to previous version
wrangler rollback

# Rollback to specific version
wrangler rollback --version <version-id>

Database Rollback

D1 doesn't support automatic rollback. Manual approach:

# 1. Create backup before migration
wrangler d1 export noozer-production --output backup.sql

# 2. If migration fails, restore
wrangler d1 execute noozer-production --file backup.sql

# 3. Or run manual rollback SQL
wrangler d1 execute noozer-production --file rollback_0005.sql

Feature Flag Rollback

Use feature flags for gradual rollouts:

// Check feature flag before using new code path
const useNewClassifier = await env.FEATURE_FLAGS.get('new_classifier') === 'true';
if (useNewClassifier) {
  return newClassify(article);
} else {
  return legacyClassify(article);
}

# Disable feature instantly
wrangler kv:key put --binding FEATURE_FLAGS "new_classifier" "false"

Queue Draining

If queue consumers are failing:

# 1. Pause queue consumption
wrangler queues consumer pause article-classify-prod

# 2. Fix the issue and deploy

# 3. Resume consumption
wrangler queues consumer resume article-classify-prod

Appendix: Package Scripts

{
  "scripts": {
    "dev": "wrangler dev --config wrangler.dev.toml",
    "dev:persist": "wrangler dev --config wrangler.dev.toml --persist",
    "build": "tsc && esbuild src/index.ts --bundle --outfile=dist/index.js",
    "typecheck": "tsc --noEmit",
    "lint": "eslint src/",
    "lint:fix": "eslint src/ --fix",
    "test": "vitest",
    "test:unit": "vitest run --config vitest.unit.config.ts",
    "test:integration": "vitest run --config vitest.integration.config.ts",
    "test:smoke": "vitest run --config vitest.smoke.config.ts",
    "deploy:staging": "wrangler deploy --config wrangler.staging.toml",
    "deploy:prod": "wrangler deploy",
    "migrate:local": "wrangler d1 migrations apply noozer-dev --local",
    "migrate:staging": "wrangler d1 migrations apply noozer-staging --config wrangler.staging.toml",
    "migrate:prod": "wrangler d1 migrations apply noozer-production",
    "tail": "wrangler tail",
    "tail:staging": "wrangler tail --config wrangler.staging.toml"
  }
}

Last updated: 2024-01-15

Table of Contents​

Prerequisites​

Required Tools​

Cloudflare Account Requirements​

Project Structure​

Wrangler Configuration​

Production: wrangler.toml​

Development: wrangler.dev.toml​

Staging: wrangler.staging.toml​

Environment Setup​

Create Cloudflare Resources​

Update wrangler.toml with IDs​

CI/CD Pipeline​

GitHub Actions: .github/workflows/deploy.yml​

Queue Worker Deployment​

Deployment Commands​

Local Development​

Staging Deployment​

Production Deployment​

Database Migrations​

Migration File Format​

Running Migrations​

Migration Best Practices​

Secrets Management​

Setting Secrets​

Required Secrets​

Environment-Specific Secrets​

Monitoring Setup​

Cloudflare Analytics​

Custom Metrics with Logpush​

Alerting with Cloudflare Notifications​

External Monitoring​

Health Check Endpoint​

Rollback Procedures​

Immediate Rollback​

Database Rollback​

Feature Flag Rollback​

Queue Draining​

Appendix: Package Scripts​

Table of Contents