Implement 3-Layer Data Persistence Architecture with Standardized Environment Variables

.env.example

@@ -1,9 +1,27 @@
+# Discord Bot Configuration
 DISCORD_BOT_TOKEN=your_discord_bot_token_here
 CLIENT_ID=your_client_id_here
 GUILD_ID=your_guild_id_here
+
+# Unthread Integration Configuration
 UNTHREAD_API_KEY=your_unthread_api_key_here
 UNTHREAD_SLACK_CHANNEL_ID=your_unthread_slack_channel_id_here
 UNTHREAD_WEBHOOK_SECRET=your_unthread_webhook_secret_here
-REDIS_URL=your_redis_url_here
+
+# 3-Layer Storage Architecture Configuration
+# PostgreSQL (L3) - Primary persistent storage
+DATABASE_URL=postgres://postgres:postgres@localhost:5432/unthread_discord_bot
+
+# Redis Cache (L2) - Distributed cache layer
+PLATFORM_REDIS_URL=redis://localhost:6379
+
+# Redis Queue - Queue processing for webhooks (automatically configured)
+WEBHOOK_REDIS_URL=redis://localhost:6380
+
+# Optional Configuration
 FORUM_CHANNEL_IDS=channel_id_1,channel_id_2,channel_id_3
-DEBUG_MODE=false
+DEBUG_MODE=false
+PORT=3000
+
+# HTTP Timeout Configuration (Optional)
+UNTHREAD_HTTP_TIMEOUT_MS=15000

README.md

@@ -87,17 +87,44 @@ You can use Railway to deploy this bot with just one click. Railway offers a sea
 
 ## 🏗️ Architecture
 
-This bot is built with **TypeScript** for enhanced maintainability, type safety, and developer experience. The codebase follows clean coding principles and the KISS (Keep It Simple, Stupid) methodology for easy maintenance and extensibility.
+This bot is built with **TypeScript** for enhanced maintainability, type safety, and developer experience. The codebase follows clean coding principles and implements a modern **3-layer data persistence architecture** for optimal performance and reliability.
+
+### 🚀 New 3-Layer Storage Architecture
+
+**Layer 1 (L1): In-Memory Cache**
+- Ultra-fast access for frequently used data
+- LRU eviction policy to manage memory efficiently
+- Automatic cache warming from lower layers
+
+**Layer 2 (L2): Redis Cache**
+- Distributed cache for persistence across restarts
+- Fast lookup with millisecond response times
+- Shared between application instances
+
+**Layer 3 (L3): PostgreSQL Database**
+- Primary source of truth for all data
+- ACID compliance and data integrity
+- Complex queries and reporting capabilities
 
 ### Technology Stack
 
 - **TypeScript**: For type safety and better code maintainability
 - **Discord.js v14**: Modern Discord API interactions
-- **Express.js**: Webhook server for Unthread integration
+- **Express.js**: RESTful API server with comprehensive monitoring
 - **Node.js 18+**: Runtime environment
 - **Yarn with PnP**: Package management and dependency resolution
 - **ESLint**: Code quality and consistent formatting
-- **Redis**: Required for caching and data persistence
+
+**Storage & Performance:**
+- **PostgreSQL**: Primary database with full ACID compliance
+- **Redis**: High-performance L2 cache and queue management
+- **BullMQ**: Robust job queue system for webhook processing
+- **IORedis**: High-performance Redis client with cluster support
+
+**Infrastructure:**
+- **Docker Compose**: Complete local development environment
+- **Health Monitoring**: Comprehensive health checks and metrics
+- **Queue Processing**: Async webhook handling with retry logic
 
 ### Build System
 
@@ -115,6 +142,54 @@ yarn deploycommand
 
 # Production start
 yarn start
+
+# Linting
+yarn lint
+yarn lint:fix
+```
+
+### 🐳 Local Development with Docker
+
+For the complete development experience with all dependencies:
+
+```bash
+# Start all services (PostgreSQL, Redis, Bot)
+docker-compose up -d
+
+# View logs
+docker-compose logs -f discord-bot
+
+# Stop all services
+docker-compose down
+
+# Reset all data
+docker-compose down -v
+```
+
+### 📊 Monitoring & Health Checks
+
+The bot provides comprehensive monitoring endpoints:
+
+- **GET /health** - Overall system health (Discord + Storage layers)
+- **GET /webhook/health** - Queue system health and metrics
+- **GET /webhook/metrics** - Detailed processing statistics
+- **POST /webhook/retry** - Manual retry of failed webhook jobs
+
+Example health check response:
+```json
+{
+  "status": "healthy",
+  "timestamp": "2024-01-01T00:00:00.000Z",
+  "services": {
+    "discord": "connected",
+    "storage": "healthy",
+    "storage_layers": {
+      "memory": true,
+      "redis": true,
+      "postgres": true
+    }
+  }
+}
 ```
 
 ## 📦 Manual Installation
@@ -166,23 +241,45 @@ yarn start
 
 > **Note**: The bot requires these specific permissions to create and manage threads for ticket handling. Missing permissions may cause functionality issues.
 
-### 4. Fill Out the Environment Files
+### 4. Setup Storage Dependencies
+
+**For Docker Compose (Recommended):**
+```bash
+# Use the provided Docker Compose configuration
+docker-compose up -d postgres redis-cache redis-queue
+```
+
+**For Manual Setup:**
+- **PostgreSQL 16+**: Required for L3 persistent storage
+- **Redis 7+**: Required for L2 cache and queue processing
+
+### 5. Fill Out the Environment Files
 
 1. Create a `.env` file in the root directory of your project.
 2. Copy the contents of `.env.example` to `.env`.
 3. Fill in the required information:
+
+**Discord Configuration:**
    - `DISCORD_BOT_TOKEN`: The token you copied from the "Bot" tab.
    - `CLIENT_ID`: Your application's client ID, found in the "General Information" tab.
    - `GUILD_ID`: The ID of the Discord server where you want to deploy the bot. [How to Get Your Discord Server ID](#how-to-get-your-discord-server-id)
+
+**Unthread Configuration:**
    - `UNTHREAD_API_KEY`: Your Unthread API key.
    - `UNTHREAD_SLACK_CHANNEL_ID`: Your Unthread Slack channel ID for ticket routing.
    - `UNTHREAD_WEBHOOK_SECRET`: Your Unthread webhook secret.
-   - `REDIS_URL`: Redis connection URL for caching and data persistence (required).
+
+**Storage Configuration (3-Layer Architecture):**
+   - `DATABASE_URL`: PostgreSQL connection string (e.g., `postgres://user:password@localhost:5432/database`)
+   - `PLATFORM_REDIS_URL`: Redis cache connection URL (e.g., `redis://localhost:6379`)
+   - `WEBHOOK_REDIS_URL`: Redis queue connection URL (e.g., `redis://localhost:6380`)
+
+**Optional Configuration:**
    - `FORUM_CHANNEL_IDS`: Comma-separated list of forum channel IDs for automatic ticket creation.
    - `DEBUG_MODE`: Set to `true` for verbose logging during development (default: `false`).
    - `PORT`: Port for the webhook server (default: `3000`).
 
-### 5. Install and Run the Project Locally
+### 6. Install and Run the Project Locally
 
 1. Clone the repository and navigate to the project directory:
 

docker-compose.yml

@@ -0,0 +1,103 @@
+version: '3.8'
+
+services:
+  # PostgreSQL Database - Primary persistent storage
+  postgres:
+    image: postgres:16-alpine
+    container_name: unthread-postgres
+    environment:
+      POSTGRES_DB: unthread_discord_bot
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: postgres
+    ports:
+      - "5432:5432"
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+      - ./init.sql:/docker-entrypoint-initdb.d/init.sql
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  # Redis Cache - L2 cache and bot operations
+  redis-cache:
+    image: redis:7-alpine
+    container_name: unthread-redis-cache
+    command: redis-server --appendonly yes --maxmemory 256mb --maxmemory-policy allkeys-lru
+    ports:
+      - "6379:6379"
+    volumes:
+      - redis_cache_data:/data
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 10s
+      timeout: 3s
+      retries: 5
+
+  # Redis Queue - Webhook processing and job queues
+  redis-queue:
+    image: redis:7-alpine
+    container_name: unthread-redis-queue
+    command: redis-server --appendonly yes --maxmemory 128mb --maxmemory-policy noeviction
+    ports:
+      - "6380:6379"
+    volumes:
+      - redis_queue_data:/data
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 10s
+      timeout: 3s
+      retries: 5
+
+  # Discord Bot Application
+  discord-bot:
+    build: .
+    container_name: unthread-discord-bot
+    depends_on:
+      postgres:
+        condition: service_healthy
+      redis-cache:
+        condition: service_healthy
+      redis-queue:
+        condition: service_healthy
+    environment:
+      # Discord Configuration
+      DISCORD_BOT_TOKEN: ${DISCORD_BOT_TOKEN}
+      CLIENT_ID: ${CLIENT_ID}
+      GUILD_ID: ${GUILD_ID}
+
+      # Unthread Configuration
+      UNTHREAD_API_KEY: ${UNTHREAD_API_KEY}
+      UNTHREAD_SLACK_CHANNEL_ID: ${UNTHREAD_SLACK_CHANNEL_ID}
+      UNTHREAD_WEBHOOK_SECRET: ${UNTHREAD_WEBHOOK_SECRET}
+
+      # Storage Configuration
+      DATABASE_URL: postgres://postgres:postgres@postgres:5432/unthread_discord_bot
+      PLATFORM_REDIS_URL: redis://redis-cache:6379
+      WEBHOOK_REDIS_URL: redis://redis-queue:6379
+
+      # Optional Configuration
+      FORUM_CHANNEL_IDS: ${FORUM_CHANNEL_IDS:-}
+      DEBUG_MODE: ${DEBUG_MODE:-false}
+      PORT: ${PORT:-3000}
+    ports:
+      - "${PORT:-3000}:${PORT:-3000}"
+    volumes:
+      - .:/app
+      - /app/node_modules
+    command: yarn dev
+    restart: unless-stopped
+
+volumes:
+  postgres_data:
+    driver: local
+  redis_cache_data:
+    driver: local
+  redis_queue_data:
+    driver: local
+
+networks:
+  default:
+    name: unthread-network
+    driver: bridge

init.sql

@@ -0,0 +1,82 @@
+-- Initialize Unthread Discord Bot Database
+-- This script sets up the basic schema for the 3-layer storage architecture
+
+-- Create customers table for user data persistence
+CREATE TABLE IF NOT EXISTS customers (
+    id SERIAL PRIMARY KEY,
+    discord_id VARCHAR(50) UNIQUE NOT NULL,
+    unthread_customer_id VARCHAR(100),
+    email VARCHAR(255),
+    username VARCHAR(100),
+    display_name VARCHAR(100),
+    avatar_url TEXT,
+    created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
+    updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
+);
+
+-- Create thread_ticket_mappings table for Discord-Unthread relationship tracking
+CREATE TABLE IF NOT EXISTS thread_ticket_mappings (
+    id SERIAL PRIMARY KEY,
+    discord_thread_id VARCHAR(50) UNIQUE NOT NULL,
+    unthread_ticket_id VARCHAR(100) NOT NULL,
+    discord_channel_id VARCHAR(50),
+    customer_id INTEGER REFERENCES customers(id),
+    status VARCHAR(20) DEFAULT 'active',
+    created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
+    updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
+    last_sync_at TIMESTAMP WITH TIME ZONE
+);
+
+-- Create storage_cache table for L3 persistent cache
+CREATE TABLE IF NOT EXISTS storage_cache (
+    id SERIAL PRIMARY KEY,
+    cache_key VARCHAR(500) UNIQUE NOT NULL,
+    data JSONB,
+    expires_at TIMESTAMP WITH TIME ZONE,
+    created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
+    updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
+);
+
+-- Create indexes for performance
+CREATE INDEX IF NOT EXISTS idx_customers_discord_id ON customers(discord_id);
+CREATE INDEX IF NOT EXISTS idx_customers_unthread_id ON customers(unthread_customer_id);
+CREATE INDEX IF NOT EXISTS idx_thread_mappings_discord_thread ON thread_ticket_mappings(discord_thread_id);
+CREATE INDEX IF NOT EXISTS idx_thread_mappings_unthread_ticket ON thread_ticket_mappings(unthread_ticket_id);
+CREATE INDEX IF NOT EXISTS idx_thread_mappings_channel ON thread_ticket_mappings(discord_channel_id);
+CREATE INDEX IF NOT EXISTS idx_storage_cache_key ON storage_cache(cache_key);
+CREATE INDEX IF NOT EXISTS idx_storage_cache_expires ON storage_cache(expires_at);
+
+-- Create function to update updated_at timestamp
+CREATE OR REPLACE FUNCTION update_updated_at_column()
+RETURNS TRIGGER AS $$
+BEGIN
+    NEW.updated_at = NOW();
+    RETURN NEW;
+END;
+$$ language 'plpgsql';
+
+-- Create triggers to automatically update updated_at
+CREATE TRIGGER update_customers_updated_at BEFORE UPDATE ON customers
+    FOR EACH ROW EXECUTE FUNCTION update_updated_at_column();
+
+CREATE TRIGGER update_thread_mappings_updated_at BEFORE UPDATE ON thread_ticket_mappings
+    FOR EACH ROW EXECUTE FUNCTION update_updated_at_column();
+
+CREATE TRIGGER update_storage_cache_updated_at BEFORE UPDATE ON storage_cache
+    FOR EACH ROW EXECUTE FUNCTION update_updated_at_column();
+
+-- Create cleanup function for expired cache entries
+CREATE OR REPLACE FUNCTION cleanup_expired_cache()
+RETURNS INTEGER AS $$
+DECLARE
+    deleted_count INTEGER;
+BEGIN
+    DELETE FROM storage_cache WHERE expires_at IS NOT NULL AND expires_at < NOW();
+    GET DIAGNOSTICS deleted_count = ROW_COUNT;
+    RETURN deleted_count;
+END;
+$$ LANGUAGE plpgsql;
+
+-- Grant necessary permissions
+GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO postgres;
+GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO postgres;

package.json

@@ -31,16 +31,22 @@
   "packageManager": "[email protected]",
   "dependencies": {
     "@keyv/redis": "^4.4.0",
+    "@types/ioredis": "^5.0.0",
     "@wgtechlabs/log-engine": "2.2.0",
+    "bullmq": "^5.58.4",
     "cacheable": "^1.10.0",
     "discord.js": "^14.20.0",
     "dotenv": "^16.5.0",
     "express": "^4.21.2",
-    "keyv": "^5.3.4"
+    "ioredis": "^5.7.0",
+    "keyv": "^5.3.4",
+    "pg": "^8.16.3",
+    "redis": "^5.8.2"
   },
   "devDependencies": {
     "@types/express": "^4.17.21",
     "@types/node": "^18.19.0",
+    "@types/pg": "^8.15.5",
     "@typescript-eslint/eslint-plugin": "^8.41.0",
     "@typescript-eslint/parser": "^8.41.0",
     "eslint": "^9.34.0",