diff --git a/.env.example b/.env.example index ce8220e01..c2dce63e7 100644 --- a/.env.example +++ b/.env.example @@ -1,54 +1,69 @@ -AGENTS_API_KEY=myauthkey -AGENTS_API_KEY_HEADER_NAME=Authorization -AGENTS_API_URL=http://agents-api:8080 -COZO_AUTH_TOKEN=myauthkey -COZO_HOST=http://memory-store:9070 -COZO_PORT=9070 -COZO_ROCKSDB_DIR=cozo.db -DTYPE=float16 -EMBEDDING_SERVICE_BASE=http://text-embeddings-inference -EMBEDDING_SERVICE_URL=${EMBEDDING_SERVICE_BASE}/embed -GATEWAY_PORT=80 -GPU_MEMORY_UTILIZATION=0.90 - -HF_TOKEN= -HUGGING_FACE_HUB_TOKEN= -JWT_SHARED_KEY= - -MAX_MODEL_LEN=8192 -MAX_NUM_SEQS=1 -MNT_DIR=/data - +# Security +# -------- +JWT_SHARED_KEY= +AGENTS_API_KEY= +COZO_AUTH_TOKEN= +TEMPORAL_POSTGRES_PASSWORD= +LITELLM_POSTGRES_PASSWORD= +LITELLM_MASTER_KEY= +LITELLM_REDIS_PASSWORD= SKIP_CHECK_DEVELOPER_HEADERS=true -SUMMARIZATION_TOKENS_THRESHOLD=2048 -TEMPERATURE_SCALING_FACTOR=0.9 -TEMPERATURE_SCALING_POWER=0.9 -TEMPORAL_ENDPOINT=temporal:7233 -TEMPORAL_NAMESPACE=default -TEMPORAL_WORKER_URL=temporal:7233 -TP_SIZE=1 -TRUNCATE_EMBED_TEXT=true -TRAEFIK_LOG_LEVEL=DEBUG -WORKER_URL=temporal:7233 - -AGENTS_API_DEBUG=false -OPENAI_API_KEY= -ANTHROPIC_API_KEY= -GROQ_API_KEY= -CLOUDFLARE_API_KEY= -CLOUDFLARE_ACCOUNT_ID= -NVIDIA_NIM_API_KEY= -GITHUB_API_KEY= -VOYAGE_API_KEY= -GOOGLE_APPLICATION_CREDENTIALS= - -LITELLM_URL=http://litellm:4000 -POSTGRES_DB=litellm -POSTGRES_USER=llmproxy -POSTGRES_PASSWORD= -LITELLM_DATABASE_URL=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@litellm-db:5432/${POSTGRES_DB} -LITELLM_MASTER_KEY= -LITELLM_REDIS_HOST=litellm-redis -LITELLM_REDIS_PORT=6379 -LITELLM_REDIS_PASSWORD= -REDIS_ARGS="--requirepass ${LITELLM_REDIS_PASSWORD}" \ No newline at end of file +EMBEDDING_SERVICE_BASE=http://text-embeddings-inference- # Use the 'gpu' profile to run on GPU + +# Memory Store +# ----------- + +# COZO_HOST=http://memory-store:9070 +# COZO_PORT=9070 +# COZO_ROCKSDB_DIR=cozo.db +# COZO_BACKUP_DIR=backup +# COZO_MNT_DIR=/data + +# Gateway +# ------ + +# GATEWAY_PORT=80 +# TRAEFIK_LOG_LEVEL=INFO + +# Agents API +# --------- + +# AGENTS_API_KEY_HEADER_NAME=Authorization +# AGENTS_API_URL=http://agents-api:8080 +# TRUNCATE_EMBED_TEXT=true +# WORKER_URL=temporal:7233 +# AGENTS_API_DEBUG=false +# EMBEDDING_MODEL_ID=Alibaba-NLP/gte-large-en-v1.5 +# NUM_GPUS=1 + +# Temporal +# -------- + +# TEMPORAL_ENDPOINT=temporal:7233 +# TEMPORAL_NAMESPACE=default +# TEMPORAL_WORKER_URL=temporal:7233 +# TEMPORAL_POSTGRES_DB=temporal +# TEMPORAL_POSTGRES_USER=temporal + +# LiteLLM +# ------- + +# LITELLM_URL=http://litellm:4000 +# LITELLM_POSTGRES_DB=litellm +# LITELLM_POSTGRES_USER=llmproxy +# LITELLM_REDIS_HOST=litellm-redis +# LITELLM_REDIS_PORT=6379 + +# LLM Providers +# -------------- + +# OPENAI_API_KEY= +# HUGGING_FACE_HUB_TOKEN= +# ANTHROPIC_API_KEY= +# GROQ_API_KEY= +# CLOUDFLARE_API_KEY= +# CLOUDFLARE_ACCOUNT_ID= +# NVIDIA_NIM_API_KEY= +# GITHUB_API_KEY= +# VOYAGE_API_KEY= +# GOOGLE_APPLICATION_CREDENTIALS=.keys/julep-vertexai-svc.json \ No newline at end of file diff --git a/agents-api/docker-compose.yml b/agents-api/docker-compose.yml index 0e8cca2bd..c8a84e669 100644 --- a/agents-api/docker-compose.yml +++ b/agents-api/docker-compose.yml @@ -1,16 +1,49 @@ name: julep-agents-api +# Base for embedding service +x--text-embeddings-inference: &text-embeddings-inference + container_name: text-embeddings-inference + environment: + - MODEL_ID=${EMBEDDING_MODEL_ID:-Alibaba-NLP/gte-large-en-v1.5} + + image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.5 + ports: + - "8082:80" + volumes: + - ~/.cache/huggingface/hub:/data + +# Shared environment variables +x-shared-environment: &shared-environment + AGENTS_API_KEY: ${AGENTS_API_KEY} + LITELLM_MASTER_KEY: ${LITELLM_MASTER_KEY} + COZO_AUTH_TOKEN: ${COZO_AUTH_TOKEN} + SKIP_CHECK_DEVELOPER_HEADERS: ${SKIP_CHECK_DEVELOPER_HEADERS:-True} + AGENTS_API_KEY_HEADER_NAME: ${AGENTS_API_KEY_HEADER_NAME:-Authorization} + AGENTS_API_URL: ${AGENTS_API_URL:-http://agents-api:8080} + TRUNCATE_EMBED_TEXT: ${TRUNCATE_EMBED_TEXT:-False} + WORKER_URL: ${WORKER_URL:-temporal:7233} + DEBUG: ${AGENTS_API_DEBUG:-False} + EMBEDDING_SERVICE_BASE: ${EMBEDDING_SERVICE_BASE:-http://text-embeddings-inference} + EMBEDDING_MODEL_ID: ${EMBEDDING_MODEL_ID:-Alibaba-NLP/gte-large-en-v1.5} + LITELLM_URL: ${LITELLM_URL:-http://litellm:4000} + COZO_HOST: ${COZO_HOST:-http://memory-store:9070} + SUMMARIZATION_MODEL_NAME: ${SUMMARIZATION_MODEL_NAME:-gpt-4-turbo} + TEMPORAL_WORKER_URL: ${TEMPORAL_WORKER_URL:-temporal:7233} + TEMPORAL_NAMESPACE: ${TEMPORAL_NAMESPACE:-default} + TEMPORAL_ENDPOINT: ${TEMPORAL_ENDPOINT:-temporal:7233} + TEMPORAL_TASK_QUEUE: ${TEMPORAL_TASK_QUEUE:-julep-task-queue} + services: agents-api: image: julepai/agents-api:${TAG:-dev} - env_file: - - ../.env container_name: agents-api depends_on: memory-store: condition: service_started worker: condition: service_started + environment: + <<: *shared-environment build: context: . dockerfile: Dockerfile @@ -31,15 +64,12 @@ services: worker: image: julepai/worker:${TAG:-dev} - env_file: - - ../.env - + environment: + <<: *shared-environment build: context: . dockerfile: Dockerfile.worker depends_on: - text-embeddings-inference: - condition: service_started temporal: condition: service_started @@ -55,32 +85,32 @@ services: - action: rebuild path: Dockerfile.worker - text-embeddings-inference: - container_name: text-embeddings-inference + text-embeddings-inference-cpu: + <<: *text-embeddings-inference + profiles: + - '' # Acts as a default profile. See: https://stackoverflow.com/questions/75758174/how-to-make-profile-default-for-docker-compose + + text-embeddings-inference-gpu: + <<: *text-embeddings-inference + profiles: + - gpu + image: ghcr.io/huggingface/text-embeddings-inference:1.5 environment: - DTYPE=float16 - - MODEL_ID=Alibaba-NLP/gte-large-en-v1.5 + - MODEL_ID=${EMBEDDING_MODEL_ID:-Alibaba-NLP/gte-large-en-v1.5} + - NVIDIA_VISIBLE_DEVICES=all - image: ghcr.io/huggingface/text-embeddings-inference:1.5 - ports: - - "8082:80" - volumes: - - ~/.cache/huggingface/hub:/data shm_size: "2gb" deploy: resources: reservations: devices: - driver: nvidia - count: all + count: ${NUM_GPUS:-1} capabilities: [gpu] - cozo-migrate: image: julepai/cozo-migrate:${TAG:-dev} - env_file: - - ../.env - container_name: cozo-migrate depends_on: memory-store: @@ -89,6 +119,9 @@ services: context: . dockerfile: Dockerfile.migration restart: "no" # Make sure to double quote this + environment: + - COZO_HOST=${COZO_HOST:-http://cozo:9070} + - COZO_AUTH_TOKEN=${COZO_AUTH_TOKEN:-myauthkey} develop: watch: diff --git a/deploy/.env.example b/deploy/.env.example deleted file mode 100644 index df26d975e..000000000 --- a/deploy/.env.example +++ /dev/null @@ -1,13 +0,0 @@ -AGENTS_API_KEY=myauthkey -AGENTS_API_KEY_HEADER_NAME=Authorization -AGENTS_API_URL=http://agents-api:8080 -COZO_AUTH_TOKEN=myauthkey -COZO_HOST=http://memory-store:9070 -EMBEDDING_SERVICE_URL=http://text-embeddings-inference/embed -SKIP_CHECK_DEVELOPER_HEADERS=true -TEMPORAL_ENDPOINT=temporal:7233 -TEMPORAL_NAMESPACE=default -TEMPORAL_WORKER_URL=temporal:7233 -TRUNCATE_EMBED_TEXT=true -WORKER_URL=temporal:7233 -OPENAI_API_KEY=your_openai_api_key \ No newline at end of file diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml deleted file mode 100644 index 9da92dfa9..000000000 --- a/deploy/docker-compose.yml +++ /dev/null @@ -1,106 +0,0 @@ -name: julep-api - -services: - memory-store: - image: julepai/memory-store:${TAG:-dev} - platform: linux/x86_64 - environment: - - COZO_AUTH_TOKEN=${COZO_AUTH_TOKEN} - - COZO_PORT=9070 - - MNT_DIR=/data - container_name: julep-memory-store - volumes: - - cozo_data:/data - env_file: - - .env - ports: - - "9070:9070" - - agents-api: - image: julepai/agents-api:${TAG:-dev} - platform: linux/x86_64 - container_name: julep-agents-api - depends_on: - memory-store: - condition: service_started - worker: - condition: service_started - ports: - - "8080:8080" - env_file: - - .env - - worker: - image: julepai/worker:${TAG:-dev} - platform: linux/x86_64 - container_name: julep-worker - depends_on: - text-embeddings-inference: - condition: service_started - temporal: - condition: service_started - env_file: - - .env - - text-embeddings-inference: - container_name: julep-embeddings - environment: - - DTYPE=float16 - - MODEL_ID=BAAI/llm-embedder - - image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.0 - platform: linux/x86_64 - ports: - - "8082:80" - shm_size: "2gb" - volumes: - - ~/.cache/huggingface/hub:/data - env_file: - - .env - - docs-text-embeddings-inference: - container_name: docs-text-embeddings-inference - environment: - - DTYPE=float16 - - MODEL_ID=BAAI/bge-m3 - image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.0 - platform: linux/x86_64 - ports: - - "8083:80" - volumes: - - ~/.cache/huggingface/hub:/data - shm_size: "2gb" - env_file: - - .env - - temporal: - image: julepai/temporal:${TAG:-dev} - platform: linux/x86_64 - container_name: julep-temporal - ports: - - 7233:7233 - build: - context: . - dockerfile: Dockerfile.temporal - volumes: - - temporal_data:/home/temporal - env_file: - - .env - - cozo-migrate: - environment: - - COZO_AUTH_TOKEN=${COZO_AUTH_TOKEN} - - COZO_HOST=http://memory-store:9070 - env_file: - - .env - image: julepai/cozo-migrate:${TAG:-dev} - platform: linux/x86_64 - container_name: julep-cozo-migrate - depends_on: - memory-store: - condition: service_started - restart: "no" # Make sure to double quote this - -volumes: - temporal_data: - cozo_data: diff --git a/gateway/docker-compose.yml b/gateway/docker-compose.yml index a901f0559..ead857958 100644 --- a/gateway/docker-compose.yml +++ b/gateway/docker-compose.yml @@ -3,7 +3,6 @@ name: julep-gateway services: gateway: image: julepai/gateway:${TAG:-dev} - env_file: "../.env" environment: - GATEWAY_PORT=80 - JWT_SHARED_KEY=${JWT_SHARED_KEY} @@ -14,7 +13,6 @@ services: - AGENTS_API_KEY=${AGENTS_API_KEY} - AGENTS_API_KEY_HEADER_NAME=${AGENTS_API_KEY_HEADER_NAME} - TRAEFIK_LOG_LEVEL=${TRAEFIK_LOG_LEVEL} - container_name: gateway depends_on: agents-api: diff --git a/llm-proxy/docker-compose.yml b/llm-proxy/docker-compose.yml index ad0f84ef2..5b7f9d116 100644 --- a/llm-proxy/docker-compose.yml +++ b/llm-proxy/docker-compose.yml @@ -8,8 +8,24 @@ services: - .keys:/app/.keys ports: - "4000:4000" - env_file: - - ../.env + environment: + - LITELLM_MASTER_KEY=${LITELLM_MASTER_KEY} + - LITELLM_POSTGRES_DB=${LITELLM_POSTGRES_DB} + - LITELLM_POSTGRES_USER=${LITELLM_POSTGRES_USER} + - LITELLM_POSTGRES_PASSWORD=${LITELLM_POSTGRES_PASSWORD} + - LITELLM_REDIS_HOST=${LITELLM_REDIS_HOST} + - LITELLM_REDIS_PORT=${LITELLM_REDIS_PORT} + - LITELLM_REDIS_PASSWORD=${LITELLM_REDIS_PASSWORD} + - LITELLM_DATABASE_URL=postgresql://${LITELLM_POSTGRES_USER}:${LITELLM_POSTGRES_PASSWORD}@litellm-db:5432/${LITELLM_POSTGRES_DB} + - OPENAI_API_KEY=${OPENAI_API_KEY} + - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY} + - GROQ_API_KEY=${GROQ_API_KEY} + - CLOUDFLARE_API_KEY=${CLOUDFLARE_API_KEY} + - CLOUDFLARE_ACCOUNT_ID=${CLOUDFLARE_ACCOUNT_ID} + - NVIDIA_NIM_API_KEY=${NVIDIA_NIM_API_KEY} + - GITHUB_API_KEY=${GITHUB_API_KEY} + - VOYAGE_API_KEY=${VOYAGE_API_KEY} + - GOOGLE_APPLICATION_CREDENTIALS=${GOOGLE_APPLICATION_CREDENTIALS} command: [ "--config", @@ -21,7 +37,6 @@ services: "--telemetry", "False" ] - depends_on: - litellm-db - litellm-redis @@ -33,10 +48,6 @@ services: - litellm-db-data:/var/lib/postgresql/data ports: - "15432:5432" - - # FIXME: This combination of env_file and environment is not working - env_file: - - ../.env environment: - POSTGRES_DB=${LITELLM_POSTGRES_DB} - POSTGRES_USER=${LITELLM_POSTGRES_USER} @@ -50,12 +61,12 @@ services: litellm-redis: image: redis/redis-stack-server restart: always + environment: + REDIS_ARGS: --requirepass ${LITELLM_REDIS_PASSWORD} volumes: - litellm-redis-data:/data ports: - "16379:6379" - env_file: - - ../.env volumes: litellm-db-data: diff --git a/memory-store/Dockerfile b/memory-store/Dockerfile index e131116f8..9f77b660f 100644 --- a/memory-store/Dockerfile +++ b/memory-store/Dockerfile @@ -32,12 +32,14 @@ FROM debian:bookworm-slim # Install dependencies RUN \ apt-get update -yqq && \ - apt-get install -y liburing-dev curl ca-certificates tini nfs-common nfs-kernel-server procps netbase && \ + apt-get install -y \ + ca-certificates tini nfs-common nfs-kernel-server procps netbase \ + liburing-dev curl && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* # Set fallback mount directory -ENV MNT_DIR=/data APP_HOME=/app BACKUP_DIR=/backup +ENV COZO_MNT_DIR=/data COZO_BACKUP_DIR=/backup APP_HOME=/app WORKDIR $APP_HOME # Copy the cozo binary @@ -49,7 +51,7 @@ COPY ./backup.sh ./backup.sh # Ensure the script is executable RUN \ - mkdir -p $MNT_DIR $BACKUP_DIR && \ + mkdir -p $COZO_MNT_DIR $COZO_BACKUP_DIR && \ chmod +x $APP_HOME/bin/cozo && \ chmod +x $APP_HOME/run.sh diff --git a/memory-store/docker-compose.yml b/memory-store/docker-compose.yml index 9487d8b95..2c224c0f5 100644 --- a/memory-store/docker-compose.yml +++ b/memory-store/docker-compose.yml @@ -3,14 +3,14 @@ name: julep-memory-store services: memory-store: image: julepai/memory-store:${TAG:-dev} - env_file: "../.env" environment: - COZO_AUTH_TOKEN=${COZO_AUTH_TOKEN} - - COZO_PORT=${COZO_PORT} - - MNT_DIR=/data - - COZO_BACKUP_DIR=/backup + - COZO_PORT=${COZO_PORT:-9070} + - COZO_MNT_DIR=${MNT_DIR:-/data} + - COZO_BACKUP_DIR=${COZO_BACKUP_DIR:-/backup} volumes: - cozo_data:/data + - cozo_backup:/backup build: context: . ports: @@ -31,4 +31,5 @@ services: - /var/run/docker.sock:/var/run/docker.sock:ro volumes: - cozo_data: \ No newline at end of file + cozo_data: + cozo_backup: \ No newline at end of file diff --git a/memory-store/run.sh b/memory-store/run.sh index 852350cda..838280452 100755 --- a/memory-store/run.sh +++ b/memory-store/run.sh @@ -3,17 +3,17 @@ set -eo pipefail # Create mount directory for service. -mkdir -p ${MNT_DIR:=/data} +mkdir -p ${COZO_MNT_DIR:=/data} # Create auth token if not exists. export COZO_AUTH_TOKEN=${COZO_AUTH_TOKEN:=`tr -dc A-Za-z0-9 $MNT_DIR/${COZO_ROCKSDB_DIR}.rocksdb.cozo_auth +echo $COZO_AUTH_TOKEN > $COZO_MNT_DIR/${COZO_ROCKSDB_DIR}.rocksdb.cozo_auth # Start server ${APP_HOME:=.}/bin/cozo server \ --engine rocksdb \ - --path $MNT_DIR/${COZO_ROCKSDB_DIR} \ + --path $COZO_MNT_DIR/${COZO_ROCKSDB_DIR} \ --bind 0.0.0.0 \ --port ${COZO_PORT:=9070} diff --git a/monitoring/docker-compose.yml b/monitoring/docker-compose.yml index 78e048450..d13bd3bb6 100644 --- a/monitoring/docker-compose.yml +++ b/monitoring/docker-compose.yml @@ -4,16 +4,9 @@ services: portainer: image: portainer/portainer-ce:latest container_name: portainer - env_file: - - path: .env - required: false - - path: ../.env - required: false - ports: - "9000:8000" - "9443:9443" - volumes: - portainer_data:/data - /var/run/docker.sock:/var/run/docker.sock @@ -21,11 +14,6 @@ services: prometheus: image: prom/prometheus container_name: prometheus - env_file: - - path: .env - required: false - - path: ../.env - required: false command: - "--config.file=/etc/prometheus/prometheus.yml" ports: @@ -38,11 +26,6 @@ services: grafana: image: grafana/grafana container_name: grafana - env_file: - - path: .env - required: false - - path: ../.env - required: false ports: - 3000:3000 restart: unless-stopped diff --git a/scheduler/docker-compose.yml b/scheduler/docker-compose.yml index b371bcb0a..9d0dd5856 100644 --- a/scheduler/docker-compose.yml +++ b/scheduler/docker-compose.yml @@ -4,8 +4,6 @@ services: temporal: depends_on: - temporal-db - env_file: - - ../.env environment: - DB=postgres12 - DB_PORT=5432 @@ -32,8 +30,6 @@ services: temporal-db: image: postgres:16 - env_file: - - ../.env environment: - POSTGRES_DB=${TEMPORAL_POSTGRES_DB} - POSTGRES_USER=${TEMPORAL_POSTGRES_USER} diff --git a/sweep.yaml b/sweep.yaml deleted file mode 100644 index c19a99779..000000000 --- a/sweep.yaml +++ /dev/null @@ -1,8 +0,0 @@ -branch: dev -blocked_dirs: - - agents-api/notebooks/ - - docs/python-sdk-docs/julep/api/ - - docs/js-sdk-docs/ - - sdks/python/julep/api/ - - sdks/ts/src/api/ - - sdks/postman/