Start with --traefik/--traefik_local option. Closes intelowlproject#2305

(intelowlproject#2351) * add traefik config and options for dev and prod working config with traefik finish traefik config prod/dev add documentation * Update traefik_local.override.yml - remove comment * rework prod/local traefik and add deletion of get-docker.sh * split traefik compose into base, prod and local * remove print of compose files * parent c45c84a author David Mihajlovic <david.mihajlovic@protonmail.com> 1716908101 +0200 committer David Mihajlovic <david.mihajlovic@protonmail.com> 1717135119 +0200 add traefik config and options for dev and prod working config with traefik finish traefik config prod/dev add documentation Vulners#1257 (intelowlproject#2340) * vulners * vulners wrapper * docs * lesser variables * migrations * code quality * migration * code --------- Co-authored-by: g4ze <bhaiyajionline@gmail.com> bump 6.0.3 updated docs Bump django-ses from 4.0.0 to 4.1.0 in /requirements (intelowlproject#2342) Bumps [django-ses](https://github.com/django-ses/django-ses) from 4.0.0 to 4.1.0. - [Release notes](https://github.com/django-ses/django-ses/releases) - [Changelog](https://github.com/django-ses/django-ses/blob/main/CHANGES.md) - [Commits](django-ses/django-ses@v4.0.0...v4.1.0) --- updated-dependencies: - dependency-name: django-ses dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> rework prod/local traefik and add deletion of get-docker.sh split traefik compose into base, prod and local get-docker.sh deletion without sudo change traefik compose naming * remove unnecessary files * remove print of compose files * change doc --------- Co-authored-by: Ubuntu <ubuntu@intelowldev.novalocal>
standa4 · Oct 11, 2024 · ade47a6 · ade47a6
1 parent a38945e
commit ade47a6
Show file tree

Hide file tree

Showing 9 changed files with 144 additions and 47 deletions.
diff --git a/docker/default.yml b/docker/default.yml
@@ -63,8 +63,6 @@ services:
       - ../configuration/nginx/locations.conf:/etc/nginx/locations.conf
       - nginx_logs:/var/log/nginx
       - static_content:/var/www/static
-    ports:
-      - "80:80"
     depends_on:
       uwsgi:
         condition: service_healthy

diff --git a/docker/nginx.override.yml b/docker/nginx.override.yml
@@ -0,0 +1,4 @@
+services:
+  nginx:
+    ports:
+      - "80:80"
diff --git a/docker/traefik.override.yml b/docker/traefik.override.yml
diff --git a/docker/traefik.yml b/docker/traefik.yml
@@ -0,0 +1,15 @@
+services:
+  traefik:
+    image: "traefik:3.0"
+    labels:
+      - "traefik.enable=true"
+
+  nginx:
+      depends_on:
+        - traefik
+      labels:
+        - "traefik.enable=true"
+        - "traefik.http.services.nginx.loadbalancer.server.port=80"
+      expose: 
+        - "80"
+
diff --git a/docker/traefik_local.yml b/docker/traefik_local.yml
@@ -0,0 +1,28 @@
+services:
+  traefik:
+    container_name: "intelowl_traefik_local"
+    command:
+      # Pleases refer to the official documentation: https://doc.traefik.io/traefik/
+      # LOGS
+      - "--log.level=DEBUG"
+      # DASHBOARD
+      - "--api.insecure=true"
+      - "--api.dashboard=true"
+      # ENTRYPOINTS - redirect every request to use HTTPS
+      - "--entrypoints.web.address=:80"
+      # PROVIDERS
+      - "--providers.docker=true"
+      - "--providers.docker.watch=true"
+      - "--providers.docker.exposedbydefault=false"
+    ports:
+      - "80:80"
+      - "8080:8080"
+    volumes:
+      - "/var/run/docker.sock:/var/run/docker.sock:ro"
+
+  nginx:
+      depends_on:
+        - traefik
+      labels:
+        - "traefik.http.routers.nginx.rule=Host(`localhost`)"
+        - "traefik.http.routers.nginx.entrypoints=web"
diff --git a/docker/traefik_prod.yml b/docker/traefik_prod.yml
@@ -0,0 +1,66 @@
+services:
+  traefik:
+    container_name: "intelowl_traefik_prod"
+    command:
+      # Pleases refer to the official documentation: https://doc.traefik.io/traefik/
+      # LOGS - may be omitted if you don't need logs
+      - "--accesslog=true"
+      - "--accesslog.filepath=/var/log/traefik/access.log"
+      - "--log.filePath=/var/log/traefik/traefik.log"
+      - "--log.level=DEBUG"
+      # DASHBOARD
+      - "--api.dashboard=true"
+      # PROVIDERS
+      - "--providers.docker=true"
+      - "--providers.docker.watch=true"
+      - "--providers.docker.exposedbydefault=false"
+      # ENTRYPOINTS - redirect every request to use HTTPS
+      - "--entrypoints.web.address=:80"
+      - "--entryPoints.web.http.redirections.entryPoint.to=websecure"
+      - "--entryPoints.web.http.redirections.entryPoint.scheme=https"
+      - "--entryPoints.web.http.redirections.entrypoint.permanent=true"
+      - "--entrypoints.websecure.address=:443"
+      # CERTIFICATE RESOLVERS
+      - "--certificatesresolvers.le.acme.httpchallenge=true"
+      - "--certificatesresolvers.le.acme.httpchallenge.entrypoint=web"
+      # DEV - use this for testing purposes or else you might get blocked - # CHANGE THIS
+      - "--certificatesresolvers.le.acme.caserver=https://acme-staging-v02.api.letsencrypt.org/directory"
+      # PROD - use this if everything works fine - # CHANGE THIS
+      #- "--certificatesresolvers.le.acme.caserver=https://acme-v02.api.letsencrypt.org/directory"
+      - "--certificatesresolvers.le.acme.email=postmaster@example.com" # CHANGE THIS
+      - "--certificatesresolvers.le.acme.storage=/etc/letsencrypt/acme.json"  
+    labels:
+      # DASHBOARD - setup for secure dashboard access
+      - "traefik.http.routers.dashboard.rule=Host(`traefik.intelowl.example.com`) && (PathPrefix(`/api`) || PathPrefix(`/dashboard`))" # CHANGE THIS (Only "Host"!)
+      - "traefik.http.routers.dashboard.service=api@internal"
+      - "traefik.http.routers.dashboard.entrypoints=websecure"
+      - "traefik.http.routers.dashboard.tls=true" 
+      - "traefik.http.routers.dashboard.tls.certresolver=le"
+      # auth/ipallowlist middlewares allow to limit/secure access - may be omitted
+      # Here you may define which IPs/CIDR ranges are allowed to access this resource - may be omitted
+      # - "traefik.http.routers.dashboard.middlewares=dashboard-ipallowlist"
+      # - "traefik.http.middlewares.dashboard-ipallowlist.ipallowlist.sourcerange=0.0.0.0" # CHANGE THIS
+      # You can create a new user and password for basic auth with this command: 
+      # echo $(htpasswd -nbB user password) | sed -e s/\\$/\\$\\$/g 
+      # - "traefik.http.routers.dashboard.middlewares=auth"
+      # - "traefik.http.middlewares.auth.basicauth.users=user:$$2y$$05$$v.ncVNXEJriELglCBEZJmu5I1VrhyhuaVCXATRQTUVuvOF1qgYwpa" # CHANGE THIS (default is user:password)
+      - "traefik.http.services.dashboard.loadbalancer.server.port=8080"
+    ports:
+      - "80:80"
+      - "443:443"
+    volumes:
+      - "/var/run/docker.sock:/var/run/docker.sock:ro"
+      - "/etc/letsencrypt:/etc/letsencrypt"
+      - "/var/log/traefik:/var/log/traefik"
+
+  nginx:
+      depends_on:
+        - traefik
+      labels:
+        - "traefik.http.routers.nginx.rule=Host(`intelowl.example.com`)" # CHANGE THIS
+        - "traefik.http.routers.nginx.entrypoints=websecure"
+        - "traefik.http.routers.nginx.tls=true" 
+        - "traefik.http.routers.nginx.tls.certresolver=le"
+        # Here you may define which IPs/CIDR ranges are allowed to access this resource
+        # - "traefik.http.routers.nginx.middlewares=nginx-ipallowlist"
+        # - "traefik.http.middlewares.nginx-ipallowlist.ipallowlist.sourcerange=0.0.0.0" # CHANGE THIS
diff --git a/docs/source/Installation.md b/docs/source/Installation.md
@@ -185,9 +185,12 @@ There are 3 options to execute the web server:
 
     We provide a specific docker-compose file that leverages [Traefik](https://docs.traefik.io/) to allow fast deployments of public-faced and HTTPS-enabled applications.
 
-    Before using it, you should configure the configuration file `docker/traefik.override.yml` by changing the email address and the hostname where the application is served. For a detailed explanation follow the official documentation: [Traefix doc](https://docs.traefik.io/user-guides/docker-compose/acme-http/).
+    Before using the production deployment, you should change the configuration file `docker/traefik_prod.yml` by customising every line which has a "# CHANGE THIS" comment appended to reflect your environment. For a detailed explanation follow the official documentation: [Traefix doc](https://doc.traefik.io/traefik/).
+
+    The development deployment is ready to go and running on localhost. 
 
-    After the configuration is done, you can add the option `--traefik` while executing [`./start`](#run)
+    After the configuration is done, you can add the option `--traefik_prod` for a production ready deployment while executing [`./start`](#run)
+    If you just want to test things out you can add the option `--traefik_local` for a development deployment.
 
 ## Run
 

diff --git a/initialize.sh b/initialize.sh
@@ -76,6 +76,7 @@ if ! [ -x "$(command -v docker)" ]; then
       echo 'Error: Could not install docker.' >&2
       exit 1
     fi
+    rm get-docker.sh
   else
     echo 'You chose to do not install Docker. Exiting'
     exit 1

diff --git a/start b/start
@@ -9,8 +9,7 @@ declare -A env_arguments=(["prod"]=1 ["test"]=1 ["ci"]=1)
 declare -A test_mode=(["test"]=1 ["ci"]=1)
 declare -A cmd_arguments=(["build"]=1 ["up"]=1 ["start"]=1 ["restart"]=1 ["down"]=1 ["stop"]=1 ["kill"]=1 ["logs"]=1 ["ps"]=1)
 
-declare -A path_mapping=(["default"]="docker/default.yml" ["postgres"]="docker/postgres.override.yml" ["rabbitmq"]="docker/rabbitmq.override.yml" ["test"]="docker/test.override.yml" ["ci"]="docker/ci.override.yml" ["custom"]="docker/custom.override.yml" ["traefik"]="docker/traefik.override.yml" ["multi_queue"]="docker/multi-queue.override.yml" ["test_multi_queue"]="docker/test.multi-queue.override.yml" ["flower"]="docker/flower.override.yml" ["test_flower"]="docker/test.flower.override.yml" ["elastic"]="docker/elasticsearch.override.yml" ["https"]="docker/https.override.yml" ["nfs"]="docker/nfs.override.yml")
-
+declare -A path_mapping=(["default"]="docker/default.yml" ["postgres"]="docker/postgres.override.yml" ["rabbitmq"]="docker/rabbitmq.override.yml" ["test"]="docker/test.override.yml" ["ci"]="docker/ci.override.yml" ["custom"]="docker/custom.override.yml" ["traefik"]="docker/traefik.yml" ["traefik_prod"]="docker/traefik_prod.yml" ["traefik_local"]="docker/traefik_local.yml" ["multi_queue"]="docker/multi-queue.override.yml" ["test_multi_queue"]="docker/test.multi-queue.override.yml" ["flower"]="docker/flower.override.yml" ["test_flower"]="docker/test.flower.override.yml" ["elastic"]="docker/elasticsearch.override.yml" ["https"]="docker/https.override.yml" ["nfs"]="docker/nfs.override.yml" ["nginx_default"]="docker/nginx.override.yml")
 print_synopsis () {
   echo "SYNOPSIS"
   echo -e "    start <env> <command> [OPTIONS]"
@@ -38,7 +37,8 @@ print_help () {
   echo "                                file."
   echo "    --multi_queue               Uses the multiqueue.override.yml compose file."
   echo "    --nfs                       Uses the nfs.override.yml compose file."
-  echo "    --traefik                   Uses the traefik.override.yml compose file."
+  echo "    --traefik_prod              Uses the traefik.yml and traefik_prod.yml compose file."
+  echo "    --traefik_local             Uses the traefik.yml and traefik_local.yml compose file."
   echo "    --use-external-database     Do NOT use postgres.override.yml compose file."
   echo "    --use-external-redis        Do NOT use redis.override.yml compose file."
   echo "    --rabbitmq                  Uses the rabbitmq.override.yml compose file."
@@ -101,7 +101,8 @@ current_version=${REACT_APP_INTELOWL_VERSION/"v"/""}
 
 docker_analyzers=("pcap_analyzers" "tor_analyzers" "malware_tools_analyzers" "cyberchef" "phoneinfoga")
 
-declare -A path_mapping=(["default"]="docker/default.yml" ["postgres"]="docker/postgres.override.yml" ["rabbitmq"]="docker/rabbitmq.override.yml" ["test"]="docker/test.override.yml" ["ci"]="docker/ci.override.yml" ["custom"]="docker/custom.override.yml" ["traefik"]="docker/traefik.override.yml" ["multi_queue"]="docker/multi-queue.override.yml" ["test_multi_queue"]="docker/test.multi-queue.override.yml" ["flower"]="docker/flower.override.yml" ["test_flower"]="docker/test.flower.override.yml" ["elastic"]="docker/elasticsearch.override.yml" ["https"]="docker/https.override.yml" ["nfs"]="docker/nfs.override.yml" ["redis"]="docker/redis.override.yml")
+declare -A path_mapping=(["default"]="docker/default.yml" ["postgres"]="docker/postgres.override.yml" ["rabbitmq"]="docker/rabbitmq.override.yml" ["test"]="docker/test.override.yml" ["ci"]="docker/ci.override.yml" ["custom"]="docker/custom.override.yml" ["traefik"]="docker/traefik.yml" ["traefik_prod"]="docker/traefik_prod.yml" ["traefik_local"]="docker/traefik_local.yml" ["multi_queue"]="docker/multi-queue.override.yml" ["test_multi_queue"]="docker/test.multi-queue.override.yml" ["flower"]="docker/flower.override.yml" ["test_flower"]="docker/test.flower.override.yml" ["elastic"]="docker/elasticsearch.override.yml" ["https"]="docker/https.override.yml" ["nfs"]="docker/nfs.override.yml" ["redis"]="docker/redis.override.yml")
+
 for value in "${docker_analyzers[@]}"; do
   path_mapping["${value}"]+="integrations/${value}/compose.yml"
   path_mapping["${value}.test"]+="integrations/${value}/compose-tests.yml"
@@ -215,8 +216,12 @@ while [[ $# -gt 0 ]]; do
     params["https"]=true
     shift 1
   ;;
-  --traefik)
-    params["traefik"]=true
+  --traefik_prod)
+    params["traefik_prod"]=true
+    shift 1
+  ;;
+  --traefik_local)
+    params["traefik_local"]=true
     shift 1
   ;;
   -h | --help)
@@ -271,12 +276,24 @@ if [ "$is_test" = true ]; then
   compose_files+=("${path_mapping["$env_argument"]}")
 fi
 
+# Check for the traefik_prod or traefik_local argument and include traefik base compose
+if [ "${params["traefik_prod"]}" ] || [ "${params["traefik_local"]}" ]; then
+  compose_files+=("${path_mapping["traefik"]}")
+  traefik_enabled=true
+fi
+
+# Add the default nginx configuration if traefik is not used
+if [ "$traefik_enabled" = false ]; then
+  compose_files+=("${path_mapping["nginx_default"]}")
+fi
+
 # add all the other ones
 for value in "${!params[@]}"; do
   if [ "${params[$value]}" ]; then
     compose_files+=("${path_mapping["$value"]}")
   fi
 done
+
 # add all the test files
 if [[ $env_argument == "test" ]]; then
   test_values=("multi_queue" "flower")
@@ -330,8 +347,9 @@ for value in "${compose_files[@]}" ; do
     to_run+=" -f $value"
   fi
 done
+
 if grep "docker" <<< "$(groups)" > /dev/null 2>&1; then
   docker compose --project-directory docker ${to_run[@]} -p "$project_name" "$cmd_argument" "$@"
 else
   sudo docker compose --project-directory docker ${to_run[@]} -p "$project_name" "$cmd_argument" "$@"
-fi
+fi
-Original file line number
+Diff line change
@@ Expand Up / @@ -76,6 +76,7 @@ if ! [ -x "$(command -v docker)" ]; then @@
           echo 'Error: Could not install docker.' >&2
           exit 1
         fi
+        rm get-docker.sh
       else
         echo 'You chose to do not install Docker. Exiting'
         exit 1
@@ Expand Down @@