Skip to content

Commit

Permalink
Blaufuss/prune os lists (#18)
Browse files Browse the repository at this point in the history 
* clean up some very old stuff

* Remove unused Dockerfiles from unsupported base images

* Cleaned up base image generation:

Only ubuntu 22.04 and rocky 9
modern photospline
optional push to docker (for local testing)
docs update

* Simplify the icetray install business.

Less test-data, tables, etc.  People usually bind-mount /cvmfs anyway if
needed

New install path for modern icetray-install and entry point for
container

* Update NS containers:  latest releases.  Use 22.04 devel base

* Remove 20.04 stuff, experimental stuff, bump pytorch version

* don't forget default

* Use /opt/icetray for all things icetray

* Use latest icetray, copy from new location

* Update to cude 12.1, and new icetray

* Add some extra needs for the NB

* Remove unused tags

* Fix prod - devel issue, restrict ml builds for a bit.

See what's timing out
  • Loading branch information
@blaufuss
blaufuss authored Jan 13, 2025
1 parent 7a461d4 commit fd3bb29
Show file tree
Hide file tree
Showing 39 changed files with 158 additions and 1,977 deletions.
175 changes: 38 additions & 137 deletions .github/workflows/build_base.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,12 @@ name: build_base_images

env:
PLATFORMS_DEFAULT: linux/amd64
CUDA_VERSION_DEFAULT: 11.8.0
CUDA_VERSION_DEFAULT: 12.1.0
CUDNN_VERSION_DEFAULT: 8
ROCM_VERSION_DEFAULT: 5.7
#ROCM_VERSION_DEFAULT: 5.7
PYTORCH_VERSION_DEFAULT: 2.2.2
TENSORFLOW_VERSION_DEFAULT: 2.17.0
PUSH: true


on:
Expand All @@ -21,23 +24,23 @@ on:
- linux/arm64/v8
cuda_version:
description: 'CUDA Version to use'
default: '11.8.0'
default: '12.1.0'
required: true
cudnn_version:
description: 'cuDNN Version to use'
default: '8'
required: true
rocm_version:
description: 'rocm Version to use'
default: '5.7'
required: true
# rocm_version:
# description: 'rocm Version to use'
# default: '5.7'
# required: true
pytorch_version:
description: 'pytorch Version to use'
default: '2.2.2'
required: true
tensorflow_version:
description: 'tensorflow Version to use'
default: '2.17'
default: '2.17.0'
required: true
push:
branches: [ master ]
Expand Down Expand Up @@ -66,10 +69,10 @@ jobs:
id: cudnn
run: echo "CUDNN_VERSION=${{ github.event.inputs.cudnn_version || env.CUDNN_VERSION_DEFAULT }}" >> $GITHUB_ENV
-
name: set rocm version
id: rocm
run: echo "ROCM_VERSION=${{ github.event.inputs.rocm_version || env.ROCM_VERSION_DEFAULT }}" >> $GITHUB_ENV
-
# name: set rocm version
# id: rocm
# run: echo "ROCM_VERSION=${{ github.event.inputs.rocm_version || env.ROCM_VERSION_DEFAULT }}" >> $GITHUB_ENV
# -
name: set pytorch version
id: pytorch
run: echo "PYTORCH_VERSION=${{ github.event.inputs.pytorch_version || env.PYTORCH_VERSION_DEFAULT }}" >> $GITHUB_ENV
Expand Down Expand Up @@ -98,7 +101,7 @@ jobs:
-
name: More Free Space
run: |
sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /usr/share/dotnet "$AGENT_TOOLSDIRECTORY"
sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /usr/share/dotnet
-
name: Set up QEMU
uses: docker/setup-qemu-action@v2
Expand All @@ -111,51 +114,23 @@ jobs:
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
-
name: Build and push base devel 20 04
uses: docker/build-push-action@v3
with:
push: true
file: icetray_base/Dockerfile_base-devel-20.04
platforms: ${{ env.PLATFORMS }}
tags: >
icecube/icetray-base:devel-ubuntu20.04,
icecube/icetray-base:devel-ubuntu20.04-${{ env.current_date }}
-
name: Build and push base prod 20 04
uses: docker/build-push-action@v3
with:
push: true
file: icetray_base/Dockerfile_base-prod-20.04
platforms: ${{ env.PLATFORMS }}
tags: >
icecube/icetray-base:prod-ubuntu20.04,
icecube/icetray-base:prod-ubuntu20.04-${{ env.current_date }}
-
name: Build and push base devel 22 04
uses: docker/build-push-action@v3
with:
push: true
push: ${{env.PUSH}}
file: icetray_base/Dockerfile_base-devel-22.04
platforms: ${{ env.PLATFORMS }}
build-args: |
BASE_IMAGE=ubuntu:22.04
tags: >
icecube/icetray-base:devel-ubuntu22.04,
icecube/icetray-base:devel-ubuntu22.04-${{ env.current_date }}
-
name: Build and push base devel 24 04
uses: docker/build-push-action@v3
with:
push: true
file: icetray_base/Dockerfile_base-devel-24.04
platforms: ${{ env.PLATFORMS }}
tags: >
icecube/icetray-base:devel-ubuntu24.04,
icecube/icetray-base:devel-ubuntu24.04-${{ env.current_date }}
-
name: Build and push base devel 22 04 CUDA
uses: docker/build-push-action@v3
with:
push: true
push: ${{env.PUSH}}
file: icetray_base/Dockerfile_base-devel-22.04
platforms: ${{ env.PLATFORMS }}
build-args: |
Expand All @@ -167,7 +142,7 @@ jobs:
name: Build and push base devel 22 04 pytorch
uses: docker/build-push-action@v3
with:
push: true
push: ${{env.PUSH}}
file: icetray_base/Dockerfile_base-devel-22.04
platforms: ${{ env.PLATFORMS }}
build-args: |
Expand All @@ -179,7 +154,7 @@ jobs:
name: Build and push base devel 22 04 tensorflow
uses: docker/build-push-action@v3
with:
push: true
push: ${{env.PUSH}}
file: icetray_base/Dockerfile_base-devel-22.04
platforms: ${{ env.PLATFORMS }}
build-args: |
Expand All @@ -188,112 +163,38 @@ jobs:
icecube/icetray-base:devel-ubuntu22.04-tensorflow${{env.TENSORFLOW_VERSION}},
icecube/icetray-base:devel-ubuntu22.04-tensorflow${{env.TENSORFLOW_VERSION}}-${{ env.current_date }}
-
name: Build and push base devel 24 04 CUDA
uses: docker/build-push-action@v3
with:
push: true
file: icetray_base/Dockerfile_base-devel-24.04
platforms: ${{ env.PLATFORMS }}
build-args: |
BASE_IMAGE=nvidia/cuda:12.6.1-cudnn-devel-ubuntu24.04
tags: >
icecube/icetray-base:devel-ubuntu24.04-cuda12.6.1,
icecube/icetray-base:devel-ubuntu24.04-cuda12.6.1-${{ env.current_date }}
-
name: Build and push base devel 22 04 ROCm
uses: docker/build-push-action@v3
with:
push: true
file: icetray_base/Dockerfile_base-devel-22.04
platforms: ${{ env.PLATFORMS }}
build-args: |
BASE_IMAGE=rocm/dev-ubuntu-22.04:${{ env.ROCM_VERSION }}-complete
tags: >
icecube/icetray-base:devel-ubuntu22.04-rocm${{ env.ROCM_VERSION }},
icecube/icetray-base:devel-ubuntu22.04-rocm${{ env.ROCM_VERSION }}-${{ env.current_date }}
# -
# name: Build and push base devel 24 04 ROCm
# uses: docker/build-push-action@v3
# with:
# push: true
# file: icetray_base/Dockerfile_base-devel-24.04
# platforms: ${{ env.PLATFORMS }}
# build-args: |
# BASE_IMAGE=rocm/dev-ubuntu-24.04:${{ env.ROCM_VERSION }}-complete
# tags: >
# icecube/icetray-base:devel-ubuntu24.04-rocm${{ env.ROCM_VERSION }},
# icecube/icetray-base:devel-ubuntu24.04-rocm${{ env.ROCM_VERSION }}-${{ env.current_date }}
-
name: Build and push base prod 22 04
uses: docker/build-push-action@v3
with:
push: true
file: icetray_base/Dockerfile_base-prod-22.04
platforms: ${{ env.PLATFORMS }}
tags: >
icecube/icetray-base:prod-ubuntu22.04,
icecube/icetray-base:prod-ubuntu22.04-${{ env.current_date }}
-
name: Build and push base prod 24 04
uses: docker/build-push-action@v3
with:
push: true
file: icetray_base/Dockerfile_base-prod-24.04
platforms: ${{ env.PLATFORMS }}
tags: >
icecube/icetray-base:prod-ubuntu24.04,
icecube/icetray-base:prod-ubuntu24.04-${{ env.current_date }}
-
name: Build and push base devel rocky 8
uses: docker/build-push-action@v3
with:
push: true
file: icetray_base/Dockerfile_base-devel-rocky8
platforms: ${{ env.PLATFORMS }}
tags: >
icecube/icetray-base:devel-rocky8,
icecube/icetray-base:devel-rocky8-${{ env.current_date }}
-
name: Build and push base devel rocky8 CUDA
uses: docker/build-push-action@v3
with:
push: true
file: icetray_base/Dockerfile_base-devel-rocky8
platforms: ${{ env.PLATFORMS }}
build-args: |
BASE_IMAGE=nvidia/cuda:${{env.CUDA_VERSION}}-cudnn${{env.CUDNN_VERSION}}-devel-rockylinux8
tags: >
icecube/icetray-base:devel-rocky8-cuda${{env.CUDA_VERSION}}-cudnn${{env.CUDNN_VERSION}},
icecube/icetray-base:devel-rocky8-cuda${{env.CUDA_VERSION}}-cudnn${{env.CUDNN_VERSION}}-${{ env.current_date }}
-
# name: Build and push base devel 22 04 ROCm
# uses: docker/build-push-action@v3
# with:
# push: ${{env.PUSH}}
# file: icetray_base/Dockerfile_base-devel-22.04
# platforms: ${{ env.PLATFORMS }}
# build-args: |
# BASE_IMAGE=rocm/dev-ubuntu-22.04:${{ env.ROCM_VERSION }}-complete
# tags: >
# icecube/icetray-base:devel-ubuntu22.04-rocm${{ env.ROCM_VERSION }},
# icecube/icetray-base:devel-ubuntu22.04-rocm${{ env.ROCM_VERSION }}-${{ env.current_date }}
# -
name: Build and push base devel rocky 9
uses: docker/build-push-action@v3
with:
push: true
push: ${{env.PUSH}}
file: icetray_base/Dockerfile_base-devel-rocky9
platforms: ${{ env.PLATFORMS }}
build-args: |
BASE_IMAGE=rockylinux:9
tags: >
icecube/icetray-base:devel-rocky9,
icecube/icetray-base:devel-rocky9-${{ env.current_date }}
-
name: Build and push base devel rocky9 CUDA
uses: docker/build-push-action@v3
with:
push: true
push: ${{env.PUSH}}
file: icetray_base/Dockerfile_base-devel-rocky9
platforms: ${{ env.PLATFORMS }}
build-args: |
BASE_IMAGE=nvidia/cuda:12.2.0-devel-rockylinux9
tags: >
icecube/icetray-base:devel-rocky9-cuda${{env.CUDA_VERSION}}-cudnn${{env.CUDNN_VERSION}},
icecube/icetray-base:devel-rocky9-cuda${{env.CUDA_VERSION}}-cudnn${{env.CUDNN_VERSION}}-${{ env.current_date }}
# -
# name: Build and push base devel debian sid
# uses: docker/build-push-action@v3
# with:
# push: true
# file: icetray_base/Dockerfile_base-devel-debian-sid
# platforms: linux/amd64,linux/arm64/v8
# tags: >
# icecube/icetray-base:devel-debian-sid,
# icecube/icetray-base:devel-debian-sid-${{ env.current_date }}
Loading

0 comments on commit fd3bb29

Please # to comment.