heads 追加

melpon · Jan 6, 2022 · 2a1f025 · 2a1f025
1 parent 3f96ac3
commit 2a1f025
Show file tree

Hide file tree

Showing 5 changed files with 240 additions and 22 deletions.
diff --git a/.github/workflows/heads.yml b/.github/workflows/heads.yml
@@ -4,6 +4,10 @@ on:
     # 毎日 09:00 JST
     - cron: "0 0 * * *"
 
+env:
+  UPLOAD_URL: "https://uploads.github.com/repos/melpon/wandbox-builder/releases/22561082/assets{?name,label}"
+  RELEASE_ID: 22561082
+
 jobs:
   heads:
     strategy:
@@ -12,35 +16,51 @@ jobs:
         name:
           - gcc-head
           - clang-head
-          - swift-head
-          - nodejs-head
-          - gdc-head
-          - openjdk-head
-    runs-on: ubuntu-16.04
+    runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v1
-      - run: sudo ./ga-install.sh
-        working-directory: build/${{ matrix.name }}
+      - run: sudo ./install.sh
+        working-directory: ga-build-heads/${{ matrix.name }}
       - run: tar czf ${{ matrix.name }}.tar.gz ${{ matrix.name }}/
         working-directory: /opt/wandbox
-      - name: Release ${{ matrix.name }} latest
+      - name: Release ${{ matrix.name }}
         uses: WebFreak001/deploy-nightly@v1.0.3
         env:
-          GITHUB_TOKEN: ${{ secrets.PERSONAL_GITHUB_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
-          upload_url: "https://uploads.github.com/repos/melpon/wandbox-builder/releases/22561082/assets{?name,label}"
-          release_id: 22561082
+          upload_url: ${{ env.UPLOAD_URL }}
+          release_id: ${{ env.RELEASE_ID }}
           asset_path: /opt/wandbox/${{ matrix.name }}.tar.gz
           asset_name: ${{ matrix.name }}.tar.gz
           asset_content_type: application/gzip
-      - name: Release ${{ matrix.name }} nightly
-        uses: WebFreak001/deploy-nightly@v1.0.3
+  _deploy:
+    needs: [heads]
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+      # SSH 秘密鍵の登録
+      - name: Register SSH key
         env:
-          GITHUB_TOKEN: ${{ secrets.PERSONAL_GITHUB_TOKEN }}
-        with:
-          upload_url: "https://uploads.github.com/repos/melpon/wandbox-builder/releases/22561082/assets{?name,label}"
-          release_id: 22561082
-          asset_path: /opt/wandbox/${{ matrix.name }}.tar.gz
-          asset_name: ${{ matrix.name }}-$$.tar.gz
-          asset_content_type: application/gzip
-          max_releases: 7
+          SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
+        run: |
+          mkdir -p $HOME/.ssh
+          echo "$SSH_PRIVATE_KEY" > $HOME/.ssh/id_ed25519
+          chmod 600 $HOME/.ssh/id_ed25519
+      # デプロイ先を設定する
+      - name: Add SSH config
+        env:
+          SSH_HOST: ${{ secrets.SSH_HOST }}
+          SSH_USER: ${{ secrets.SSH_USER }}
+        run: |
+          echo "Host wandbox" >> $HOME/.ssh/config
+          echo "    HostName $SSH_HOST" >> $HOME/.ssh/config
+          echo "    User $SSH_USER" >> $HOME/.ssh/config
+          echo "    ServerAliveInterval 60" >> $HOME/.ssh/config
+          echo "    TCPKeepAlive yes" >> $HOME/.ssh/config
+          ssh-keyscan -H $SSH_HOST >> $HOME/.ssh/known_hosts
+      # ソースをデプロイ先にコピーして、デプロイスクリプトを実行する
+      - name: Deploy
+        run: |
+          ssh wandbox 'mkdir -p /opt/wandbox-data'
+          rsync -a -v . wandbox:/opt/wandbox-data/wandbox-builder
+          ssh wandbox 'python3 /opt/wandbox-data/wandbox-builder/deploy_heads.py  ${{ env.ASSETS_URL }} --github_token ${{ env.GITHUB_TOKEN }}'
diff --git a/deploy.py b/deploy.py
@@ -84,7 +84,7 @@ def download(url: str, output_dir: Optional[str] = None, filename: Optional[str]
         return output_path
 
     retry = 5
-    while retry > 0:
+    while True:
         try:
             if shutil.which('curl') is not None:
                 cmd(["curl", "-fLo", output_path, url] + args)

diff --git a/deploy_heads.py b/deploy_heads.py
@@ -0,0 +1,85 @@
+import os
+import yaml
+import tempfile
+import argparse
+import get_asset_info
+from typing import Optional, List
+
+from deploy import download, mkdir_p, extract, rm_rf
+
+
+# デプロイしたいコンパイラ
+def get_heads(filename) -> List[str]:
+    with open(filename) as f:
+        data = yaml.load(f, Loader=yaml.Loader)
+    r = []
+    for name in data['jobs']['heads']['strategy']['matrix']['name']:
+        r.append(name)
+    return r
+
+
+def find_download_url(asset_info, name):
+    name = f"{name}.tar.gz"
+    for obj in asset_info:
+        if obj['name'] == name:
+            return name, obj['url']
+    raise Exception(f'{name} not in asset info')
+
+
+# デプロイする
+def deploy(head: str, version_dir: str, deploy_dir: str, download_url: str, archive_name: str, github_token: Optional[str] = None):
+    with tempfile.TemporaryDirectory() as tempdir:
+        # コンパイラをダウンロード
+        header_args = ["-H", "Accept: application/octet-stream"]
+        if github_token is not None:
+            header_args += ["-H", f"Authorization: token {github_token}"]
+        # なんかよく切断されるのでリトライを設定する
+        header_args += ["--retry", "5"]
+
+        archive_path = download(download_url, tempdir, archive_name, header_args)
+
+        # テンポラリディレクトリ上に解凍
+        mkdir_p(version_dir)
+        extract_path = extract(archive_path, tempdir)
+
+        # バージョンファイルと本体を消して、解凍した新しいバージョンを配置する
+        version_path = os.path.join(version_dir, head)
+        if os.path.exists(version_path):
+            with open(version_path, 'rb') as f:
+                rm_rf(f.read().decode('utf-8'))
+        rm_rf(version_path)
+
+        mkdir_p(deploy_dir)
+        deploy_path = os.path.join(deploy_dir, head)
+        os.rename(extract_path, deploy_path)
+
+        # デプロイ完了したのでバージョン情報とパスを書き込む
+        with open(version_path, 'wb') as f:
+            f.write(deploy_path.encode('utf-8'))
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("url")
+    parser.add_argument("--github_token", default=None)
+
+    args = parser.parse_args()
+
+    # 各コンパイラのダウンロード URL を取得する
+    asset_info = get_asset_info.get_asset_info(args.url, args.github_token)
+
+    version_dir = '/opt/wandbox-data/wandbox-deploy-heads'
+    deploy_dir = '/opt/wandbox'
+
+    heads = get_heads('.github/workflows/heads.yml')
+    for head in heads:
+        name, download_url = find_download_url(asset_info, head)
+        deploy(head, version_dir, deploy_dir, download_url, name, args.github_token)
+
+
+BASE_DIR = os.path.abspath(os.path.dirname(__file__))
+
+
+if __name__ == '__main__':
+    os.chdir(BASE_DIR)
+    main()
diff --git a/ga-build-heads/clang-head/ga-install.sh b/ga-build-heads/clang-head/ga-install.sh
@@ -0,0 +1,66 @@
+#!/bin/bash
+
+set -ex
+
+PREFIX=/opt/wandbox/clang-head
+
+apt-get update
+apt-get install -y \
+  build-essential \
+  clang-12 \
+  coreutils \
+  git \
+  libgmp-dev \
+  libmpc-dev \
+  libmpfr-dev \
+  libtool \
+  python3 \
+  wget
+
+CMAKE_VERSION="3.16.3"
+CMAKE_SHA256="3e15dadfec8d54eda39c2f266fc1e571c1b88bf32f9d221c8a039b07234206fa"
+
+CMAKE_PREFIX="/usr/local/wandbox/camke-${CMAKE_VERSION}"
+
+wget https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz
+echo "${CMAKE_SHA256} *cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz" | sha256sum -c
+tar xf cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz
+mkdir -p /usr/local/wandbox/
+mv cmake-${CMAKE_VERSION}-Linux-x86_64 $CMAKE_PREFIX
+
+mkdir -p ~/tmp/clang-head/
+cd ~/tmp/clang-head/
+
+# get sources
+
+git clone --depth 1 https://github.com/llvm/llvm-project.git
+
+# build
+
+mkdir build
+
+# case %{_host_cpu} in
+#   i?86* | amd64* | x86_64*) arch="X86" ;;
+#   sparc*)               arch="Sparc" ;;
+#   powerpc*)             arch="PowerPC" ;;
+#   arm*)                 arch="ARM" ;;
+#   aarch64*)             arch="AArch64" ;;
+#   mips* | mips64*)      arch="Mips" ;;
+#   xcore*)               arch="XCore" ;;
+#   msp430*)              arch="MSP430" ;;
+#   hexagon*)             arch="Hexagon" ;;
+#   nvptx*)               arch="NVPTX" ;;
+#   s390x*)               arch="SystemZ" ;;
+#   *)                    arch="Unknown" ;;
+# esac
+cd build
+
+export CC="clang-12"
+export CXX="clang++-12"
+/usr/local/wandbox/camke-3.16.3/bin/cmake -G "Unix Makefiles" \
+  -DCMAKE_BUILD_TYPE=Release \
+  -DCMAKE_INSTALL_PREFIX=$PREFIX \
+  -DLLVM_ENABLE_PROJECTS="clang;compiler-rt;clang-tools-extra;libcxx;libcxxabi" \
+  ../llvm-project/llvm
+make -j`nproc`
+make install
diff --git a/ga-build-heads/gcc-head/ga-install.sh b/ga-build-heads/gcc-head/ga-install.sh
@@ -0,0 +1,47 @@
+#!/bin/bash
+
+set -ex
+
+PREFIX=/opt/wandbox/gcc-head
+
+apt-get update
+apt-get install -y \
+  bison \
+  build-essential \
+  coreutils \
+  curl \
+  flex \
+  git \
+  libgmp-dev \
+  libmpc-dev \
+  libmpfr-dev \
+  libtool \
+  m4 \
+  unzip \
+  wget
+
+# get sources
+
+mkdir ~/gcc-head
+cd ~/gcc-head
+
+git clone --depth 1 https://github.com/gcc-mirror/gcc.git source
+
+# build
+
+mkdir build
+cd build
+../source/configure \
+  --prefix=$PREFIX \
+  --enable-languages=c,c++ \
+  --disable-multilib \
+  --without-ppl \
+  --without-cloog-ppl \
+  --enable-checking=release \
+  --disable-nls \
+  --enable-lto \
+  LDFLAGS="-Wl,-rpath,$PREFIX/lib,-rpath,$PREFIX/lib64,-rpath,$PREFIX/lib32"
+
+make -j`nproc`
+rm -rf $PREFIX
+make install