From 2a1f025686c06c7b214735aac50ad633e7e40073 Mon Sep 17 00:00:00 2001 From: melpon Date: Fri, 7 Jan 2022 08:51:33 +0900 Subject: [PATCH] =?UTF-8?q?heads=20=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/heads.yml | 62 ++++++++++++------ deploy.py | 2 +- deploy_heads.py | 85 +++++++++++++++++++++++++ ga-build-heads/clang-head/ga-install.sh | 66 +++++++++++++++++++ ga-build-heads/gcc-head/ga-install.sh | 47 ++++++++++++++ 5 files changed, 240 insertions(+), 22 deletions(-) create mode 100644 deploy_heads.py create mode 100755 ga-build-heads/clang-head/ga-install.sh create mode 100755 ga-build-heads/gcc-head/ga-install.sh diff --git a/.github/workflows/heads.yml b/.github/workflows/heads.yml index 20419994..22219a97 100644 --- a/.github/workflows/heads.yml +++ b/.github/workflows/heads.yml @@ -4,6 +4,10 @@ on: # 毎日 09:00 JST - cron: "0 0 * * *" +env: + UPLOAD_URL: "https://uploads.github.com/repos/melpon/wandbox-builder/releases/22561082/assets{?name,label}" + RELEASE_ID: 22561082 + jobs: heads: strategy: @@ -12,35 +16,51 @@ jobs: name: - gcc-head - clang-head - - swift-head - - nodejs-head - - gdc-head - - openjdk-head - runs-on: ubuntu-16.04 + runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v1 - - run: sudo ./ga-install.sh - working-directory: build/${{ matrix.name }} + - run: sudo ./install.sh + working-directory: ga-build-heads/${{ matrix.name }} - run: tar czf ${{ matrix.name }}.tar.gz ${{ matrix.name }}/ working-directory: /opt/wandbox - - name: Release ${{ matrix.name }} latest + - name: Release ${{ matrix.name }} uses: WebFreak001/deploy-nightly@v1.0.3 env: - GITHUB_TOKEN: ${{ secrets.PERSONAL_GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - upload_url: "https://uploads.github.com/repos/melpon/wandbox-builder/releases/22561082/assets{?name,label}" - release_id: 22561082 + upload_url: ${{ env.UPLOAD_URL }} + release_id: ${{ env.RELEASE_ID }} asset_path: /opt/wandbox/${{ matrix.name }}.tar.gz asset_name: ${{ matrix.name }}.tar.gz asset_content_type: application/gzip - - name: Release ${{ matrix.name }} nightly - uses: WebFreak001/deploy-nightly@v1.0.3 + _deploy: + needs: [heads] + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + # SSH 秘密鍵の登録 + - name: Register SSH key env: - GITHUB_TOKEN: ${{ secrets.PERSONAL_GITHUB_TOKEN }} - with: - upload_url: "https://uploads.github.com/repos/melpon/wandbox-builder/releases/22561082/assets{?name,label}" - release_id: 22561082 - asset_path: /opt/wandbox/${{ matrix.name }}.tar.gz - asset_name: ${{ matrix.name }}-$$.tar.gz - asset_content_type: application/gzip - max_releases: 7 + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} + run: | + mkdir -p $HOME/.ssh + echo "$SSH_PRIVATE_KEY" > $HOME/.ssh/id_ed25519 + chmod 600 $HOME/.ssh/id_ed25519 + # デプロイ先を設定する + - name: Add SSH config + env: + SSH_HOST: ${{ secrets.SSH_HOST }} + SSH_USER: ${{ secrets.SSH_USER }} + run: | + echo "Host wandbox" >> $HOME/.ssh/config + echo " HostName $SSH_HOST" >> $HOME/.ssh/config + echo " User $SSH_USER" >> $HOME/.ssh/config + echo " ServerAliveInterval 60" >> $HOME/.ssh/config + echo " TCPKeepAlive yes" >> $HOME/.ssh/config + ssh-keyscan -H $SSH_HOST >> $HOME/.ssh/known_hosts + # ソースをデプロイ先にコピーして、デプロイスクリプトを実行する + - name: Deploy + run: | + ssh wandbox 'mkdir -p /opt/wandbox-data' + rsync -a -v . wandbox:/opt/wandbox-data/wandbox-builder + ssh wandbox 'python3 /opt/wandbox-data/wandbox-builder/deploy_heads.py ${{ env.ASSETS_URL }} --github_token ${{ env.GITHUB_TOKEN }}' diff --git a/deploy.py b/deploy.py index a237c41b..34146959 100644 --- a/deploy.py +++ b/deploy.py @@ -84,7 +84,7 @@ def download(url: str, output_dir: Optional[str] = None, filename: Optional[str] return output_path retry = 5 - while retry > 0: + while True: try: if shutil.which('curl') is not None: cmd(["curl", "-fLo", output_path, url] + args) diff --git a/deploy_heads.py b/deploy_heads.py new file mode 100644 index 00000000..976c92b2 --- /dev/null +++ b/deploy_heads.py @@ -0,0 +1,85 @@ +import os +import yaml +import tempfile +import argparse +import get_asset_info +from typing import Optional, List + +from deploy import download, mkdir_p, extract, rm_rf + + +# デプロイしたいコンパイラ +def get_heads(filename) -> List[str]: + with open(filename) as f: + data = yaml.load(f, Loader=yaml.Loader) + r = [] + for name in data['jobs']['heads']['strategy']['matrix']['name']: + r.append(name) + return r + + +def find_download_url(asset_info, name): + name = f"{name}.tar.gz" + for obj in asset_info: + if obj['name'] == name: + return name, obj['url'] + raise Exception(f'{name} not in asset info') + + +# デプロイする +def deploy(head: str, version_dir: str, deploy_dir: str, download_url: str, archive_name: str, github_token: Optional[str] = None): + with tempfile.TemporaryDirectory() as tempdir: + # コンパイラをダウンロード + header_args = ["-H", "Accept: application/octet-stream"] + if github_token is not None: + header_args += ["-H", f"Authorization: token {github_token}"] + # なんかよく切断されるのでリトライを設定する + header_args += ["--retry", "5"] + + archive_path = download(download_url, tempdir, archive_name, header_args) + + # テンポラリディレクトリ上に解凍 + mkdir_p(version_dir) + extract_path = extract(archive_path, tempdir) + + # バージョンファイルと本体を消して、解凍した新しいバージョンを配置する + version_path = os.path.join(version_dir, head) + if os.path.exists(version_path): + with open(version_path, 'rb') as f: + rm_rf(f.read().decode('utf-8')) + rm_rf(version_path) + + mkdir_p(deploy_dir) + deploy_path = os.path.join(deploy_dir, head) + os.rename(extract_path, deploy_path) + + # デプロイ完了したのでバージョン情報とパスを書き込む + with open(version_path, 'wb') as f: + f.write(deploy_path.encode('utf-8')) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("url") + parser.add_argument("--github_token", default=None) + + args = parser.parse_args() + + # 各コンパイラのダウンロード URL を取得する + asset_info = get_asset_info.get_asset_info(args.url, args.github_token) + + version_dir = '/opt/wandbox-data/wandbox-deploy-heads' + deploy_dir = '/opt/wandbox' + + heads = get_heads('.github/workflows/heads.yml') + for head in heads: + name, download_url = find_download_url(asset_info, head) + deploy(head, version_dir, deploy_dir, download_url, name, args.github_token) + + +BASE_DIR = os.path.abspath(os.path.dirname(__file__)) + + +if __name__ == '__main__': + os.chdir(BASE_DIR) + main() \ No newline at end of file diff --git a/ga-build-heads/clang-head/ga-install.sh b/ga-build-heads/clang-head/ga-install.sh new file mode 100755 index 00000000..2e1cf035 --- /dev/null +++ b/ga-build-heads/clang-head/ga-install.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +set -ex + +PREFIX=/opt/wandbox/clang-head + +apt-get update +apt-get install -y \ + build-essential \ + clang-12 \ + coreutils \ + git \ + libgmp-dev \ + libmpc-dev \ + libmpfr-dev \ + libtool \ + python3 \ + wget + +CMAKE_VERSION="3.16.3" +CMAKE_SHA256="3e15dadfec8d54eda39c2f266fc1e571c1b88bf32f9d221c8a039b07234206fa" + +CMAKE_PREFIX="/usr/local/wandbox/camke-${CMAKE_VERSION}" + +wget https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz +echo "${CMAKE_SHA256} *cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz" | sha256sum -c +tar xf cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz +mkdir -p /usr/local/wandbox/ +mv cmake-${CMAKE_VERSION}-Linux-x86_64 $CMAKE_PREFIX + +mkdir -p ~/tmp/clang-head/ +cd ~/tmp/clang-head/ + +# get sources + +git clone --depth 1 https://github.com/llvm/llvm-project.git + +# build + +mkdir build + +# case %{_host_cpu} in +# i?86* | amd64* | x86_64*) arch="X86" ;; +# sparc*) arch="Sparc" ;; +# powerpc*) arch="PowerPC" ;; +# arm*) arch="ARM" ;; +# aarch64*) arch="AArch64" ;; +# mips* | mips64*) arch="Mips" ;; +# xcore*) arch="XCore" ;; +# msp430*) arch="MSP430" ;; +# hexagon*) arch="Hexagon" ;; +# nvptx*) arch="NVPTX" ;; +# s390x*) arch="SystemZ" ;; +# *) arch="Unknown" ;; +# esac +cd build + +export CC="clang-12" +export CXX="clang++-12" +/usr/local/wandbox/camke-3.16.3/bin/cmake -G "Unix Makefiles" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=$PREFIX \ + -DLLVM_ENABLE_PROJECTS="clang;compiler-rt;clang-tools-extra;libcxx;libcxxabi" \ + ../llvm-project/llvm +make -j`nproc` +make install \ No newline at end of file diff --git a/ga-build-heads/gcc-head/ga-install.sh b/ga-build-heads/gcc-head/ga-install.sh new file mode 100755 index 00000000..63cec3f4 --- /dev/null +++ b/ga-build-heads/gcc-head/ga-install.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +set -ex + +PREFIX=/opt/wandbox/gcc-head + +apt-get update +apt-get install -y \ + bison \ + build-essential \ + coreutils \ + curl \ + flex \ + git \ + libgmp-dev \ + libmpc-dev \ + libmpfr-dev \ + libtool \ + m4 \ + unzip \ + wget + +# get sources + +mkdir ~/gcc-head +cd ~/gcc-head + +git clone --depth 1 https://github.com/gcc-mirror/gcc.git source + +# build + +mkdir build +cd build +../source/configure \ + --prefix=$PREFIX \ + --enable-languages=c,c++ \ + --disable-multilib \ + --without-ppl \ + --without-cloog-ppl \ + --enable-checking=release \ + --disable-nls \ + --enable-lto \ + LDFLAGS="-Wl,-rpath,$PREFIX/lib,-rpath,$PREFIX/lib64,-rpath,$PREFIX/lib32" + +make -j`nproc` +rm -rf $PREFIX +make install