Merge branch 'main' into 17054-custom-name

Author: mr-c

.github/workflows/quay-publish.yml

@@ -0,0 +1,36 @@
+name: publish-quay
+on:
+  push:
+    branches:
+      - main
+    tags:
+      - '*'
+  workflow_dispatch: {}
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@master
+      - name: Get image tags
+        id: image_tags
+        run: |
+          echo -n ::set-output name=IMAGE_TAGS::$(git describe --tags)
+      - name: Build and publish cwltool_module image to Quay
+        uses: docker/build-push-action@v1
+        with:
+          file: cwltool.Dockerfile
+          registry: ${{ secrets.REGISTRY_SERVER }}
+          repository: ${{ secrets.REGISTRY_NAMESPACE }}/cwltool_module
+          username: ${{ secrets.REGISTRY_USERNAME }}
+          password: ${{ secrets.REGISTRY_PASSWORD }}
+          tags: "${{ steps.image_tags.outputs.IMAGE_TAGS }}"
+          target: module
+      - name: Build and publish cwltool image to Quay
+        uses: docker/build-push-action@v1
+        with:
+          file: cwltool.Dockerfile
+          registry: ${{ secrets.REGISTRY_SERVER }}
+          repository: ${{ secrets.REGISTRY_NAMESPACE }}/cwltool
+          username: ${{ secrets.REGISTRY_USERNAME }}
+          password: ${{ secrets.REGISTRY_PASSWORD }}
+          tags: "${{ steps.image_tags.outputs.IMAGE_TAGS }}"

README.rst

@@ -10,6 +10,8 @@ Conda: |Conda Version| |Conda Installs|
 
 Debian: |Debian Testing package| |Debian Stable package|
 
+Quay.io (Docker): |Quay.io Container|
+
 .. |Linux Status| image:: https://github.com/common-workflow-language/cwltool/actions/workflows/ci-tests.yml/badge.svg?branch=main
    :target: https://github.com/common-workflow-language/cwltool/actions/workflows/ci-tests.yml
 
@@ -37,6 +39,9 @@ Debian: |Debian Testing package| |Debian Stable package|
 .. |Conda Installs| image:: https://anaconda.org/conda-forge/cwltool/badges/downloads.svg
    :target: https://anaconda.org/conda-forge/cwltool
 
+.. |Quay.io Container| image:: https://quay.io/repository/commonwl/cwltool/status
+   :target: https://quay.io/repository/commonwl/cwltool
+
 This is the reference implementation of the Common Workflow Language.  It is
 intended to be feature complete and provide comprehensive validation of CWL
 files as well as provide other tools related to working with CWL.

build-cwl-docker.sh

@@ -1,10 +1,10 @@
 #!/bin/bash
-set -e
-docker build --file=cwltool.Dockerfile --tag=commonworkflowlanguage/cwltool_module --target module .
-docker build --file=cwltool.Dockerfile --tag=commonworkflowlanguage/cwltool .
+set -ex
+docker build --file=cwltool.Dockerfile --tag=quay.io/commonwl/cwltool_module --target module .
+docker build --file=cwltool.Dockerfile --tag=quay.io/commonwl/cwltool .
 
 version=$(git describe --tags)
 if echo "$version" | grep -vq '\-' >& /dev/null ; then
-    docker tag commonworkflowlanguage/cwltool_module commonworkflowlanguage/cwltool_module:"$version"
-    docker tag commonworkflowlanguage/cwltool commonworkflowlanguage/cwltool:"$version"
+    docker tag quay.io/commonwl/cwltool_module quay.io/commonwl/cwltool_module:"$version"
+    docker tag quay.io/commonwl/cwltool quay.io/commonwl/cwltool:"$version"
 fi

cwltool.Dockerfile

@@ -1,6 +1,14 @@
-FROM python:3.8-alpine as builder
-
-RUN apk add --no-cache git gcc python3-dev libxml2-dev libxslt-dev libc-dev linux-headers
+FROM python:3.9-slim as builder
+
+RUN apt-get update && apt-get install -y \
+	gcc \
+	git \
+	libc-dev \
+	libxml2-dev \
+	libxslt1-dev \
+	python3-dev \
+  && rm -rf /var/lib/apt/lists/*
+#	linux-headers \
 
 WORKDIR /cwltool
 COPY . .
@@ -9,16 +17,28 @@ RUN python setup.py bdist_wheel --dist-dir=/wheels
 RUN pip wheel -r requirements.txt --wheel-dir=/wheels
 RUN pip install --no-index --no-warn-script-location --root=/pythonroot/ /wheels/*.whl
 
-FROM python:3.8-alpine as module
-LABEL maintainer peter.amstutz@curoverse.com
+FROM python:3.9-slim as module
+LABEL maintainer peter.amstutz@curii.com
 
-RUN apk add --no-cache docker nodejs graphviz libxml2 libxslt
+RUN apt-get update && apt-get install -y \
+	docker.io \
+	graphviz \
+	libxml2 \
+	libxslt1.1 \
+	nodejs \
+  && rm -rf /var/lib/apt/lists/*
 COPY --from=builder /pythonroot/ /
 
-FROM python:3.8-alpine
-LABEL maintainer peter.amstutz@curoverse.com
+FROM python:3.9-slim
+LABEL maintainer peter.amstutz@curii.com
 
-RUN apk add --no-cache docker nodejs graphviz libxml2 libxslt
+RUN apt-get update && apt-get install -y \
+	docker.io \
+	graphviz \
+	libxml2 \
+	libxslt1.1 \
+	nodejs \
+  && rm -rf /var/lib/apt/lists/*
 COPY --from=builder /pythonroot/ /
 COPY cwltool-in-docker.sh /cwltool-in-docker.sh
 

cwltool/process.py

@@ -309,7 +309,7 @@ def stage_files(
                 os.makedirs(entry.target)
             else:
                 shutil.copytree(entry.resolved, entry.target)
-                ensure_writable(entry.target)
+                ensure_writable(entry.target, include_root=True)
         elif entry.type == "CreateFile" or entry.type == "CreateWritableFile":
             with open(entry.target, "wb") as new:
                 if secret_store is not None:

cwltool/utils.py

@@ -367,23 +367,30 @@ def downloadHttpFile(httpurl):
     return str(f.name)
 
 
-def ensure_writable(path):  # type: (str) -> None
+def ensure_writable(path: str, include_root: bool = False) -> None:
+    """
+    Ensure that 'path' is writable.
+
+    If 'path' is a directory, then all files and directories under 'path' are
+    made writable, recursively. If 'path' is a file or if 'include_root' is
+    `True`, then 'path' itself is made writable.
+    """
+
+    def add_writable_flag(p: str) -> None:
+        st = os.stat(p)
+        mode = stat.S_IMODE(st.st_mode)
+        os.chmod(p, mode | stat.S_IWUSR)
+
     if os.path.isdir(path):
+        if include_root:
+            add_writable_flag(path)
         for root, dirs, files in os.walk(path):
             for name in files:
-                j = os.path.join(root, name)
-                st = os.stat(j)
-                mode = stat.S_IMODE(st.st_mode)
-                os.chmod(j, mode | stat.S_IWUSR)
+                add_writable_flag(os.path.join(root, name))
             for name in dirs:
-                j = os.path.join(root, name)
-                st = os.stat(j)
-                mode = stat.S_IMODE(st.st_mode)
-                os.chmod(j, mode | stat.S_IWUSR)
+                add_writable_flag(os.path.join(root, name))
     else:
-        st = os.stat(path)
-        mode = stat.S_IMODE(st.st_mode)
-        os.chmod(path, mode | stat.S_IWUSR)
+        add_writable_flag(path)
 
 
 def ensure_non_writable(path):  # type: (str) -> None

tests/test_iwdr.py

@@ -1,4 +1,5 @@
 from pathlib import Path
+from stat import S_IWRITE, S_IWGRP, S_IWOTH
 from typing import Any
 
 from cwltool.factory import Factory
@@ -80,6 +81,53 @@ def test_iwdr_permutations(tmp_path_factory: Any) -> None:
     )
 
 
+def test_iwdr_permutations_readonly(tmp_path_factory: Any) -> None:
+    """Confirm that readonly input files are properly made writable."""
+    misc = tmp_path_factory.mktemp("misc")
+    fifth = tmp_path_factory.mktemp("fifth")
+    fifth_file = fifth / "bar"
+    fifth_dir = fifth / "foo"
+    fifth_file.touch()
+    fifth_dir.mkdir()
+    sixth = tmp_path_factory.mktemp("sixth")
+    first = misc / "first"
+    first.touch()
+    second = misc / "second"
+    second.touch()
+    outdir = str(tmp_path_factory.mktemp("outdir"))
+    for entry in [first, second, fifth, sixth, fifth_file, fifth_dir]:
+        mode = entry.stat().st_mode
+        ro_mask = 0o777 ^ (S_IWRITE | S_IWGRP | S_IWOTH)
+        entry.chmod(mode & ro_mask)
+    assert (
+        main(
+            [
+                "--no-container",
+                "--debug",
+                "--leave-outputs",
+                "--outdir",
+                outdir,
+                get_data("tests/wf/iwdr_permutations_nocontainer.cwl"),
+                "--first",
+                str(first),
+                "--second",
+                str(second),
+                "--fifth",
+                str(fifth),
+                "--sixth",
+                str(sixth),
+            ]
+        )
+        == 0
+    )
+    for entry in [first, second, fifth, sixth, fifth_file, fifth_dir]:
+        try:
+            mode = entry.stat().st_mode
+            entry.chmod(mode | S_IWRITE)
+        except PermissionError:
+            pass
+
+
 @needs_docker
 def test_iwdr_permutations_inplace(tmp_path_factory: Any) -> None:
     misc = tmp_path_factory.mktemp("misc")

tests/wf/iwdr_permutations_nocontainer.cwl

@@ -0,0 +1,43 @@
+#!/usr/bin/env cwl-runner
+class: CommandLineTool
+cwlVersion: v1.2
+requirements:
+  InitialWorkDirRequirement:
+    listing:
+      - entry: $(inputs.first)
+        entryname: first_writable_file
+        writable: true
+      - entry: $(inputs.second)
+        entryname: second_read_only_file
+        writable: false
+      - entry: $(inputs.fifth)
+        entryname: fifth_writable_directory
+        writable: true
+      - entry: $(inputs.sixth)
+        entryname: sixth_read_only_directory
+        writable: false
+      - entry: $(inputs.ninth)
+        entryname: nineth_writable_directory_literal
+        writable: true
+inputs:
+  first: File
+  second: File
+  fifth: Directory
+  sixth: Directory
+  ninth:
+    type: Directory
+    default:
+      class: Directory
+      basename: foo
+      listing: []
+outputs:
+  out:
+    type: Directory
+    outputBinding:
+      glob: .
+baseCommand: [bash, -c]
+arguments:
+  - |
+    find .
+    echo "a" > first_writable_file
+    touch fifth_writable_directory/c