build more wheels (musllinux_1_1_{x86_64,aarch64}, manylinux_2_28_x86_64, manylinux_2_{24,28}_aarch64)

.circleci/config.yml

@@ -0,0 +1,84 @@
+version: 2.1
+
+jobs:
+  arm-wheels:
+    parameters:
+      build:
+        type: string
+      many_image:
+        type: string
+
+    machine:
+      image: ubuntu-2204:2022.07.1
+    resource_class: arm.medium  # two vCPUs
+
+    environment:
+      CIBW_ARCHS: "aarch64"
+      CIBW_MANYLINUX_AARCH64_IMAGE: "<< parameters.many_image >>"
+      CIBW_MUSLLINUX_AARCH64_IMAGE: quay.io/pypa/musllinux_1_1_aarch64
+      CIBW_BUILD: "<< parameters.build >>"
+
+    steps:
+      - checkout
+      - run:
+          name: install cibuildwheel and other build reqs
+          command: |
+            python3 -m pip install --upgrade pip setuptools setuptools_scm[toml]
+            python3 -m pip install cibuildwheel
+            python3 -m setuptools_scm
+
+      - run:
+          name: pip freeze
+          command: |
+            python3 -m pip freeze
+
+      - run:
+          name: list wheels
+          command: |
+            python3 -m cibuildwheel . --print-build-identifiers
+
+      - run:
+          name: cibuildwheel
+          command: |
+            source .github/workflows/wheel-prep.sh
+            export CIBW_ENVIRONMENT="SCHEMA_SALAD_USE_MYPYC=1 MYPYPATH=/project/mypy-stubs SETUPTOOLS_SCM_PRETEND_VERSION=${SETUPTOOLS_SCM_PRETEND_VERSION}"
+            python3 -m cibuildwheel .
+            # "/project" is the project path inside the cibuildwheel container itself
+
+      - store_artifacts:
+          path: wheelhouse/
+
+      - when:
+          condition:
+            matches:
+              pattern: ".+"
+              value: "<< pipeline.git.tag >>"
+          steps:
+            - run:
+                environment:
+                  TWINE_NONINTERACTIVE: "1"
+                  TWINE_USERNAME: __token__
+                  TWINE_PASSWORD: $TWINE_PASSWORD
+                command: |
+                  python3 -m pip install twine
+                  twine upload --skip-existing wheelhouse/*
+
+workflows:
+  wheels: # This is the name of the workflow, feel free to change it to better match your workflow.
+    # Inside the workflow, you define the jobs you want to run.
+    jobs:
+      - arm-wheels:
+          matrix:
+            parameters:
+              build:
+                - "*manylinux*"
+                - "*musllinux*"
+              many_image:
+                - quay.io/pypa/manylinux2014_aarch64
+                - quay.io/pypa/manylinux_2_24_aarch64
+                - quay.io/pypa/manylinux_2_28_aarch64
+            exclude:
+              - build: "*musllinux*"
+                many_image: quay.io/pypa/manylinux_2_24_aarch64
+              - build: "*musllinux*"
+                many_image: quay.io/pypa/manylinux_2_28_aarch64

.github/workflows/ci-tests.yml

@@ -61,47 +61,6 @@ jobs:
         uses: codecov/codecov-action@v3
         with:
           fail_ci_if_error: true
-  test-aarch64:
-    name: "Test ${{ matrix.pyver }} aarch64"
-    strategy:
-      matrix:
-        pyver: [cp39-cp39, cp310-cp310]
-        arch: [aarch64]
-      fail-fast: false
-    runs-on: ubuntu-latest
-    env:
-      py: /opt/python/${{ matrix.pyver }}/bin/python
-      img: quay.io/pypa/manylinux2014_${{ matrix.arch }}
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
-    - name: Set up QEMU
-      id: qemu
-      uses: docker/setup-qemu-action@v2
-    - name: Install tools
-      run: |
-        docker run --rm -v ${{ github.workspace }}:/ws:rw --workdir=/ws \
-          ${{ env.img }} ${{ env.py }} -m pip install -U  pip build
-    - name: Install dependencies
-      run: |
-            docker run --rm -v ${{ github.workspace }}:/ws:rw --workdir=/ws \
-            ${{ env.img }} \
-            bash -exc '${{ env.py }} -m pip install virtualenv && ${{ env.py }} -m venv .env && \
-            source .env/bin/activate && \
-            python -m pip install --upgrade pip wheel setuptools_scm[toml] && \
-            python -m pip install -r mypy-requirements.txt -r requirements.txt && \
-            MYPYPATH=mypy-stubs SCHEMA_SALAD_USE_MYPYC=1 python -m pip install -e .[pycodegen] && \
-            deactivate'
-    - name: Test with pytest
-      run: |
-          docker run --rm -v ${{ github.workspace }}:/ws:rw --workdir=/ws \
-            ${{ env.img }} \
-            bash -exc '\
-            source .env/bin/activate && \
-            pip install setuptools wheel -r test-requirements.txt && \
-            make test
-            deactivate'
 
   tox-style:
     name: CI linters via Tox

.github/workflows/wheels.yml

@@ -8,66 +8,40 @@ on:
 jobs:
   deploy:
     runs-on: ubuntu-latest
+    # strategy; can't use a matrix reference in "uses": https://github.com/orgs/community/discussions/25824#discussioncomment-3249394
     steps:
     - uses: actions/checkout@v3
       with:
         fetch-depth: 0  # slow, but gets all the tags
     - name: set version
       run: pip install setuptools_scm[toml] && python -m setuptools_scm
-    - name: Build manylinux2014 x86 Python wheels
-      uses: RalfG/python-wheels-manylinux-build@v0.4.2-manylinux2014_x86_64
+    - name: Build musllinux_1_1 x86_64 Python wheels
+      uses: RalfG/python-wheels-manylinux-build@v0.5.0-musllinux_1_1_x86_64
       with:
         pre-build-command: source .github/workflows/wheel-prep.sh
         build-requirements: -rrequirements.txt -rmypy-requirements.txt
-    - name: Build manylinux_2_24 x86 Python wheels
-      uses: RalfG/python-wheels-manylinux-build@v0.4.2-manylinux_2_24_x86_64
+    - name: Build manylinux2014 x86_64 Python wheels
+      uses: RalfG/python-wheels-manylinux-build@v0.5.0-manylinux2014_x86_64
       with:
         pre-build-command: source .github/workflows/wheel-prep.sh
         build-requirements: -rrequirements.txt -rmypy-requirements.txt
-    # - name: Build manylinux_2_28 x86 Python wheels
-    #   uses: RalfG/python-wheels-manylinux-build@v0.4.2-manylinux_2_28_x86_64
-    #   with:
-    #     pre-build-command: source .github/workflows/wheel-prep.sh
-    #     build-requirements: -rrequirements.txt -rmypy-requirements.txt
-    - name: Move audited wheels to new directory
-      run: mkdir audited_wheels && cp dist/*-manylinux*.whl audited_wheels/
-    - name: Publish wheels to PyPI
-      uses: pypa/gh-action-pypi-publish@release/v1
-      with:
-        user: __token__
-        password: ${{ secrets.PYPI_API_TOKEN }}
-        packages_dir: audited_wheels
-
-  deploy_aarch64:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v3
+    - name: Build manylinux_2_24 x86_64 Python wheels
+      uses: RalfG/python-wheels-manylinux-build@v0.5.0-manylinux_2_24_x86_64
       with:
-        fetch-depth: 0  # slow, but gets all the tags
-    - name: Set up QEMU
-      uses: docker/setup-qemu-action@v2
-    - name: set version
-      run: pip install setuptools_scm[toml] && python -m setuptools_scm
-    - name: Build manylinux2014 aarch64 Python wheels
-      uses: RalfG/python-wheels-manylinux-build@v0.4.2-manylinux2014_aarch64
+        pre-build-command: source .github/workflows/wheel-prep.sh
+        build-requirements: -rrequirements.txt -rmypy-requirements.txt
+    - name: Build manylinux_2_28 x86_64 Python wheels
+      uses: RalfG/python-wheels-manylinux-build@v0.5.0-manylinux_2_28_x86_64
       with:
         pre-build-command: source .github/workflows/wheel-prep.sh
         build-requirements: -rrequirements.txt -rmypy-requirements.txt
-    # - name: Build manylinux2_24 aarch64 Python wheels
-    #   uses: RalfG/python-wheels-manylinux-build@v0.4.2-manylinux2_24_aarch64
-    #   with:
-    #     pre-build-command: source .github/workflows/wheel-prep.sh
-    #     build-requirements: -rrequirements.txt -rmypy-requirements.txt
-    # - name: Build manylinux2_28 aarch64 Python wheels
-    #   uses: RalfG/python-wheels-manylinux-build@v0.4.2-manylinux2_28_aarch64
-    #   with:
-    #     pre-build-command: source .github/workflows/wheel-prep.sh
-    #     build-requirements: -rrequirements.txt -rmypy-requirements.txt
     - name: Move audited wheels to new directory
-      run: mkdir audited_wheels && cp dist/*-manylinux*.whl audited_wheels/
+      run: mkdir audited_wheels && cp dist/*-manylinux*.whl dist/*-musllinux*.whl audited_wheels/
     - name: Publish wheels to PyPI
+      if: ${{ github.event_name == 'release' }}
       uses: pypa/gh-action-pypi-publish@release/v1
       with:
         user: __token__
         password: ${{ secrets.PYPI_API_TOKEN }}
         packages_dir: audited_wheels
+        skip_existing: true

pyproject.toml

@@ -4,3 +4,13 @@ build-backend = "setuptools.build_meta"
 
 [tool.setuptools_scm]
 write_to = "schema_salad/_version.py"
+
+[tool.cibuildwheel]
+before-build = "python -m pip install -r requirements.txt -r mypy-requirements.txt"
+test-command = "python -m pytest -n 2 --pyargs schema_salad"
+test-requires = "-r test-requirements.txt black"
+skip = "pp* "
+# ^ skip building wheels on pypy
+test-skip = "cp*"
+# ^ tests skipped to save time
+#build-verbosity = "1"

setup.py

@@ -20,7 +20,22 @@
 if os.getenv("SCHEMA_SALAD_USE_MYPYC", None) == "1":
     USE_MYPYC = True
 
-if USE_MYPYC:
+if USE_MYPYC and any(
+    item in sys.argv
+    for item in [
+        "build",
+        "bdist_wheel",
+        "build_ext",
+        "install",
+        "install-lib",
+        "bdist",
+        "bdist_dumb",
+        "bdist_rpm",
+        "develop",
+        "bdist_egg",
+        "editable_wheel",
+    ]
+):
     mypyc_targets = [
         # "schema_salad/codegen_base.py",  # interpreted classes cannot inherit from compiled
         # "schema_salad/codegen.py",
@@ -59,7 +74,9 @@
     from mypyc.build import mypycify
 
     opt_level = os.getenv("MYPYC_OPT_LEVEL", "3")
-    ext_modules = mypycify(mypyc_targets, opt_level=opt_level, verbose=True)
+    ext_modules = mypycify(
+        mypyc_targets, opt_level=opt_level, debug_level="0", verbose=True
+    )
 else:
     ext_modules = []
 
@@ -69,6 +86,7 @@
     "ruamel.yaml >= 0.17.6, < 0.17.22;python_version>='3.7'",
     "ruamel.yaml >= 0.16.12, < 0.17.22",
     "rdflib >= 4.2.2, < 7.0.0",
+    "rdflib-jsonld>=0.4.0, <= 0.6.1;python_version<='3.6'",
     "mistune >= 0.8.1, < 0.9",
     "CacheControl >= 0.11.7, < 0.13",
     "lockfile >= 0.9",  # needed for CacheControl's FileCache
@@ -99,11 +117,29 @@
     download_url="https://github.com/common-workflow-language/schema_salad/releases",
     ext_modules=ext_modules,
     license="Apache 2.0",
-    python_requires=">=3.6",
+    python_requires=">=3.6,<3.11",
     setup_requires=pytest_runner + ["setuptools_scm"],
-    packages=["schema_salad", "schema_salad.tests"],
-    package_data={"schema_salad": ["metaschema/*", "py.typed"]},
-    include_package_data=True,
+    packages=["schema_salad", "schema_salad.tests", "schema_salad.avro"],
+    package_data={
+        "schema_salad": [
+            "metaschema/*",
+            "py.typed",
+            "dotnet/*",
+            "java/*",
+            "typescript/*",
+            "typescript/.*",
+        ],
+        "schema_salad.tests": [
+            "*.yml",
+            "docimp/*",
+            "*.owl",
+            "*.cwl",
+            "*.txt",
+            "foreign/*.cwl",
+            "test_real_cwl/*",
+            "test_schema/*",
+        ],
+    },
     install_requires=install_requires,
     extras_require=extras_require,
     test_suite="tests",