feat: Open-source initial commit

.github/CODEOWNERS

@@ -0,0 +1 @@
+* @pall-j

.github/workflows/dynamic_workflow.yaml

@@ -3,17 +3,20 @@ name: Dynamic Pipeline
 on: [push]
 
 env:
-  PYTHON_VERSION: 3.10.12
   CHANGELOG_FILE: CHANGELOG.md
   FULL_CHANGELOG_FILE: FULL_CHANGELOG.md
-  CONVCO_VERSION: v0.5.0
+  CONVCO_VERSION: v0.6.1
   POETRY_VERSION: 1.8.2
-  PACKAGE_NAME: "dtml-dbx-pyspark-testing"
+  PACKAGE_NAME: "pysparkdt"
+  PACKAGE_PATH: "pysparkdt"
 
 jobs:
 
   tests:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
     steps:
       - uses: actions/checkout@v4
         with:
@@ -22,7 +25,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: ${{ env.PYTHON_VERSION }}
+          python-version: ${{ matrix.python-version }}
 
       - name: Bootstrap poetry
         run: |
@@ -39,12 +42,12 @@ jobs:
         id: venv-cache
         with:
           path: .venv/
-          key: poetry-${{ hashFiles('poetry.lock') }}
+          key: poetry-${{ matrix.python-version }}-${{ hashFiles('poetry.lock') }}
 
       - name: Install dependencies
         if: steps.venv-cache.outputs.cache-hit != 'true'
         run: |
-          poetry env use ${{ env.PYTHON_VERSION }}
+          poetry env use ${{ matrix.python-version }}
           poetry install
 
       - name: Run checks
@@ -54,13 +57,8 @@ jobs:
           poetry run pytest .
 
   publish:
-    runs-on: ubuntu-22.04  # convco needs GLIBC_2.32 which is not in 20.04
-    # Do not use core-tools-python-runtime because:
-    # - it doesn't have preinstalled GH CLI and GH CLI installation takes
-    #   longer than installation of poetry
-    # - there are issues with using convco in python-runtime:
-    #   GitHub writes: Error: Could not open the git repository.
-    # - image pull of python-runtime is slower than installation of poetry
+    runs-on: ubuntu-22.04
+    needs: tests
     if: github.ref == 'refs/heads/main'
     steps:
       - uses: actions/checkout@v4
@@ -78,7 +76,7 @@ jobs:
             convco
             version
             --prefix=${{ env.PACKAGE_NAME }}/
-            --paths=dtml
+            --paths=${{ env.PACKAGE_PATH }}
             --paths=pyproject.toml
             --paths=poetry.lock
           "
@@ -89,15 +87,12 @@ jobs:
             new_version=$($base_version_command --bump)
           fi
           new_tag=${{ env.PACKAGE_NAME }}/"$new_version"
-          package_path=.
           echo "OLD_VERSION=$old_version" >> $GITHUB_ENV
           echo "NEW_VERSION=$new_version" >> $GITHUB_ENV
           echo "NEW_TAG=$new_tag" >> $GITHUB_ENV
-          echo "PACKAGE_PATH=$package_path" >> $GITHUB_ENV
           echo "old version: $old_version"
           echo "new version: $new_version"
           echo "new tag: $new_tag"
-          echo "packages path: $package_path"
       - name: Release and publish
         if: ${{ env.OLD_VERSION != env.NEW_VERSION }}
         env:
@@ -126,7 +121,6 @@ jobs:
           ########################################
           # Publish to JFrog
           ########################################
-          cd ${{ env.PACKAGE_PATH }}
           poetry version ${{ env.NEW_VERSION }}
           poetry config repositories.dtml "https://datamole.jfrog.io/artifactory/api/pypi/pypi-general-local"
           poetry config http-basic.dtml "" ${{ secrets.JFROG_SERVICE_JWT }}

.github/workflows/pr.yaml

@@ -6,7 +6,27 @@ on:
     types: [opened, edited, reopened, synchronize]
 
 jobs:
-  pr-title-check:
-    uses: datamole-ai/core-mle-git-workflow/.github/workflows/convco-pr-title-check.yaml@main
+  pr-convco-title-check:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install convco
+        run: |
+          curl -sSfL "https://github.com/convco/convco/releases/download/${{ env.CONVCO_VERSION }}/convco-ubuntu.zip" | zcat > /usr/local/bin/convco
+          chmod +x /usr/local/bin/convco
+      - name: Check PR Title is Conventional
+        run: echo "${{ github.event.pull_request.title }}" | convco check --from-stdin
   dismiss-stale-pr-reviews:
-    uses: datamole-ai/core-mle-git-workflow/.github/workflows/dismiss-stale-pr-reviews.yaml@main
+    runs-on: ubuntu-22.04
+    if: github.event.action == 'edited'
+    steps:
+      - name: Auth. Github-CLI
+        run: echo ${{ secrets.GITHUB_TOKEN }} | gh auth login --with-token
+        shell: bash
+      - name: Dismiss approvals
+        run: |
+          gh api "repos/${{ github.repository }}/pulls/${{ github.event.number }}/reviews" \
+            --jq '.[] | select(.state == "APPROVED") | .id' \
+            | xargs -I '{}' gh api --method=PUT -f message="Dismissed due to PR edit." \
+            "repos/${{ github.repository }}/pulls/${{ github.event.number }}/reviews/{}/dismissals"
+        shell: bash

CONTRIBUTING.md

@@ -0,0 +1,68 @@
+# Contributing to pysparkdt
+
+## Suggesting changes
+1. Create an [issue](https://github.com/datamole-ai/pysparkdt/issues) describing the change you want to make.
+
+## General workflow
+
+### Environment setup
+pysparkdt uses [Poetry](https://python-poetry.org/) for managing dependencies.
+Follow the instructions on the Poetry website to install it.
+We recommend [pyenv](https://github.com/pyenv/pyenv)
+([installer](https://github.com/pyenv/pyenv-installer)) for managing Python versions.
+```bash
+# Install Python 3.12
+pyenv install 3.12
+
+# Use pyenv's Python 3.12 for the current folder
+pyenv local 3.12
+
+# Create virtual environment (install all optional dependencies)
+poetry install --extras all
+
+# Activate Poetry virtual environment in the current shell
+poetry shell
+```
+
+You can also use `poetry run` to run commands in the virtual environment without activating it in the current shell (via `poetry shell`).
+
+
+### Test the newly implemented changes
+Create unit tests by creating a Python script in the folder `tests` prefixed with `test_`.
+The script should contain functions also prefixed with `test_` that make assertions.
+See the `tests` folder for reference.
+
+## Pull Requests & Git
+
+* Split your work into separate and atomic pull requests. Put any
+  non-obvious reasoning behind any change to the pull request description.
+  Separate “preparatory” changes and modifications from new features &
+  improvements.
+* The pull requests are squashed when merged. The PR title is used as the commit title.
+  The PR description is used as the commit description.
+* Use conventional commit messages in the PR title and description.
+  See [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/).
+  Usage of conventional commit PR titles and descriptions is enforced by the CI pipeline.
+* Prefer adding new commits over amending existing ones during the review process.
+  The latter makes it harder to review changes and track down modifications.
+
+
+## Code style
+
+* The line length is limited to 79 characters in Python code,
+except if it would make the code less readable.
+* `ruff` is used for formatting and linting Python code.
+The following commands can be used to properly format the code and check
+for linting errors with automatic fixing:
+```bash
+poetry run ruff format .
+poetry run ruff check . --fix
+```
+The following commands can be used to check if the code is properly
+formatted and check for linting errors:
+```bash
+poetry run ruff format --check .
+poetry run ruff check .
+```
+
+All of the above code style requirements are enforced by the CI pipeline.

LICENSE

@@ -0,0 +1,7 @@
+Copyright 2024 Datamole, s.r.o.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.