From e09c011f671b1b9a1fe20bafe561544baf34f3d4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Edgar=20Ram=C3=ADrez=20Mondrag=C3=B3n?= <edgar@meltano.com>
Date: Mon, 16 Oct 2023 18:15:56 -0600
Subject: [PATCH] feat: Bump Singer SDK to 0.33.0 to support comprehensions
 (#183)

---
 .github/dependabot.yml            |  1 +
 .github/workflows/meltano-run.yml | 21 ++++++++++++++++++---
 fixtures/nested.singer            |  5 +++++
 meltano.yml                       |  9 +++++++++
 poetry.lock                       | 22 +++++++++++++++++-----
 pyproject.toml                    |  7 ++-----
 scripts/nested.sh                 |  3 +++
 7 files changed, 55 insertions(+), 13 deletions(-)
 create mode 100644 fixtures/nested.singer
 create mode 100755 scripts/nested.sh

diff --git a/.github/dependabot.yml b/.github/dependabot.yml
index 92b53d7..4dbb3c8 100644
--- a/.github/dependabot.yml
+++ b/.github/dependabot.yml
@@ -16,6 +16,7 @@ updates:
       - "edgarrmondragon"
     labels:
       - "dependencies"
+    versioning-strategy: increase-if-necessary
   - package-ecosystem: "pip"
     directory: "/.github/workflows"
     schedule:
diff --git a/.github/workflows/meltano-run.yml b/.github/workflows/meltano-run.yml
index 892fb59..6748d97 100644
--- a/.github/workflows/meltano-run.yml
+++ b/.github/workflows/meltano-run.yml
@@ -22,6 +22,12 @@ jobs:
           mapping: flatten
           output_db: people.db
           target_table: people
+          install_extractor: false
+        - tap: nested
+          mapping: comprehension
+          output_db: nested.db
+          target_table: users
+          install_extractor: false
     steps:
     - uses: actions/checkout@v4
 
@@ -33,15 +39,24 @@ jobs:
       env:
         PIP_CONSTRAINT: .github/workflows/constraints.txt
       run: |
-        pipx install meltano
+        pipx install meltano --python python3.11
 
     - name: Check Meltano version
       run: |
         meltano --version
 
-    - name: Install Plugins
+    - name: Install Mapper
       run: |
-        meltano install
+        meltano install mapper meltano-map-transformer
+
+    - name: Install Extractor
+      if: matrix.install_extractor != 'false'
+      run: |
+        meltano install extractor ${{ matrix.tap }}
+
+    - name: Install Loader
+      run: |
+        meltano install loader target-sqlite
 
     - name: Run
       run: |
diff --git a/fixtures/nested.singer b/fixtures/nested.singer
new file mode 100644
index 0000000..592db46
--- /dev/null
+++ b/fixtures/nested.singer
@@ -0,0 +1,5 @@
+{"type": "SCHEMA", "stream": "users", "schema": {"required": ["id"], "type": "object", "properties": {"id": {"type": "integer"}, "fields": {"type": "array", "items": {"type": "object", "properties": {"key": {"type": "string"}, "value": {"type": "string"}}}}}}, "key_properties": ["id"]}
+{"type": "RECORD", "stream": "users", "record": {"id": 1, "fields": [{"key": "name", "value": "John Doe"}, {"key": "age", "value": "25"}]}}
+{"type": "RECORD", "stream": "users", "record": {"id": 2, "fields": [{"key": "name", "value": "Jane Doe"}, {"key": "age", "value": "30"}]}}
+{"type": "RECORD", "stream": "users", "record": {"id": 3, "fields": [{"key": "name", "value": "John Smith"}, {"key": "age", "value": "40"}]}}
+{"type": "RECORD", "stream": "users", "record": {"id": 4, "fields": [{"key": "name", "value": "Jane Smith"}, {"key": "age", "value": "50"}]}}
diff --git a/meltano.yml b/meltano.yml
index 392c41c..1e45ca9 100644
--- a/meltano.yml
+++ b/meltano.yml
@@ -23,6 +23,9 @@ plugins:
   - name: people
     namespace: people
     executable: scripts/people.sh
+  - name: nested
+    namespace: nested
+    executable: scripts/nested.sh
   loaders:
   - name: target-sqlite
     variant: meltanolabs
@@ -63,6 +66,12 @@ plugins:
         stream_maps: {}
         flattening_enabled: true
         flattening_max_depth: 1
+    - name: comprehension
+      config:
+        stream_maps:
+          users:
+            id: id
+            fields: "[f for f in fields if f['key'] != 'age']"
 environments:
 - name: dev
   config:
diff --git a/poetry.lock b/poetry.lock
index 63e4d56..7d49bda 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -849,6 +849,17 @@ docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-g
 testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"]
 testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"]
 
+[[package]]
+name = "simpleeval"
+version = "0.9.13"
+description = "A simple, safe single expression evaluator library."
+optional = false
+python-versions = "*"
+files = [
+    {file = "simpleeval-0.9.13-py2.py3-none-any.whl", hash = "sha256:22a2701a5006e4188d125d34accf2405c2c37c93f6b346f2484b6422415ae54a"},
+    {file = "simpleeval-0.9.13.tar.gz", hash = "sha256:4a30f9cc01825fe4c719c785e3762623e350c4840d5e6855c2a8496baaa65fac"},
+]
+
 [[package]]
 name = "simplejson"
 version = "3.19.1"
@@ -945,13 +956,13 @@ files = [
 
 [[package]]
 name = "singer-sdk"
-version = "0.32.0"
+version = "0.33.0"
 description = "A framework for building Singer taps"
 optional = false
 python-versions = ">=3.7.1,<4"
 files = [
-    {file = "singer_sdk-0.32.0-py3-none-any.whl", hash = "sha256:19b56e67bc9e6c7410e1830711894ae25d54609a72d06bef569d0c5a5a1b2b54"},
-    {file = "singer_sdk-0.32.0.tar.gz", hash = "sha256:af666788faa027869fa045859bd070b3d0df487c89fda18e7b683e8bb94b4ced"},
+    {file = "singer_sdk-0.33.0-py3-none-any.whl", hash = "sha256:ed0f9c67ca79964a8f517e7360197d4ddb342f341448fb3b310214ce91e48ac5"},
+    {file = "singer_sdk-0.33.0.tar.gz", hash = "sha256:e2b43496b324ca9214b731c8d2c2711d9100b6238d20b1e3187c8ae3de1b0c9d"},
 ]
 
 [package.dependencies]
@@ -973,6 +984,7 @@ python-dotenv = ">=0.20,<0.22"
 pytz = ">=2022.2.1,<2024.0.0"
 PyYAML = ">=6.0"
 requests = ">=2.25.1"
+simpleeval = ">=0.9.13,<0.10.0"
 simplejson = ">=3.17.6"
 sqlalchemy = ">=1.4,<3.0"
 typing-extensions = ">=4.2.0"
@@ -1124,5 +1136,5 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more
 
 [metadata]
 lock-version = "2.0"
-python-versions = "<3.12,>=3.7.1"
-content-hash = "89d1f563ee86bcb65c5f882c4d0b725ee8453d85f3e725beaf4ac933322e56a0"
+python-versions = ">=3.7.1,<4"
+content-hash = "26b8cc9f6b904ba6a244733fe1fc186ae266b21f4df7a723e70bf3bf42821dd8"
diff --git a/pyproject.toml b/pyproject.toml
index 7dc8cfa..1601b99 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -17,11 +17,8 @@ repository = "https://github.com/MeltanoLabs/meltano-map-transform"
 documentation = "https://github.com/MeltanoLabs/meltano-map-transform#readme"
 
 [tool.poetry.dependencies]
-python = "<3.12,>=3.7.1"
-
-[tool.poetry.dependencies.singer-sdk]
-allow-prereleases = true
-version = "~=0.32.0"
+python = ">=3.7.1,<4"
+singer-sdk = "~=0.33.0"
 
 [tool.poetry.dev-dependencies]
 pytest = "^7.4.2"
diff --git a/scripts/nested.sh b/scripts/nested.sh
new file mode 100755
index 0000000..4c93160
--- /dev/null
+++ b/scripts/nested.sh
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+cat fixtures/nested.singer