feat: Bump Singer SDK to 0.33.0 to support comprehensions (#183)

.github/dependabot.yml

@@ -16,6 +16,7 @@ updates:
       - "edgarrmondragon"
     labels:
       - "dependencies"
+    versioning-strategy: increase-if-necessary
   - package-ecosystem: "pip"
     directory: "/.github/workflows"
     schedule:

.github/workflows/meltano-run.yml

@@ -22,6 +22,12 @@ jobs:
           mapping: flatten
           output_db: people.db
           target_table: people
+          install_extractor: false
+        - tap: nested
+          mapping: comprehension
+          output_db: nested.db
+          target_table: users
+          install_extractor: false
     steps:
     - uses: actions/checkout@v4
 
@@ -33,15 +39,24 @@ jobs:
       env:
         PIP_CONSTRAINT: .github/workflows/constraints.txt
       run: |
-        pipx install meltano
+        pipx install meltano --python python3.11
 
     - name: Check Meltano version
       run: |
         meltano --version
 
-    - name: Install Plugins
+    - name: Install Mapper
       run: |
-        meltano install
+        meltano install mapper meltano-map-transformer
+
+    - name: Install Extractor
+      if: matrix.install_extractor != 'false'
+      run: |
+        meltano install extractor ${{ matrix.tap }}
+
+    - name: Install Loader
+      run: |
+        meltano install loader target-sqlite
 
     - name: Run
       run: |

fixtures/nested.singer

@@ -0,0 +1,5 @@
+{"type": "SCHEMA", "stream": "users", "schema": {"required": ["id"], "type": "object", "properties": {"id": {"type": "integer"}, "fields": {"type": "array", "items": {"type": "object", "properties": {"key": {"type": "string"}, "value": {"type": "string"}}}}}}, "key_properties": ["id"]}
+{"type": "RECORD", "stream": "users", "record": {"id": 1, "fields": [{"key": "name", "value": "John Doe"}, {"key": "age", "value": "25"}]}}
+{"type": "RECORD", "stream": "users", "record": {"id": 2, "fields": [{"key": "name", "value": "Jane Doe"}, {"key": "age", "value": "30"}]}}
+{"type": "RECORD", "stream": "users", "record": {"id": 3, "fields": [{"key": "name", "value": "John Smith"}, {"key": "age", "value": "40"}]}}
+{"type": "RECORD", "stream": "users", "record": {"id": 4, "fields": [{"key": "name", "value": "Jane Smith"}, {"key": "age", "value": "50"}]}}

meltano.yml

@@ -23,6 +23,9 @@ plugins:
   - name: people
     namespace: people
     executable: scripts/people.sh
+  - name: nested
+    namespace: nested
+    executable: scripts/nested.sh
   loaders:
   - name: target-sqlite
     variant: meltanolabs
@@ -63,6 +66,12 @@ plugins:
         stream_maps: {}
         flattening_enabled: true
         flattening_max_depth: 1
+    - name: comprehension
+      config:
+        stream_maps:
+          users:
+            id: id
+            fields: "[f for f in fields if f['key'] != 'age']"
 environments:
 - name: dev
   config:

pyproject.toml

@@ -17,11 +17,8 @@ repository = "https://github.com/MeltanoLabs/meltano-map-transform"
 documentation = "https://github.com/MeltanoLabs/meltano-map-transform#readme"
 
 [tool.poetry.dependencies]
-python = "<3.12,>=3.7.1"
-
-[tool.poetry.dependencies.singer-sdk]
-allow-prereleases = true
-version = "~=0.32.0"
+python = ">=3.7.1,<4"
+singer-sdk = "~=0.33.0"
 
 [tool.poetry.dev-dependencies]
 pytest = "^7.4.2"

scripts/nested.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+cat fixtures/nested.singer