Merge pull request #188 from keboola/fix/request-body-string

CFT-3054 Fix/request body string
keboola · Aug 14, 2024 · 3187f50 · 3187f50
2 parents a1c8ccd + 5db43c0
commit 3187f50
Show file tree

Hide file tree

Showing 10 changed files with 128 additions and 18 deletions.
diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml
@@ -106,12 +106,12 @@ jobs:
           target_image: ${{ env.APP_IMAGE}}
           tag_as_latest: true
       - name: Python Tests
-        run: docker-compose run --rm ci python -v -m unittest discover /code/python-sync-actions
+        run: docker compose run --rm ci python -v -m unittest discover /code/python-sync-actions
       - name: Run Python Functional Tests
         working-directory: ./python-sync-actions
         run: docker compose run test-calls
       - name: PHP Tests
-        run: docker-compose run --rm -v $(pwd)/build:/code/build ci
+        run: docker compose run --rm -v $(pwd)/build:/code/build ci
 
   tests-examples:
     needs:

diff --git a/README.md b/README.md
@@ -1179,8 +1179,8 @@ Best way to create and test new configurations is run extractor in docker contai
 
 - Clone this repository `git clone https://github.com/keboola/generic-extractor.git`
 - Switch to extractor directory `cd generic-extractor`
-- Build container `docker-compose build`
-- Install dependencies locally `docker-compose run --rm dev composer install`
+- Build container `docker compose build`
+- Install dependencies locally `docker compose run --rm dev composer install`
 - Create data folder for configuration `mkdir data`
 
 ## Execution
@@ -1209,20 +1209,20 @@ Best way to create and test new configurations is run extractor in docker contai
     }
   }
     ```
-- Run extraction `docker-compose run --rm dev`
+- Run extraction `docker compose run --rm dev`
 - You will find extracted data in folder `data/out`
-- Clear `data/out` by running `docker-compose run --rm dev rm -rf data/out`
+- Clear `data/out` by running `docker compose run --rm dev rm -rf data/out`
 - Repeat :)
 
 # Running tests:
 ```
-docker-compose run --rm tests
+docker compose run --rm tests
 ``` 
 
 or (with local source code and vendor copy)
 
 ```
-docker-compose run --rm tests-local
+docker compose run --rm tests-local
 ``` 
 
 ## License

diff --git a/doc/run-samples.sh b/doc/run-samples.sh
@@ -11,7 +11,7 @@ run_example() {
     EXAMPLE_NAME=$1
     rm -rf examples/$1/out/*
     mkdir -p examples/$1/out/tables/
-    docker-compose run --rm -e "KBC_EXAMPLE_NAME=$EXAMPLE_NAME" extractor
+    docker compose run --rm -e "KBC_EXAMPLE_NAME=$EXAMPLE_NAME" extractor
     if diff --brief --recursive examples/${EXAMPLE_NAME}/out/tables/ examples/${EXAMPLE_NAME}/_sample_out/ ; then
         printf "Example $EXAMPLE_NAME successful.\n"
     else
@@ -21,7 +21,7 @@ run_example() {
 }
 
 # Start mock server
-docker-compose build --force-rm --pull
+docker compose build --force-rm --pull
 
 # Run examples
 run_example "001-simple-job"

diff --git a/doc/run-single-example.sh b/doc/run-single-example.sh
@@ -11,7 +11,7 @@ run_example() {
     EXAMPLE_NAME=$1
     rm -rf examples/$1/out/*
     mkdir -p examples/$1/out/tables/
-    docker-compose run -e "KBC_EXAMPLE_NAME=$EXAMPLE_NAME" extractor
+    docker compose run -e "KBC_EXAMPLE_NAME=$EXAMPLE_NAME" extractor
     if diff --brief --recursive examples/${EXAMPLE_NAME}/out/tables/ examples/${EXAMPLE_NAME}/_sample_out/ ; then
         printf "Example $EXAMPLE_NAME successful.\n"
     else
@@ -21,7 +21,7 @@ run_example() {
 }
 
 # Start mock server
-docker-compose build --force-rm --pull
+docker compose build --force-rm --pull
 
 # Run example
 run_example $1

diff --git a/python-sync-actions/README.md b/python-sync-actions/README.md
@@ -150,7 +150,7 @@ The above value is then available in contexts as:
 
 ## Development
 
-If required, change local data folder (the `CUSTOM_FOLDER` placeholder) path to your custom path in the docker-compose
+If required, change local data folder (the `CUSTOM_FOLDER` placeholder) path to your custom path in the docker compose
 file:
 
 ```yaml
@@ -164,14 +164,14 @@ Clone this repository, init the workspace and run the component with following c
 ```
 git clone repo_path my-new-component
 cd my-new-component
-docker-compose build
-docker-compose run --rm dev
+docker compose build
+docker compose run --rm dev
 ```
 
 Run the test suite and lint check using this command:
 
 ```
-docker-compose run --rm test
+docker compose run --rm test
 ```
 
 # Integration

diff --git a/python-sync-actions/src/component.py b/python-sync-actions/src/component.py
@@ -473,7 +473,10 @@ def test_request(self):
 
         body = None
         if response.request.body:
-            body = response.request.body.decode('utf-8')
+            if isinstance(response.request.body, bytes):
+                body = response.request.body.decode('utf-8')
+            else:
+                body = response.request.body
 
         secrets_to_hide = self._get_values_to_hide()
         filtered_response = self._deep_copy_and_replace_words(self._final_response, secrets_to_hide)

diff --git a/python-sync-actions/src/configuration.py b/python-sync-actions/src/configuration.py
@@ -293,6 +293,9 @@ def build_api_request(configuration: dict) -> List[Tuple[ApiRequest, RequestCont
         method = endpoint_config.get('method', 'GET')
 
         request_content = build_request_content(method, endpoint_config.get('params', {}))
+        # use real method
+        if method.upper() == 'FORM':
+            method = 'POST'
 
         endpoint_path = endpoint_config.get('endpoint')
 
@@ -313,12 +316,18 @@ def build_api_request(configuration: dict) -> List[Tuple[ApiRequest, RequestCont
             delimiter = "."
             data_path = DataPath(path=path, delimiter=delimiter)
 
+        # query params are supported only for GET requests
+        if request_content.content_type == ContentType.none:
+            query_params = endpoint_config.get('params', {})
+        else:
+            query_params = {}
+
         result_requests.append(
             (ApiRequest(method=method,
                         endpoint_path=endpoint_path,
                         placeholders=placeholders,
                         headers=endpoint_config.get('headers', {}),
-                        query_parameters=endpoint_config.get('params', {}),
+                        query_parameters=query_params,
                         scroller=scroller),
              request_content,
              data_path))

diff --git a/python-sync-actions/tests/data_tests/test_006_post_form/config.json b/python-sync-actions/tests/data_tests/test_006_post_form/config.json
@@ -0,0 +1,25 @@
+{
+  "parameters": {
+    "api": {
+      "baseUrl": "http://private-834388-extractormock.apiary-mock.com",
+      "pagination": {
+        "method": "response.url",
+        "urlKey": "next"
+      }
+    },
+    "config": {
+      "outputBucket": "getPost",
+      "jobs": [
+        {
+          "endpoint": "post",
+          "method": "FORM",
+          "params": {
+            "parameter": "value"
+          }
+        }
+      ]
+    },
+    "__SELECTED_JOB": "0"
+  },
+  "action": "test_request"
+}
diff --git a/python-sync-actions/tests/test_component.py b/python-sync-actions/tests/test_component.py
@@ -68,6 +68,10 @@ def test_005_post(self):
         self.assertEqual(output['response']['data'], expected_data)
         expected_request_data = '{"parameter": "value"}'
         self.assertEqual(output['request']['data'], expected_request_data)
+        # url params are dropped
+        self.assertEqual(output['request']['url'], 'http://private-834388-extractormock.apiary-mock.com/post')
+        # correct content type
+        self.assertEqual(output['request']['headers']['Content-Type'], 'application/json')
 
     def test_006_post_fail(self):
         component = self._get_test_component(self._testMethodName)
@@ -79,6 +83,20 @@ def test_006_post_fail(self):
         expected_request_data = '{"parameter": "value"}'
         self.assertEqual(output['request']['data'], expected_request_data)
 
+    def test_006_post_form(self):
+        component = self._get_test_component(self._testMethodName)
+        output = component.test_request()
+        expected_data = [{'id': '123', 'status': 'post'}, {'id': 'potato', 'status': 'mashed'}]
+        self.assertEqual(output['response']['data'], expected_data)
+        expected_request_data = 'parameter=value'
+        self.assertEqual(output['request']['data'], expected_request_data)
+        # url params are dropped
+        self.assertEqual(output['request']['url'], 'http://private-834388-extractormock.apiary-mock.com/post')
+        # request method is POST
+        self.assertEqual(output['request']['method'], 'POST')
+        # correct content type
+        self.assertEqual(output['request']['headers']['Content-Type'], 'application/x-www-form-urlencoded')
+
     def test_009_empty_datafield(self):
         component = self._get_test_component(self._testMethodName)
         results, response, log, error_message = component.make_call()

diff --git a/python-sync-actions/tests/test_configuration.py b/python-sync-actions/tests/test_configuration.py
@@ -1,5 +1,6 @@
 import unittest
 
+import configuration
 from configuration import ConfigHelpers
 
 
@@ -57,3 +58,57 @@ def test_eval_source_function_true(self):
                     'params': {'grant_type': 'client_credentials', 'scope': 'read'}}
         result = self.helpers.fill_in_user_parameters(conf_objects, user_params, True)
         self.assertEqual(result, expected)
+
+    def test_query_parameters_dropped_in_post_mode(self):
+        config = {
+            "__SELECTED_JOB": "0",
+            "config": {
+                "userData": {},
+                "outputBucket": "",
+                "incrementalOutput": False,
+                "jobs": [
+                    {
+                        "__NAME": "63aa2677-41d6-49d9-8add-2ccc18e8062e",
+                        "endpoint": "v3/63aa2677-41d6-49d9-8add-2ccc18e8062e",
+                        "method": "POST",
+                        "dataType": "63aa2677-41d6-49d9-8add-2ccc18e8062e",
+                        "params": {
+                            "test": "test"
+                        }
+                    }
+                ]
+            },
+            "api": {
+                "baseUrl": "https://run.mocky.io/"
+            }
+        }
+
+        configs = configuration.convert_to_v2(config)
+        self.assertEqual(configs[0].request_parameters.query_parameters, {})
+
+    def test_query_parameters_kept_in_get_mode(self):
+        config = {
+            "__SELECTED_JOB": "0",
+            "config": {
+                "userData": {},
+                "outputBucket": "",
+                "incrementalOutput": False,
+                "jobs": [
+                    {
+                        "__NAME": "63aa2677-41d6-49d9-8add-2ccc18e8062e",
+                        "endpoint": "v3/63aa2677-41d6-49d9-8add-2ccc18e8062e",
+                        "method": "GET",
+                        "dataType": "63aa2677-41d6-49d9-8add-2ccc18e8062e",
+                        "params": {
+                            "test": "testValue"
+                        }
+                    }
+                ]
+            },
+            "api": {
+                "baseUrl": "https://run.mocky.io/"
+            }
+        }
+
+        configs = configuration.convert_to_v2(config)
+        self.assertDictEqual(configs[0].request_parameters.query_parameters, {"test": "testValue"})