* [Removed unneeded PseudoJSON and DBPersistence classes, migrated al…

…l json contained column's type to JSONB](getredash/redash#6687)

* enhanced SETUP documentation
* postgres docker image version from 12 to latest
* [Add default limit (1000) to SQL queries](getredash/redash#5088)
* new settings are implemented

SETUP.md

@@ -13,7 +13,6 @@ see https://learn.microsoft.com/en-us/windows/dev-environment/javascript/nodejs-
 nodenv install 14
 nodenv local 14
 ```
-
 Alternatively you can use nvm
 ```
 sudo apt update
@@ -47,8 +46,8 @@ nvm use v14 > /dev/null
 
 * Setup docker compose
     * `make up` or `docker-compose up --build`  to start required services like postgres app server
-    * `docker-compose run --rm server create_db` Will start server and run. exec /app/manage.py database create_tables.
-      This step is required **only once**.
+    * `docker-compose run --rm server create_db` Will start server and run. exec /app/manage.py database create_tables. This step is required **only once**.
+    * In case you get an error stating that Target database is not up to date, run: `docker-compose run server manage db stamp head`
     * Any change to SQL data made on python side requires to create a migration file for upgrading the required database columns: `docker-compose run server manage db migrate`
     * Later on and only if necessary, in order to upgrade local database run: `docker-compose run --rm server manage db upgrade`
 
@@ -99,17 +98,14 @@ docker-compose stop server && docker-compose run --rm --service-ports server deb
 ```
 
 ### Running tests locally
-
 First ensure that the "tests" database is created:
 ```
 docker-compose run --rm postgres psql -h postgres -U postgres -c "create database tests"
 ```
-
 Then run the tests:
 ```
 docker-compose run --rm server tests
 ```
-
 In order to test viz-lib folder you need to install dependencies and run tests because you cant have 2 react versions in the same project. To do that run below commands in the viz-lib folder:
 ```
 npm install antd@^3 react@^16.8 react-dom@^16.8 && npm run test
@@ -154,16 +150,18 @@ npm install
 npm run compile
 npm publish
 ```
-### Debugging notes
 
+### Debugging notes
+client side debugger
+```
 cd client
 npm start
-
-visit http://localhost:8080/ instead of using port 5050
-
-To run Python debugger:
+# visit http://localhost:8080/
+```
+Python debugger:
+```
 docker-compose stop server && docker-compose run --rm --service-ports server debug && docker-compose start server
-
+```
 To log messages to/from Plywood add to the Plywood env (in docker-compose) following variable: `LOG_MODE=request_and_response` or `LOG_MODE=response_only`
 
 ### Docker installation issues
@@ -174,6 +172,7 @@ rm  ~/.docker/config.json
 ```
 
 ## Docker connectivity issues for testing connection between containers
+This is usefull when testing fresh datasources
 ```bash
 >> docker network connect datareporter_default router
 >> docker inspect -f '{{range $key, $value := .NetworkSettings.Networks}}{{$key}} {{end}}' router
@@ -190,10 +189,10 @@ PING router (172.19.0.9) 56(84) bytes of data.
 rtt min/avg/max/mdev = 0.057/0.781/1.506/0.725 ms
 ```
 
-### How to handle package controll on Python side
+### How to handle package controll on Python side, [see settings](https://github.com/getredash/redash/blob/c97afeb327d8d54e7219ac439cc93d0f234763e5)
 ```
-# Install Poetry locally, [see-settings](https://github.com/getredash/redash/blob/c97afeb327d8d54e7219ac439cc93d0f234763e5)
-pip3 install poetry==1.6.1 # it has to be 1.6.1 or upper because of group usages
+# Install Poetry locally, it has to be 1.6.1 or upper because of group usages
+pip3 install poetry==1.6.1
 
 # Uninstall Poetry locally
 curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/901bdf0491005f1b3db41947d0d938da6838ecb9/get-poetry.py | python3 - --uninstall

client/app/components/queries/QueryEditor/AutoLimitCheckbox.jsx

@@ -0,0 +1,37 @@
+import React, { useCallback } from "react";
+import PropTypes from "prop-types";
+import recordEvent from "@/services/recordEvent";
+import Checkbox from "antd/lib/checkbox";
+import Tooltip from "antd/lib/tooltip";
+
+export default function AutoLimitCheckbox({ available, checked, onChange }) {
+  const handleClick = useCallback(() => {
+    recordEvent("checkbox_auto_limit", "screen", "query_editor", { state: !checked });
+    onChange(!checked);
+  }, [checked, onChange]);
+
+  let tooltipMessage = null;
+  if (!available) {
+    tooltipMessage = "Auto limiting is not available for this Data Source type.";
+  } else {
+    tooltipMessage = "Auto limit results to first 1000 rows.";
+  }
+
+  return (
+    <Tooltip placement="top" title={tooltipMessage}>
+      <Checkbox
+        className="query-editor-controls-checkbox"
+        disabled={!available}
+        onClick={handleClick}
+        checked={available && checked}>
+        LIMIT 1000
+      </Checkbox>
+    </Tooltip>
+  );
+}
+
+AutoLimitCheckbox.propTypes = {
+  available: PropTypes.bool,
+  checked: PropTypes.bool.isRequired,
+  onChange: PropTypes.func.isRequired,
+};

client/app/components/queries/QueryEditor/QueryEditorControls.jsx

@@ -7,6 +7,7 @@ import Select from "antd/lib/select";
 import KeyboardShortcuts, { humanReadableShortcut } from "@/services/KeyboardShortcuts";
 
 import AutocompleteToggle from "./AutocompleteToggle";
+import AutoLimitCheckbox from "@/components/queries/QueryEditor/AutoLimitCheckbox";
 import "./QueryEditorControls.less";
 
 export function ButtonTooltip({ title, shortcut, ...props }) {
@@ -38,6 +39,7 @@ export default function EditorControl({
   saveButtonProps,
   executeButtonProps,
   autocompleteToggleProps,
+  autoLimitCheckboxProps,
   dataSourceSelectorProps,
 }) {
   useEffect(() => {
@@ -84,6 +86,7 @@ export default function EditorControl({
           onToggle={autocompleteToggleProps.onToggle}
         />
       )}
+      {autoLimitCheckboxProps !== false && <AutoLimitCheckbox {...autoLimitCheckboxProps} />}
       {dataSourceSelectorProps === false && <span className="query-editor-controls-spacer" />}
       {dataSourceSelectorProps !== false && (
         <Select
@@ -153,6 +156,10 @@ EditorControl.propTypes = {
       onToggle: PropTypes.func,
     }),
   ]),
+  autoLimitCheckboxProps: PropTypes.oneOfType([
+    PropTypes.bool, // `false` to hide
+    PropTypes.shape(AutoLimitCheckbox.propTypes),
+  ]),
   dataSourceSelectorProps: PropTypes.oneOfType([
     PropTypes.bool, // `false` to hide
     PropTypes.shape({
@@ -175,5 +182,6 @@ EditorControl.defaultProps = {
   saveButtonProps: false,
   executeButtonProps: false,
   autocompleteToggleProps: false,
+  autoLimitCheckboxProps: false,
   dataSourceSelectorProps: false,
 };

client/app/components/queries/QueryEditor/QueryEditorControls.less

@@ -21,6 +21,12 @@
     }
   }
 
+  .query-editor-controls-checkbox {
+    display: inline-block;
+    white-space: nowrap;
+    margin: auto 5px;
+  }
+
   .query-editor-controls-spacer {
     flex: 1 1 auto;
     height: 35px; // same as Antd <Select>

client/app/pages/queries/QuerySource.jsx

@@ -26,6 +26,7 @@ import { getEditorComponents } from "@/components/queries/editor-components";
 import useQuery from "./hooks/useQuery";
 import useVisualizationTabHandler from "./hooks/useVisualizationTabHandler";
 import useAutocompleteFlags from "./hooks/useAutocompleteFlags";
+import useAutoLimitFlags from "./hooks/useAutoLimitFlags";
 import useQueryExecute from "./hooks/useQueryExecute";
 import useQueryResultData from "@/lib/useQueryResultData";
 import useQueryDataSources from "./hooks/useQueryDataSources";
@@ -77,6 +78,7 @@ function QuerySource(props) {
 
   const editorRef = useRef(null);
   const [autocompleteAvailable, autocompleteEnabled, toggleAutocomplete] = useAutocompleteFlags(schema);
+  const [autoLimitAvailable, autoLimitChecked, setAutoLimit] = useAutoLimitFlags(dataSource, query, setQuery);
 
   const [handleQueryEditorChange] = useDebouncedCallback(queryText => {
     setQuery(extend(query.clone(), { query: queryText }));
@@ -305,6 +307,11 @@ function QuerySource(props) {
                         enabled: autocompleteEnabled,
                         onToggle: toggleAutocomplete,
                       }}
+                      autoLimitCheckboxProps={{
+                        available: autoLimitAvailable,
+                        checked: autoLimitChecked,
+                        onChange: setAutoLimit,
+                      }}
                       dataSourceSelectorProps={
                         dataSource
                           ? {

client/app/pages/queries/hooks/useAutoLimitFlags.js

@@ -0,0 +1,24 @@
+import { useCallback, useState } from "react";
+import localOptions from "@/lib/localOptions";
+import { get, extend } from "lodash";
+
+function isAutoLimitAvailable(dataSource) {
+  return get(dataSource, "supports_auto_limit", false);
+}
+
+export default function useAutoLimitFlags(dataSource, query, setQuery) {
+  const isAvailable = isAutoLimitAvailable(dataSource);
+  const [isChecked, setIsChecked] = useState(query.options.apply_auto_limit);
+  query.options.apply_auto_limit = isChecked;
+
+  const setAutoLimit = useCallback(
+    state => {
+      setIsChecked(state);
+      localOptions.set("applyAutoLimit", state);
+      setQuery(extend(query.clone(), { options: { ...query.options, apply_auto_limit: state } }));
+    },
+    [query, setQuery]
+  );
+
+  return [isAvailable, isChecked, setAutoLimit];
+}

client/app/pages/queries/hooks/useQuery.js

@@ -5,6 +5,7 @@ import navigateTo from "@/components/ApplicationArea/navigateTo";
 export default function useQuery(originalQuery) {
   const [query, setQuery] = useState(originalQuery);
   const [originalQuerySource, setOriginalQuerySource] = useState(originalQuery.query);
+  const [originalAutoLimit, setOriginalAutoLimit] = useState(query.options.apply_auto_limit);
 
   const updateQuery = useUpdateQuery(query, updatedQuery => {
     // It's important to update URL first, and only then update state
@@ -14,6 +15,7 @@ export default function useQuery(originalQuery) {
     }
     setQuery(updatedQuery);
     setOriginalQuerySource(updatedQuery.query);
+    setOriginalAutoLimit(updatedQuery.options.apply_auto_limit);
   });
 
   return useMemo(

client/app/services/query-result.js

@@ -441,11 +441,11 @@ class QueryResult {
     return `${queryName.replace(/ /g, "_") + moment(this.getUpdatedAt()).format("_YYYY_MM_DD")}.${fileType}`;
   }
 
-  static getByQueryId(id, parameters, maxAge) {
+  static getByQueryId(id, parameters, applyAutoLimit, maxAge) {
     const queryResult = new QueryResult();
 
     axios
-      .post(`api/queries/${id}/results`, { id, parameters, max_age: maxAge })
+      .post(`api/queries/${id}/results`, { id, parameters, apply_auto_limit: applyAutoLimit, max_age: maxAge })
       .then(response => {
         queryResult.update(response);
 
@@ -460,13 +460,14 @@ class QueryResult {
     return queryResult;
   }
 
-  static get(dataSourceId, query, parameters, maxAge, queryId) {
+  static get(dataSourceId, query, parameters, applyAutoLimit, maxAge, queryId) {
     const queryResult = new QueryResult();
 
     const params = {
       data_source_id: dataSourceId,
       parameters,
       query,
+      apply_auto_limit: applyAutoLimit,
       max_age: maxAge,
     };
 

client/app/services/query.js

@@ -50,6 +50,7 @@ export class Query {
     if (!has(this, "options")) {
       this.options = {};
     }
+    this.options.apply_auto_limit = !!this.options.apply_auto_limit;
 
     if (!isArray(this.options.parameters)) {
       this.options.parameters = [];
@@ -130,8 +131,9 @@ export class Query {
   }
 
   getQueryResult(maxAge) {
-    const execute = () => QueryResult.getByQueryId(this.id, this.getParameters().getExecutionValues(), maxAge);
-    return this.prepareQueryResultExecution(execute, maxAge);
+    const execute = () =>
+      QueryResult.getByQueryId(this.id, this.getParameters().getExecutionValues(), this.getAutoLimit(), maxAge);
+      return this.prepareQueryResultExecution(execute, maxAge);
   }
 
   getQueryResultByText(maxAge, selectedQueryText) {
@@ -141,7 +143,7 @@ export class Query {
     }
 
     const parameters = this.getParameters().getExecutionValues({ joinListValues: true });
-    const execute = () => QueryResult.get(this.data_source_id, queryText, parameters, maxAge, this.id);
+    const execute = () => QueryResult.get(this.data_source_id, queryText, parameters, this.getAutoLimit(), maxAge, this.id);
     return this.prepareQueryResultExecution(execute, maxAge);
   }
 
@@ -184,6 +186,10 @@ export class Query {
     return this.$parameters;
   }
 
+  getAutoLimit() {
+    return this.options.apply_auto_limit;
+  }
+
   getParametersDefs(update = true) {
     return this.getParameters().get(update);
   }

docker-compose.yml

@@ -69,13 +69,14 @@ services:
     ports:
       - "6379"
   postgres:
-    image: postgres:12-alpine
-    # The following turns the DB into less durable, but gains significant performance improvements for the tests run (x3
-    # improvement on my personal machine). We should consider moving this into a dedicated Docker Compose configuration for
-    # tests.
+    image: pgautoupgrade/pgautoupgrade:latest
     ports:
       - "15432:5432"
       - "5432:5432"
+    # The following turns the DB into less durable, but gains significant performance improvements for the tests run (x3
+    # improvement on my personal machine). We should consider moving this into a dedicated Docker Compose configuration for
+    # tests.
+    command: "postgres -c fsync=off -c full_page_writes=off -c synchronous_commit=OFF"
     restart: unless-stopped
     environment:
       POSTGRES_HOST_AUTH_METHOD: "trust"