Improve query performance for ClickHouse plugin

CHANGELOG.md

@@ -59,6 +59,7 @@ NOTE: As semantic versioning states all 0.y.z releases can contain breaking chan
 - [#140](https://github.com/kobsio/kobs/pull/140): Fill the chart for the distribution of the log lines with zero value.
 - [#141](https://github.com/kobsio/kobs/pull/141): Add `node_modules` to `.dockerignore`.
 - [#144](https://github.com/kobsio/kobs/pull/144): Avoid timeouts for long running requests in the ClickHouse plugin.
+- [#147](https://github.com/kobsio/kobs/pull/147): Improve query performance for ClickHouse plugin and allow custom values for the maximum amount of documents, which should be returned (see [#133](https://github.com/kobsio/kobs/pull/133)).
 
 ## [v0.5.0](https://github.com/kobsio/kobs/releases/tag/v0.5.0) (2021-08-03)
 

docs/plugins/clickhouse.md

@@ -28,6 +28,7 @@ The following options can be used for a panel with the ClickHouse plugin:
 | fields | []string | A list of fields to display in the results table. If this field is omitted, the whole document is displayed in the results table. This field is only available for the `logs`. | No |
 | order | string | Order for the returned logs. Must be `ascending` or `descending`. The default value for this field is `descending`. | No |
 | orderBy | string | The name of the field, by which the results should be orderd. The default value for this field is `timestamp`. | No |
+| maxDocuments | string | The maximum amount of documents, which should be returned. The default value for this field is `1000`. | No |
 
 ```yaml
 ---

plugins/clickhouse/clickhouse.go

@@ -87,12 +87,13 @@ func (router *Router) getLogs(w http.ResponseWriter, r *http.Request) {
 	query := r.URL.Query().Get("query")
 	order := r.URL.Query().Get("order")
 	orderBy := r.URL.Query().Get("orderBy")
+	maxDocuments := r.URL.Query().Get("maxDocuments")
 	limit := r.URL.Query().Get("limit")
 	offset := r.URL.Query().Get("offset")
 	timeStart := r.URL.Query().Get("timeStart")
 	timeEnd := r.URL.Query().Get("timeEnd")
 
-	log.WithFields(logrus.Fields{"name": name, "query": query, "order": order, "orderBy": orderBy, "limit": limit, "offset": offset, "timeStart": timeStart, "timeEnd": timeEnd}).Tracef("getLogs")
+	log.WithFields(logrus.Fields{"name": name, "query": query, "order": order, "orderBy": orderBy, "maxDocuments": maxDocuments, "limit": limit, "offset": offset, "timeStart": timeStart, "timeEnd": timeEnd}).Tracef("getLogs")
 
 	i := router.getInstance(name)
 	if i == nil {
@@ -115,6 +116,15 @@ func (router *Router) getLogs(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
+	parsedMaxDocuments := int64(1000)
+	if maxDocuments != "" {
+		parsedMaxDocuments, err = strconv.ParseInt(maxDocuments, 10, 64)
+		if err != nil {
+			errresponse.Render(w, r, err, http.StatusBadRequest, "Could not parse maxDocuments")
+			return
+		}
+	}
+
 	parsedTimeStart, err := strconv.ParseInt(timeStart, 10, 64)
 	if err != nil {
 		errresponse.Render(w, r, err, http.StatusBadRequest, "Could not parse start time")
@@ -151,7 +161,7 @@ func (router *Router) getLogs(w http.ResponseWriter, r *http.Request) {
 		}
 	}()
 
-	documents, fields, count, took, buckets, newOffset, newTimeStart, err := i.GetLogs(r.Context(), query, order, orderBy, parsedLimit, parsedOffset, parsedTimeStart, parsedTimeEnd)
+	documents, fields, count, took, buckets, newOffset, newTimeStart, err := i.GetLogs(r.Context(), query, order, orderBy, parsedMaxDocuments, parsedLimit, parsedOffset, parsedTimeStart, parsedTimeEnd)
 	if err != nil {
 		errresponse.Render(w, r, err, http.StatusBadRequest, "Could not get logs")
 		return

plugins/clickhouse/pkg/instance/instance.go

@@ -73,7 +73,7 @@ func (i *Instance) GetSQL(ctx context.Context, query string) ([][]interface{}, [
 
 // GetLogs parses the given query into the sql syntax, which is then run against the ClickHouse instance. The returned
 // rows are converted into a document schema which can be used by our UI.
-func (i *Instance) GetLogs(ctx context.Context, query, order, orderBy string, limit, offset, timeStart, timeEnd int64) ([]map[string]interface{}, []string, int64, int64, []Bucket, int64, int64, error) {
+func (i *Instance) GetLogs(ctx context.Context, query, order, orderBy string, maxDocuments, limit, offset, timeStart, timeEnd int64) ([]map[string]interface{}, []string, int64, int64, []Bucket, int64, int64, error) {
 	var count int64
 	var buckets []Bucket
 	var documents []map[string]interface{}
@@ -97,28 +97,6 @@ func (i *Instance) GetLogs(ctx context.Context, query, order, orderBy string, li
 	// following queries we can reuse the data returned by the first query, because the number of documents shouldn't
 	// change in the selected time range.
 	if offset == 0 {
-		// Determine the number of documents, which are available in the users selected time range. We are using the same
-		// query as to get the documents, but we are skipping the limit and offset parameters.
-		sqlQueryCount := fmt.Sprintf("SELECT count(*) FROM %s.logs WHERE timestamp >= ? AND timestamp <= ? %s SETTINGS skip_unavailable_shards = 1", i.database, conditions)
-		log.WithFields(logrus.Fields{"query": sqlQueryCount, "timeStart": timeStart, "timeEnd": timeEnd}).Tracef("sql count query")
-		rowsCount, err := i.client.QueryContext(ctx, sqlQueryCount, time.Unix(timeStart, 0), time.Unix(timeEnd, 0))
-		if err != nil {
-			return nil, nil, 0, 0, nil, offset, timeStart, err
-		}
-		defer rowsCount.Close()
-
-		for rowsCount.Next() {
-			if err := rowsCount.Scan(&count); err != nil {
-				return nil, nil, 0, 0, nil, offset, timeStart, err
-			}
-		}
-
-		// If the document count returns zero documents we can skip the other database calls and immediately return the
-		// result of the API request.
-		if count == 0 {
-			return nil, nil, count, time.Now().Sub(queryStartTime).Milliseconds(), nil, offset, timeStart, nil
-		}
-
 		// Now we are creating 30 buckets for the selected time range and count the documents in each bucket. This is
 		// used to render the distribution chart, which shows how many documents/rows are available within a bucket.
 		if timeEnd-timeStart > 30 {
@@ -153,25 +131,35 @@ func (i *Instance) GetLogs(ctx context.Context, query, order, orderBy string, li
 				return buckets[i].Interval < buckets[j].Interval
 			})
 
-			// We are only returning the first 10000 documents in buckets of the given limit, to speed up the following
-			// query to get the documents. For that we are looping through the sorted buckets and using the timestamp from
-			// the bucket where the sum of all newer buckets contains 10000 docuemnts.
-			// This new start time is then also returned in the response and can be used for the "load more" call as the new
-			// start date. In these follow up calls the start time isn't changed again, because we are skipping the count
-			// and bucket queries.
-			// NOTE: If a user has problems with this limit in the future, we can provide an option for this via the
-			// config.yaml file or maybe even better via an additional field in the Options component in the React UI.
+			// We are only returning the first 1000 documents in buckets of the given limit, to speed up the following
+			// query to get the documents. For that we are looping through the sorted buckets and using the timestamp
+			// from the bucket where the sum of all newer buckets contains 1000 docuemnts.
+			// This new start time is then also returned in the response and can be used for the "load more" call as the
+			// new start date. In these follow up calls the start time isn't changed again, because we are skipping the
+			// count and bucket queries.
+			// The default value of 1000 documents can be overwritten by a user, by providing the maxDocuments parameter
+			// in the request.
 			var bucketCount int64
 			for i := len(buckets) - 1; i >= 0; i-- {
 				bucketCount = bucketCount + buckets[i].Count
-				if bucketCount > 10000 {
+				if bucketCount > maxDocuments {
 					timeStart = buckets[i].Interval
 					break
 				}
 			}
+
+			for _, bucket := range buckets {
+				count = count + bucket.Count
+			}
 		}
 	}
 
+	// If the provided max documents option is zero or negative we just return the count and buckets for the provided
+	// query.
+	if maxDocuments <= 0 {
+		return documents, fields, count, time.Now().Sub(queryStartTime).Milliseconds(), buckets, offset + limit, timeStart, nil
+	}
+
 	parsedOrder := parseOrder(order, orderBy)
 
 	// Now we are building and executing our sql query. We always return all fields from the logs table, where the

plugins/clickhouse/src/components/page/Logs.tsx

@@ -27,6 +27,7 @@ import LogsFields from './LogsFields';
 interface IPageLogsProps {
   name: string;
   fields?: string[];
+  maxDocuments: string;
   order: string;
   orderBy: string;
   query: string;
@@ -37,6 +38,7 @@ interface IPageLogsProps {
 const PageLogs: React.FunctionComponent<IPageLogsProps> = ({
   name,
   fields,
+  maxDocuments,
   order,
   orderBy,
   query,
@@ -46,13 +48,13 @@ const PageLogs: React.FunctionComponent<IPageLogsProps> = ({
   const history = useHistory();
 
   const { isError, isFetching, isLoading, data, error, fetchNextPage, refetch } = useInfiniteQuery<ILogsData, Error>(
-    ['clickhouse/logs', query, order, orderBy, times],
+    ['clickhouse/logs', query, order, orderBy, maxDocuments, times],
     async ({ pageParam }) => {
       try {
         const response = await fetch(
           `/api/plugins/clickhouse/logs/${name}?query=${encodeURIComponent(
             query,
-          )}&order=${order}&orderBy=${encodeURIComponent(orderBy)}&timeStart=${
+          )}&order=${order}&orderBy=${encodeURIComponent(orderBy)}&maxDocuments=${maxDocuments}&timeStart=${
             pageParam && pageParam.timeStart ? pageParam.timeStart : times.timeStart
           }&timeEnd=${times.timeEnd}&limit=100&offset=${pageParam && pageParam.offset ? pageParam.offset : ''}`,
           {

plugins/clickhouse/src/components/page/LogsPage.tsx

@@ -20,9 +20,11 @@ const LogsPage: React.FunctionComponent<IPluginPageProps> = ({ name, displayName
 
     history.push({
       pathname: location.pathname,
-      search: `?query=${opts.query}&order=${opts.order}&orderBy=${opts.orderBy}&time=${opts.times.time}&timeEnd=${
-        opts.times.timeEnd
-      }&timeStart=${opts.times.timeStart}${fields.length > 0 ? fields.join('') : ''}`,
+      search: `?query=${opts.query}&order=${opts.order}&orderBy=${opts.orderBy}&maxDocuments=${
+        opts.maxDocuments
+      }&time=${opts.times.time}&timeEnd=${opts.times.timeEnd}&timeStart=${opts.times.timeStart}${
+        fields.length > 0 ? fields.join('') : ''
+      }`,
     });
   };
 
@@ -65,6 +67,7 @@ const LogsPage: React.FunctionComponent<IPluginPageProps> = ({ name, displayName
           query={options.query}
           order={options.order}
           orderBy={options.orderBy}
+          maxDocuments={options.maxDocuments}
           fields={options.fields}
           times={options.times}
           setOptions={changeOptions}
@@ -79,6 +82,7 @@ const LogsPage: React.FunctionComponent<IPluginPageProps> = ({ name, displayName
             query={options.query}
             order={options.order}
             orderBy={options.orderBy}
+            maxDocuments={options.maxDocuments}
             selectField={selectField}
             times={options.times}
           />

plugins/clickhouse/src/components/page/LogsToolbar.tsx

@@ -22,11 +22,13 @@ const LogsToolbar: React.FunctionComponent<ILogsToolbarProps> = ({
   query,
   order,
   orderBy,
+  maxDocuments,
   fields,
   times,
   setOptions,
 }: ILogsToolbarProps) => {
   const [data, setData] = useState<IOptions>({
+    maxDocuments: maxDocuments,
     order: order,
     orderBy: orderBy,
     query: query,
@@ -57,13 +59,14 @@ const LogsToolbar: React.FunctionComponent<ILogsToolbarProps> = ({
     timeEnd: number,
     timeStart: number,
   ): void => {
-    if (additionalFields && additionalFields.length === 2) {
+    if (additionalFields && additionalFields.length === 3) {
       const tmpData = { ...data };
 
       if (refresh) {
         setOptions({
           ...tmpData,
           fields: fields,
+          maxDocuments: additionalFields[2].value,
           order: additionalFields[1].value,
           orderBy: additionalFields[0].value,
           times: { time: time, timeEnd: timeEnd, timeStart: timeStart },
@@ -72,6 +75,7 @@ const LogsToolbar: React.FunctionComponent<ILogsToolbarProps> = ({
 
       setData({
         ...tmpData,
+        maxDocuments: additionalFields[2].value,
         order: additionalFields[1].value,
         orderBy: additionalFields[0].value,
         times: { time: time, timeEnd: timeEnd, timeStart: timeStart },
@@ -93,7 +97,7 @@ const LogsToolbar: React.FunctionComponent<ILogsToolbarProps> = ({
                   {
                     label: 'Order By',
                     name: 'orderBy',
-                    placeholder: '',
+                    placeholder: 'timestamp',
                     value: data.orderBy,
                   },
                   {
@@ -104,6 +108,12 @@ const LogsToolbar: React.FunctionComponent<ILogsToolbarProps> = ({
                     value: data.order,
                     values: ['ascending', 'descending'],
                   },
+                  {
+                    label: 'Max Documents',
+                    name: 'maxDocuments',
+                    placeholder: '1000',
+                    value: data.maxDocuments,
+                  },
                 ]}
                 time={data.times.time}
                 timeEnd={data.times.timeEnd}

plugins/clickhouse/src/components/panel/Logs.tsx

@@ -50,9 +50,9 @@ const Logs: React.FunctionComponent<ILogsProps> = ({
         const response = await fetch(
           `/api/plugins/clickhouse/logs/${name}?query=${encodeURIComponent(selectedQuery.query)}&order=${
             selectedQuery.order || ''
-          }&orderBy=${encodeURIComponent(selectedQuery.orderBy || '')}&timeStart=${times.timeStart}&timeEnd=${
-            times.timeEnd
-          }&limit=100&offset=${pageParam || ''}`,
+          }&orderBy=${encodeURIComponent(selectedQuery.orderBy || '')}&maxDocuments=${
+            selectedQuery.maxDocuments || ''
+          }&timeStart=${times.timeStart}&timeEnd=${times.timeEnd}&limit=100&offset=${pageParam || ''}`,
           {
             method: 'get',
           },

plugins/clickhouse/src/utils/helpers.ts

@@ -5,6 +5,7 @@ import { IOptions } from './interfaces';
 export const getOptionsFromSearch = (search: string): IOptions => {
   const params = new URLSearchParams(search);
   const fields = params.getAll('field');
+  const maxDocuments = params.get('maxDocuments');
   const order = params.get('order');
   const orderBy = params.get('orderBy');
   const query = params.get('query');
@@ -14,6 +15,7 @@ export const getOptionsFromSearch = (search: string): IOptions => {
 
   return {
     fields: fields.length > 0 ? fields : undefined,
+    maxDocuments: maxDocuments ? maxDocuments : '',
     order: order ? order : 'ascending',
     orderBy: orderBy ? orderBy : '',
     query: query ? query : '',

plugins/clickhouse/src/utils/interfaces.ts

@@ -7,6 +7,7 @@ export interface IOptions {
   fields?: string[];
   order: string;
   orderBy: string;
+  maxDocuments: string;
   query: string;
   times: IPluginTimes;
 }
@@ -24,6 +25,7 @@ export interface IQuery {
   fields?: string[];
   order?: string;
   orderBy?: string;
+  maxDocuments?: string;
 }
 
 // ILogsData is the interface of the data returned from our Go API for the logs view of the ClickHouse plugin.

plugins/elasticsearch/src/components/panel/LogsChart.tsx

@@ -14,8 +14,6 @@ const LogsChart: React.FunctionComponent<ILogsChartProps> = ({ buckets }: ILogsC
     return <div style={{ height: '250px' }}></div>;
   }
 
-  console.log(buckets);
-
   return (
     <div style={{ height: '250px' }}>
       <ResponsiveBarCanvas