Merge pull request #557 from Nozbe/zobeirhamid-master

raw sql queries

CHANGELOG.md

@@ -15,6 +15,7 @@ This is a **massive** new update to WatermelonDB! 🍉
   Expect more improvements in the coming releases!
 - **Improved LokiJS adapter**. Option to disable web workers, important Safari 13 fix, better performance,
   and now works in Private Modes
+- **Raw SQL queries** now available on iOS and Android thanks to the community
 - **Improved TypeScript support** — thanks to the community
 
 ### ⚠️ Breaking
@@ -24,6 +25,9 @@ This is a **massive** new update to WatermelonDB! 🍉
 
 ### New featuers
 
+- [Collection] Add `Collection.unsafeFetchRecordsWithSQL()` method. You can use it to fetch record using
+  raw SQL queries on iOS and Android. Please be careful to avoid SQL injection and other pitfalls of
+  raw queries
 - [LokiJS] Introduces new `new LokiJSAdapter({ ..., experimentalUseIncrementalIndexedDB: true })` option.
   When enabled, database will be saved to browser's IndexedDB using a new adapter that only saves the
   changed records, instead of the entire database.

docs-master/Query.md

@@ -129,7 +129,7 @@ This queries all comments that are **both** verified **and** awesome.
 | `Q.where('status', Q.like('%bl_sh%'))` | `/.*bl.sh.*/i` (See note below!) |
 | `Q.where('status', Q.notLike('%bl_sh%'))` | `/^((!?.*bl.sh.*).)*$/i` (Inverse regex match) (See note below!) |
 
-**Note:** It's NOT SAFE to use `Q.like` and `Q.notLike` with user input directly, because special characters like `%` or `_` are not escaped. Always sanitize user input like so: 
+**Note:** It's NOT SAFE to use `Q.like` and `Q.notLike` with user input directly, because special characters like `%` or `_` are not escaped. Always sanitize user input like so:
 ```js
 Q.like(`%${Q.sanitizeLikeString(userInput)}%`)
 Q.notLike(`%${Q.sanitizeLikeString(userInput)}%`)
@@ -201,9 +201,15 @@ commentCollection.query(
 )
 ```
 
-### Raw queries
+### Raw Queries
 
-If this Query syntax is not enough for you, and you need to get your hands dirty on a raw SQL or Loki query, you need **rawQueries**. How to use them? Well, **we need your help** to finish this — [please contribute!](https://github.com/Nozbe/WatermelonDB/pull/199) ❤️
+If this Query syntax is not enough for you, and you need to get your hands dirty on a raw SQL or Loki query, you need **rawQueries**. For now, only record SQL queries are available. If you need other SQL queries or LokiJS raw queries, please contribute!
+
+```js
+const records = commentCollection.unsafeFetchRecordsWithSQL('select * from comments where ...')
+```
+
+Please don't use this if you don't know what you're doing. The method name is called `unsafe` for a reason. You need to be sure to properly sanitize user values to avoid SQL injection, and filter out deleted records using `where _status is not 'deleted'` clause
 
 ### `null` behavior
 

src/Collection/index.d.ts

@@ -28,6 +28,8 @@ declare module '@nozbe/watermelondb/Collection' {
 
     public query(...conditions: Condition[]): Query<Record>
 
+    public unsafeFetchRecordsWithSQL(sql: string): Promise<Record[]>
+
     public create(recordBuilder?: (record: Record) => void): Promise<Record>
 
     public prepareCreate(recordBuilder?: (record: Record) => void): Record

src/Collection/index.js

@@ -16,6 +16,7 @@ import { type DirtyRaw } from '../RawRecord'
 
 import RecordCache from './RecordCache'
 import { CollectionChangeTypes } from './common'
+import type { SQLDatabaseAdapter } from '../adapters/type'
 
 type CollectionChangeType = 'created' | 'updated' | 'destroyed'
 export type CollectionChange<Record: Model> = { record: Record, type: CollectionChangeType }
@@ -97,6 +98,18 @@ export default class Collection<Record: Model> {
     return this._cache.recordsFromQueryResult(rawRecords)
   }
 
+  async unsafeFetchRecordsWithSQL(sql: string): Promise<Record[]> {
+    const { adapter } = this.database
+    invariant(
+      typeof (adapter: any).unsafeSqlQuery === 'function',
+      'unsafeFetchRecordsWithSQL called on database that does not support SQL',
+    )
+    const sqlAdapter: SQLDatabaseAdapter = (adapter: any)
+    const rawRecords = await sqlAdapter.unsafeSqlQuery(this.modelClass.table, sql)
+
+    return this._cache.recordsFromQueryResult(rawRecords)
+  }
+
   // See: Query.fetchCount
   fetchCount(query: Query<Record>): Promise<number> {
     return this.database.adapter.count(query.serialize())

src/adapters/__tests__/commonTests.js

@@ -229,6 +229,51 @@ export default () => [
       expect(await adapter.count(taskQuery(Q.where('order', 4)))).toBe(0)
     },
   ],
+  [
+    'can query records in raw query format',
+    async (adapter, AdapterClass) => {
+      if (AdapterClass.name === 'SQLiteAdapter') {
+        const record1 = mockTaskRaw({ id: 't1', text1: 'bar', bool1: false, order: 1 })
+        const record2 = mockTaskRaw({ id: 't2', text1: 'baz', bool1: true, order: 2 })
+        const record3 = mockTaskRaw({ id: 't3', text1: 'abc', bool1: false, order: 3 })
+
+        await adapter.batch([
+          ['create', 'tasks', record1],
+          ['create', 'tasks', record2],
+          ['create', 'tasks', record3],
+        ])
+
+        // all records
+        expectSortedEqual(await adapter.unsafeSqlQuery('tasks', `SELECT * FROM tasks`), [
+          't1',
+          't2',
+          't3',
+        ])
+
+        expectSortedEqual(
+          await adapter.unsafeSqlQuery('tasks', `SELECT * FROM tasks WHERE bool1 = 0`),
+          ['t1', 't3'],
+        )
+
+        expectSortedEqual(
+          await adapter.unsafeSqlQuery('tasks', `SELECT * FROM tasks WHERE id = 't2'`),
+          ['t2'],
+        )
+
+        expectSortedEqual(
+          await adapter.unsafeSqlQuery('tasks', `SELECT * FROM tasks WHERE \`order\` = 2`),
+          ['t2'],
+        )
+
+        expectSortedEqual(
+          await adapter.unsafeSqlQuery('tasks', `SELECT * FROM tasks WHERE text1 = 'nope'`),
+          [],
+        )
+      } else {
+        expect(adapter.unsafeSqlQuery).toBe(undefined)
+      }
+    },
+  ],
   [
     'compacts query results',
     async _adapter => {

src/adapters/common.js

@@ -3,7 +3,7 @@
 import { devMeasureTimeAsync, logger, invariant } from '../utils/common'
 import type { RecordId } from '../Model'
 import type { SerializedQuery } from '../Query'
-import type { TableSchema } from '../Schema'
+import type { TableSchema, TableName } from '../Schema'
 import type { BatchOperation, CachedQueryResult, CachedFindResult, DatabaseAdapter } from './type'
 import { sanitizedRaw, type DirtyRaw } from '../RawRecord'
 
@@ -85,6 +85,15 @@ export async function devLogQuery(
   return dirtyRecords
 }
 
+export async function devLogSQLQuery(
+  executeBlock: () => Promise<CachedQueryResult>,
+  table: TableName<any>,
+): Promise<CachedQueryResult> {
+  const [dirtyRecords, time] = await devMeasureTimeAsync(executeBlock)
+  shouldLogAdapterTimes && logger.log(`[DB] Loaded ${dirtyRecords.length} ${table} in ${time}ms`)
+  return dirtyRecords
+}
+
 export async function devLogCount(
   executeBlock: () => Promise<number>,
   query: SerializedQuery,

src/adapters/sqlite/index.d.ts

@@ -45,6 +45,8 @@ declare module '@nozbe/watermelondb/adapters/sqlite' {
 
     query<T extends Model>(query: Query<T>): Promise<CachedQueryResult>
 
+    unsafeSqlQuery<T extends Model>(sql: string, tableName: TableName<T>): Promise<CachedQueryResult>
+
     removeLocal(key: string): Promise<void>
 
     setLocal(key: string, value: string): Promise<void>

src/adapters/sqlite/index.js

@@ -8,14 +8,21 @@ import type { RecordId } from '../../Model'
 import type { SerializedQuery } from '../../Query'
 import type { TableName, AppSchema, SchemaVersion } from '../../Schema'
 import type { SchemaMigrations, MigrationStep } from '../../Schema/migrations'
-import type { DatabaseAdapter, CachedQueryResult, CachedFindResult, BatchOperation } from '../type'
+import type {
+  DatabaseAdapter,
+  SQLDatabaseAdapter,
+  CachedQueryResult,
+  CachedFindResult,
+  BatchOperation,
+} from '../type'
 import {
   type DirtyFindResult,
   type DirtyQueryResult,
   sanitizeFindResult,
   sanitizeQueryResult,
   devLogFind,
   devLogQuery,
+  devLogSQLQuery,
   devLogCount,
   devLogBatch,
   devLogSetUp,
@@ -66,7 +73,7 @@ export type SQLiteAdapterOptions = $Exact<{
   migrations?: SchemaMigrations,
 }>
 
-export default class SQLiteAdapter implements DatabaseAdapter {
+export default class SQLiteAdapter implements DatabaseAdapter, SQLDatabaseAdapter {
   schema: AppSchema
 
   migrations: ?SchemaMigrations
@@ -191,6 +198,17 @@ export default class SQLiteAdapter implements DatabaseAdapter {
     )
   }
 
+  unsafeSqlQuery(tableName: TableName<any>, sql: string): Promise<CachedQueryResult> {
+    return devLogSQLQuery(
+      async () =>
+        sanitizeQueryResult(
+          await Native.query(this._tag, tableName, sql),
+          this.schema.tables[tableName],
+        ),
+      tableName,
+    )
+  }
+
   count(query: SerializedQuery): Promise<number> {
     return devLogCount(() => Native.count(this._tag, encodeQuery(query, true)), query)
   }

src/adapters/type.d.ts

@@ -45,3 +45,7 @@ declare module '@nozbe/watermelondb/adapters/type' {
     unsafeClearCachedRecords(): Promise<void>
   }
 }
+
+export interface SQLiteDatabaseAdapter {
+  unsafeSqlQuery<T extends Model>(sql: string, tableName: TableName<any>): Promise<CachedQueryResult>
+}

src/adapters/type.js

@@ -50,3 +50,7 @@ export interface DatabaseAdapter {
   // Removes key from local storage
   removeLocal(key: string): Promise<void>;
 }
+
+export interface SQLDatabaseAdapter {
+  unsafeSqlQuery(tableName: TableName<any>, sql: string): Promise<CachedQueryResult>;
+}