From 66c4b5e72000dd03acb57fca1cad4737c85c9c9e Mon Sep 17 00:00:00 2001 From: David Luecke Date: Fri, 27 Jan 2023 21:08:19 -0800 Subject: [PATCH] fix(databases): Improve documentation for adapters and allow dynamic Knex adapter options (#3019) --- docs/api/databases/adapters.md | 16 +- docs/api/databases/common.md | 129 ++--- docs/api/databases/knex.md | 436 ++++++++-------- docs/api/databases/mongodb.md | 548 +++++++-------------- docs/api/databases/querying.md | 4 +- docs/guides/cli/databases.md | 2 +- docs/guides/cli/service.schemas.md | 13 + packages/adapter-tests/src/declarations.ts | 1 + packages/adapter-tests/src/methods.ts | 16 + packages/knex/src/adapter.ts | 49 +- packages/knex/src/hooks.ts | 3 +- packages/knex/test/index.test.ts | 157 ++++-- packages/mongodb/src/adapter.ts | 2 +- packages/mongodb/test/index.test.ts | 1 + 14 files changed, 614 insertions(+), 763 deletions(-) diff --git a/docs/api/databases/adapters.md b/docs/api/databases/adapters.md index 67f0d52f96..e92781e4d7 100644 --- a/docs/api/databases/adapters.md +++ b/docs/api/databases/adapters.md @@ -8,20 +8,20 @@ Feathers database adapters are modules that provide [services](../services.md) t
-[Services](../services.md) allow to implement access to _any_ database or API. The database adapters listed here are just convenience wrappers with a common API. See the community adapters section for support for other datastores. +[Services](../services.md) allow to implement access to _any_ database or API. The database adapters listed here are just convenience wrappers with a common API. See the community adapters section for support for other datastores.
## Core Adapters -The following data storage adapters are maintained alongside Feathers core. +The following data storage adapters are available in Feathers core -| Core Package | Supported Data Stores | -|---|---| -| [memory](./memory) | Memory | -| [mongodb](./mongodb) | MongoDB | -| [knex](./knex) | MySQL
MariaDB
PostgreSQL
CockroachDB
SQLite
Amazon Redshift
OracleDB
MSSQL | [feathers-knex](https://github.com/feathersjs-ecosystem/feathers-knex) | +| Core Package | Supported Data Stores | +| -------------------- | -------------------------------------------------------------------------------------------------------------- | +| [Memory](./memory) | Memory | +| [MongoDB](./mongodb) | MongoDB | +| [SQL (Knex)](./knex) | MySQL
MariaDB
PostgreSQL
CockroachDB
SQLite
Amazon Redshift
OracleDB
MSSQL | ## Community Adapters -You can find full-featured support for many more community-contributed adapters in [Awesome FeathersJS](https://github.com/feathersjs/awesome-feathersjs#database). \ No newline at end of file +There are also many community maintained adapters for other databases and ORMs which can be found on the [Ecosystem page](/ecosystem/?cat=Database&sort=lastPublish). diff --git a/docs/api/databases/common.md b/docs/api/databases/common.md index 68fc5b4153..aa24fea54b 100644 --- a/docs/api/databases/common.md +++ b/docs/api/databases/common.md @@ -25,19 +25,10 @@ app.use('/messages', new NameService()) app.use('/messages', new NameService({ id, events, paginate })) ``` -### `service([options])` - -The `service` function returns a new service instance initialized with the given options. Internally just calls `new NameService(options)` from above. - -```ts -import { service } from 'feathers-' - -app.use('/messages', service()) -app.use('/messages', service({ id, events, paginate })) -``` - ### Options +The following options are available for all database adapters: + - `id` (_optional_) - The name of the id field property (usually set by default to `id` or `_id`). - `paginate` (_optional_) - A [pagination object](#pagination) containing a `default` and `max` page size - `multi` (_optional_, default: `false`) - Allow `create` with arrays and `patch` and `remove` with id `null` to change multiple items. Can be `true` for all methods or an array of allowed methods (e.g. `[ 'remove', 'create' ]`) @@ -45,8 +36,10 @@ app.use('/messages', service({ id, events, paginate })) The following legacy options are still available but should be avoided: - `events` (_optional_, **deprecated**) - A list of [custom service events](../events.md#custom-events) sent by this service. Use the `events` option when [registering the service with app.use](../application.md#usepath-service--options) instead. -- `operators` (_optional_, **deprecated**) - A list of additional non-standard query parameters to allow (e.g `[ '$regex' ]`). Not necessary when using a [query schema validator](../schema/validators.md#validatequery) -- `filters` (_optional_, **deprecated**) - A list of top level `$` query parameters to allow (e.g. `[ '$populate' ]`). Not necessary when using a [query schema validator](../schema/validators.md#validatequery) +- `operators` (_optional_, **deprecated**) - A list of additional non-standard query parameters to allow (e.g `[ '$regex' ]`). Not necessary when using a [query schema](../schema/validators.md#validatequery) +- `filters` (_optional_, **deprecated**) - A list of top level `$` query parameters to allow (e.g. `[ '$populate' ]`). Not necessary when using a [query schema](../schema/validators.md#validatequery) + +For database specific options see the adapter documentation. ## Pagination @@ -102,75 +95,53 @@ Disabling or changing the default pagination is not available in the client. Onl -## Extending Adapters - -There are two ways to extend existing database adapters. Either by extending the base class or by adding functionality through hooks. - -### Classes - -All modules also export an [ES6 class](https://developer.mozilla.org/en/docs/Web/JavaScript/Reference/Classes) as `Service` that can be directly extended like this: - -```js -'use strict' - -const { Service } = require('feathers-') +## params.adapter -class MyService extends Service { - create(data, params) { - data.created_at = new Date() +Setting the `adapter` in the [service method `params`](../services.md#params) allows do dynamically modify the database adapter options based on the request. This e.g. allows to temporarily allow multiple entry creation/changes or the pagination settings. - return super.create(data, params) +```ts +const messages = [ + { + text: 'message 1' + }, + { + text: 'message 2' } +] - update(id, data, params) { - data.updated_at = new Date() - - return super.update(id, data, params) +// Enable multiple entry insertion for this request +app.service('messages').create(messages, { + adapter: { + multi: true } -} - -app.use( - '/todos', - new MyService({ - paginate: { - default: 2, - max: 4 - } - }) -) +}) ``` -### Hooks +
-Another option is weaving in functionality through [hooks](../hooks.md). For example, `createdAt` and `updatedAt` timestamps could be added like this: +If the adapter has a `Model` option, `params.adapter.Model` can be used to point to different databases based on the request to e.g. allow multi-tenant systems. This is usually done by setting `context.params.adapter` in a [hook](../hooks.md). -```js -const feathers = require('@feathersjs/feathers') +
-// Import the database adapter of choice -const service = require('feathers-') +## params.paginate -const app = feathers().use( - '/todos', - service({ - paginate: { - default: 2, - max: 4 - } - }) -) - -app.service('todos').hooks({ - before: { - create: [(context) => (context.data.createdAt = new Date())], +Setting `paginate` in the [service method `params`](../services.md#params) allows to change or disable the default pagination for a single request: - update: [(context) => (context.data.updatedAt = new Date())] - } +```ts +// Get all messages as an array +const allMessages = await app.service('messages').find({ + paginate: false }) - -app.listen(3030) ``` +## Extending Adapters + +There are two ways to extend existing database adapters. Either by extending the base class or by adding functionality through hooks. + +### Classes + +All modules also export an [ES6 class](https://developer.mozilla.org/en/docs/Web/JavaScript/Reference/Classes) as `Service` that can be directly extended. See the [Service CLI guide](../../guides/cli/service.class.md) on how to override existing and implement new methods. + ## Service methods This section describes specifics on how the [service methods](../services.md) are implemented for all adapters. @@ -194,30 +165,6 @@ These methods are only available internally on the server, not on the client sid -### adapter.Model - -If the ORM or database supports models, the model instance or reference to the collection belonging to this adapter can be found in `adapter.Model`. This allows to easily make custom queries using that model, e.g. in a hook: - -```js -// Make a MongoDB aggregation (`messages` is using `feathers-mongodb`) -app.service('messages').hooks({ - before: { - async find(context) { - const results = await service.Model.aggregate([ - { $match: { item_id: id } }, - { - $group: { _id: null, total_quantity: { $sum: '$quantity' } } - } - ]).toArray() - - // Do something with results - - return context - } - } -}) -``` - ### adapter.find(params) `adapter.find(params) -> Promise` returns a list of all records matching the query in `params.query` using the [common querying mechanism](./querying.md). Will either return an array with the results or a page object if [pagination is enabled](#pagination). diff --git a/docs/api/databases/knex.md b/docs/api/databases/knex.md index e4dd646596..18bb8242bd 100644 --- a/docs/api/databases/knex.md +++ b/docs/api/databases/knex.md @@ -25,167 +25,109 @@ The Knex adapter implements the [common database adapter API](./common) and [que ## API -### `service(options)` +### KnexService(options) -Returns a new service instance initialized with the given options. +`new KnexService(options)` returns a new service instance initialized with the given options. The following example extends the `KnexService` and then uses the `mongodbClient` from the app configuration and provides it to the `Model` option, which is passed to the new `MessagesService`. -```js -const knex = require('knex') -const service = require('feathers-knex') +```ts +import type { Params } from '@feathersjs/feathers' +import { KnexService } from '@feathersjs/knex' +import type { KnexAdapterParams, KnexAdapterOptions } from '@feathersjs/knex' -const db = knex({ - client: 'sqlite3', - connection: { - filename: './db.sqlite' - } -}) +import type { Application } from '../../declarations' +import type { Messages, MessagesData, MessagesQuery } from './messages.schema' -// Create the schema -db.schema.createTable('messages', (table) => { - table.increments('id') - table.string('text') -}) +export interface MessagesParams extends KnexAdapterParams {} + +export class MessagesService extends KnexService< + Messages, + MessagesData, + ServiceParams +> {} -app.use( - '/messages', - service({ - Model: db, +export const messages = (app: Application) => { + const options: KnexAdapterOptions = { + paginate: app.get('paginate'), + Model: app.get('sqliteClient'), name: 'messages' - }) -) -app.use('/messages', service({ Model, name, id, events, paginate })) + } + app.use('messages', new MessagesService(options)) +} ``` -**Options:** +### Options -- `Model` (**required**) - The KnexJS database instance -- `name` (**required**) - The name of the table -- `schema` (_optional_) - The name of the schema table prefix (example: `schema.table`) -- `id` (_optional_, default: `'id'`) - The name of the id field property. -- `events` (_optional_) - A list of [custom service events](https://docs.feathersjs.com/api/events.html#custom-events) sent by this service -- `paginate` (_optional_) - A [pagination object](https://docs.feathersjs.com/api/databases/common.html#pagination) containing a `default` and `max` page size -- `multi` (_optional_) - Allow `create` with arrays and `update` and `remove` with `id` `null` to change multiple items. Can be `true` for all methods or an array of allowed methods (e.g. `[ 'remove', 'create' ]`) -- `whitelist` (_optional_) - A list of additional query parameters to allow (e..g `[ '$regex', '$geoNear' ]`). Default is the supported `operators` +The Knex specific adapter options are: -### `adapter.createQuery(query)` - -Returns a KnexJS query with the [common filter criteria](https://docs.feathersjs.com/api/databases/querying.html) (without pagination) applied. - -### params.knex +- `Model {Knex}` (**required**) - The KnexJS database instance +- `name {string}` (**required**) - The name of the table +- `schema {string}` (_optional_) - The name of the schema table prefix (example: `schema.table`) -When making a [service method](https://docs.feathersjs.com/api/services.html) call, `params` can contain an `knex` property which allows to modify the options used to run the KnexJS query. See [customizing the query](#customizing-the-query) for an example. +The [common API options](./common.md#options) are: -## Example +- `id {string}` (_optional_, default: `'_id'`) - The name of the id field property. By design, MongoDB will always add an `_id` property. +- `events {string[]}` (_optional_) - A list of [custom service events](/api/events.html#custom-events) sent by this service +- `paginate {Object}` (_optional_) - A [pagination object](/api/databases/common.html#pagination) containing a `default` and `max` page size +- `multi {string[]|true}` (_optional_) - Allow `create` with arrays and `update` and `remove` with `id` `null` to change multiple items. Can be `true` for all methods or an array of allowed methods (e.g. `[ 'remove', 'create' ]`) -Here's a complete example of a Feathers server with a `messages` SQLite service. We are using the [Knex schema builder](http://knexjs.org/#Schema) and [SQLite](https://sqlite.org/) as the database. +### getModel([params]) -``` -$ npm install @feathersjs/feathers @feathersjs/errors @feathersjs/express @feathersjs/socketio feathers-knex knex sqlite3 -``` +`service.getModel([params])` returns the [Knex](https://knexjs.org/guide/query-builder.html) client for this table. -In `app.js`: +### db(params) -```js -const feathers = require('@feathersjs/feathers') -const express = require('@feathersjs/express') -const socketio = require('@feathersjs/socketio') +`service.db([params])` returns the Knex database instance for a request. This will include the `schema` table prefix and use a transaction if passed in `params`. -const service = require('feathers-knex') -const knex = require('knex') +### createQuery(params) -const db = knex({ - client: 'sqlite3', - connection: { - filename: './db.sqlite' - } -}) +`service.createQuery(params)` returns a query builder for a service request, including all conditions matching the query syntax. This method can be overriden to e.g. [include associations](#associations) or used in a hook customize the query and then passing it to the service call as [params.knex](#paramsknex). -// Create a feathers instance. -const app = express(feathers()) -// Turn on JSON parser for REST services -app.use(express.json()) -// Turn on URL-encoded parser for REST services -app.use(express.urlencoded({ extended: true })) -// Enable REST services -app.configure(express.rest()) -// Enable Socket.io services -app.configure(socketio()) -// Create Knex Feathers service with a default page size of 2 items -// and a maximum size of 4 -app.use( - '/messages', - service({ - Model: db, - name: 'messages', - paginate: { - default: 2, - max: 4 - } - }) -) -app.use(express.errorHandler()) - -// Clean up our data. This is optional and is here -// because of our integration tests -db.schema - .dropTableIfExists('messages') - .then(() => { - console.log('Dropped messages table') - - // Initialize your table - return db.schema.createTable('messages', (table) => { - console.log('Creating messages table') - table.increments('id') - table.string('text') - }) - }) - .then(() => { - // Create a dummy Message - app - .service('messages') - .create({ - text: 'Message created on server' - }) - .then((message) => console.log('Created message', message)) - }) +```ts +app.service('messages').hooks({ + before: { + find: [ + async (context: HookContext) => { + const query = context.service.createQuery(context.params) -// Start the server. -const port = 3030 + // do something with query here + query.orderBy('name', 'desc') -app.listen(port, () => { - console.log(`Feathers server listening on port ${port}`) + context.params.knex = query + } + ] + } }) ``` -Run the example with `node app` and go to localhost:3030/messages - -## Querying - -In addition to the [common querying mechanism](https://docs.feathersjs.com/api/databases/querying.html), this adapter also supports: - -### $and +### params.knex -Find all records that match all of the given criteria. The following query retrieves all messages that have foo and bar attributes as true. +When making a [service method](https://docs.feathersjs.com/api/services.html) call, `params` can contain an `knex` property which allows to modify the options used to run the KnexJS query. See [createQuery](#createqueryparams) for an example. -```js -app.service('messages').find({ - query: { - $and: [{ foo: true }, { bar: true }] - } -}) -``` +## Querying -Through the REST API: +In addition to the [common querying mechanism](./querying.md), this adapter also supports the following operators. Note that these operators need to be added for each query-able property to the [TypeBox query schema](../schema/typebox.md#query-schemas) or [JSON query schema](../schema/schema.md#querysyntax) like this: -``` -/messages?$and[][foo]=true&$and[][bar]=true +```ts +const messageQuerySchema = Type.Intersect( + [ + // This will additionally allow querying for `{ name: { $ilike: 'Dav%' } }` + querySyntax(messageQueryProperties, { + name: { + $ilike: Type.String() + } + }), + // Add additional query properties here + Type.Object({}) + ], + { additionalProperties: false } +) ``` ### $like Find all records where the value matches the given string pattern. The following query retrieves all messages that start with `Hello`: -```js +```ts app.service('messages').find({ query: { text: { @@ -205,7 +147,7 @@ Through the REST API: The opposite of `$like`; resulting in an SQL condition similar to this: `WHERE some_field NOT LIKE 'X'` -```js +```ts app.service('messages').find({ query: { text: { @@ -225,7 +167,7 @@ Through the REST API: For PostgreSQL only, the keywork $ilike can be used instead of $like to make the match case insensitive. The following query retrieves all messages that start with `hello` (case insensitive): -```js +```ts app.service('messages').find({ query: { text: { @@ -241,144 +183,166 @@ Through the REST API: /messages?text[$ilike]=hello% ``` -## Transaction Support - -The Knex adapter comes with three hooks that allows to run service method calls in a transaction. They can be used as application wide (`app.hooks.js`) hooks or per service like this: +## Search -```javascript -// A common hooks file -const { hooks } = require('feathers-knex') +Basic search can be implemented with the [query operators](#querying). -const { transaction } = hooks - -module.exports = { - before: { - all: [transaction.start()], - find: [], - get: [], - create: [], - update: [], - patch: [], - remove: [] - }, - - after: { - all: [transaction.end()], - find: [], - get: [], - create: [], - update: [], - patch: [], - remove: [] - }, - - error: { - all: [transaction.rollback()], - find: [], - get: [], - create: [], - update: [], - patch: [], - remove: [] - } -} -``` +## Associations -To use the transactions feature, you must ensure that the three hooks (start, end and rollback) are being used. +While [resolvers](../schema/resolvers.md) offer a reasonably performant way to fetch associated entities, it is also possible to join tables to populate and query related data. This can be done by overriding the [createQuery](#createqueryparams) method and using the [Knex join methods](https://knexjs.org/guide/query-builder.html#join) to join the tables of related services. -At the start of any request, a new transaction will be started. All the changes made during the request to the services that are using the `feathers-knex` will use the transaction. At the end of the request, if sucessful, the changes will be commited. If an error occurs, the changes will be forfeit, all the `creates`, `patches`, `updates` and `deletes` are not going to be commited. +### Querying -The object that contains `transaction` is stored in the `params.transaction` of each request. +Considering a table like this: -> **Important:** If you call another Knex service within a hook and want to share the transaction you will have to pass `context.params.transaction` in the parameters of the service call. +```ts +await db.schema.createTable('todos', (table) => { + table.increments('id') + table.string('text') + table.bigInteger('personId').references('id').inTable('people').notNullable() + return table +}) +``` -## Customizing the query +To query based on properties from the `people` table, join the tables you need in `createQuery` like this: -In a `find` call, `params.knex` can be passed a KnexJS query (without pagination) to customize the find results. +```ts +class TodoService> extends KnexService { + createQuery(params: KnexAdapterParams) { + const query = super.createQuery(params) -Combined with `.createQuery({ query: {...} })`, which returns a new KnexJS query with the [common filter criteria](https://docs.feathersjs.com/api/databases/querying.html) applied, this can be used to create more complex queries. The best way to customize the query is in a [before hook](https://docs.feathersjs.com/api/hooks.html) for `find`. + query.join('people as person', 'todos.personId', 'person.id') -```js -app.service('messages').hooks({ - before: { - find(context) { - const query = context.service.createQuery(context.params) + return query + } +} +``` - // do something with query here - query.orderBy('name', 'desc') +This will alias the table name from `people` to `person` (since our Todo only has a single person) and then allow to query all related properties as dot separated properties like `person.name`, including the [Feathers query syntax](./querying.md): - context.params.knex = query - return context - } +```ts +// Find the Todos for all Daves older than 100 +app.service('todos').find({ + query: { + 'person.name': 'Dave', + 'person.age': { $gt: 100 } } }) ``` -## Configuring migrations +Note that in most applications, the query-able properties have to explicitly be added to the [TypeBox query schema](../schema/typebox.md#query-schemas) or [JSON query schema](../schema/schema.md#querysyntax). Support for the query syntax for a single property can be added with the `queryProperty` helper: -For using knex's migration CLI, we need to make the configuration available by the CLI. We can do that by providing a `knexfile.js` (OR `knexfile.ts` when using TypeScript) in the root folder with the following contents: - -knexfile.js - -```js -const app = require('./src/app') -module.exports = app.get('postgres') +```ts +import { queryProperty } from '@feathersjs/typebox' + +export const todoQueryProperties = Type.Pick(userSchema, ['text']) +export const todoQuerySchema = Type.Intersect( + [ + querySyntax(userQueryProperties), + // Add additional query properties here + Type.Object( + { + // Only query the name for strings + 'person.name': Type.String(), + // Support the query syntax for the age + 'person.age': queryProperty(Type.Number()) + }, + { additionalProperties: false } + ) + ], + { additionalProperties: false } +) ``` -OR +### Populating -knexfile.ts +Related properties from the joined table can be added as aliased properties with [query.select](https://knexjs.org/guide/query-builder.html#select): ```ts -import app from './src/app' -module.exports = app.get('postgres') +class TodoService> extends KnexService { + createQuery(params: KnexAdapterParams) { + const query = super.createQuery(params) + + query + .join('people as person', 'todos.personId', 'person.id') + // This will add a `personName` property + .select('person.name as personName') + // This will add a `person.age' property + .select('person.age') + + return query + } +} ``` -You will need to replace the `postgres` part with the adapter you are using. You will also need to add a `migrations` key to your feathersjs config under your database adapter. Optionally, add a `seeds` key if you will be using [seeds](http://knexjs.org/#Seeds-CLI). +
+ +Since SQL does not have a concept of nested objects, joined properties will be dot separated strings, **not nested objects**. Conversion can be done by e.g. using Lodash `_.set` in a [resolver converter](../schema/resolvers.md#options). + +
+ +This works well for individual properties, however if you require the complete (and safe) representation of the entire related data, use a [resolver](../schema/resolvers.md) instead. -```js -// src/config/default.json -... - "postgres": { - "client": "pg", - "connection": "postgres://user:password@localhost:5432/database", - "migrations": { - "tableName": "knex_migrations" +## Transactions + +The Knex adapter comes with three hooks that allows to run service method calls in a transaction. They can be used as application wide hooks or per service like this: + +```ts +import { transaction } from '@feathersjs/knex' + +// A configure function that registers the service and its hooks via `app.configure` +export const message = (app: Application) => { + // Register our service on the Feathers application + app.use('messages', new MessageService(getOptions(app)), { + // A list of all methods this service exposes externally + methods: ['find', 'get', 'create', 'patch', 'remove'], + // You can add additional custom events to be sent to clients here + events: [] + }) + // Initialize hooks + app.service('messages').hooks({ + around: { + all: [] }, - "seeds": { - "directory": "../src/seeds" + before: { + all: [transaction.start()], + find: [], + get: [], + create: [], + patch: [], + remove: [] + }, + after: { + all: [transaction.end()] + }, + error: { + all: [transaction.rollback()] } - } + }) +} ``` -Then, by running: `knex migrate:make create-users`, a `migrations` directory will be created, with the new migration. +To use the transactions feature, you must ensure that the three hooks (start, end and rollback) are being used. -### Error handling +At the start of any request, a new transaction will be started. All the changes made during the request to the services that are using knex will use the transaction. At the end of the request, if sucessful, the changes will be commited. If an error occurs, the changes will be forfeit, all the `creates`, `patches`, `updates` and `deletes` are not going to be commited. -As of version 4.0.0 `feathers-knex` only throws [Feathers Errors](https://docs.feathersjs.com/api/errors.html) with the message. On the server, the original error can be retrieved through a secure symbol via `error[require('feathers-knex').ERROR]` +The object that contains `transaction` is stored in the `params.transaction` of each request. -```js -const { ERROR } = require('feathers-knex') +
-try { - await knexService.doSomething() -} catch (error) { - // error is a FeathersError with just the message - // Safely retrieve the Knex error - const knexError = error[ERROR] -} -``` +If you call another Knex service within a hook and want to share the transaction you will have to pass `context.params.transaction` in the parameters of the service call. -### Waiting for transactions to complete +
Sometimes it can be important to know when the transaction has been completed (committed or rolled back). For example, we might want to wait for transaction to complete before we send out any realtime events. This can be done by awaiting on the `transaction.committed` promise which will always resolve to either `true` in case the transaction has been committed, or `false` in case the transaction has been rejected. -```js -app.service('messages').publish((data, context) => { +```ts +app.service('messages').publish(async (data, context) => { const { transaction } = context.params if (transaction) { const success = await transaction.committed + if (!success) { return [] } @@ -389,3 +353,23 @@ app.service('messages').publish((data, context) => { ``` This also works with nested service calls and nested transactions. For example, if a service calls `transaction.start()` and passes the transaction param to a nested service call, which also calls `transaction.start()` in it's own hooks, they will share the top most `committed` promise that will resolve once all of the transactions have succesfully committed. + +## Error handling + +The adapter only throws [Feathers Errors](https://docs.feathersjs.com/api/errors.html) with the message to not leak sensitive information to a client. On the server, the original error can be retrieved through a secure symbol via `import { ERROR } from '@feathersjs/knex'` + +```ts +import { ERROR } from 'feathers-knex' + +try { + await knexService.doSomething() +} catch (error: any) { + // error is a FeathersError with just the message + // Safely retrieve the Knex error + const knexError = error[ERROR] +} +``` + +## Migrations + +In a generated application, migrations are already set up. See the [CLI guide](../../guides/cli/knexfile.md) and the [KnexJS migrations documentation](https://knexjs.org/guide/migrations.html) for more information. diff --git a/docs/api/databases/mongodb.md b/docs/api/databases/mongodb.md index 2b43e75777..c293ae60e2 100644 --- a/docs/api/databases/mongodb.md +++ b/docs/api/databases/mongodb.md @@ -23,89 +23,11 @@ The MongoDB adapter implements the [common database adapter API](./common) and [ -## Setup - -There are two typical setup steps for using `@feathersjs/mongodb` in an application: - -- connect to the database and -- setup schemas for types and validation. - -### Connect to the Database - -Before using `@feathersjs/mongodb`, you'll need to create a connection to the database. This example connects to a MongoDB database similar to how the CLI-generated app connects. It uses `app.get('mongodb')` to read the connection string from `@feathersjs/configuration`. The connection string would be something similar to `mongodb://127.0.0.1:27017/my-app-dev` for local development or one provided by your database host. - -Once the connection attempt has been started, the code uses `app.set('monodbClient', mongoClient)` to store the connection promise back into the config, which allows it to be looked up when initializing individual services. - -```ts -import { MongoClient } from 'mongodb' -import type { Db } from 'mongodb' -import type { Application } from './declarations' - -export const mongodb = (app: Application) => { - const connection = app.get('mongodb') - const database = new URL(connection).pathname.substring(1) - const mongoClient = MongoClient.connect(connection).then((client) => client.db(database)) - app.set('mongodbClient', mongoClient) -} - -declare module './declarations' { - interface Configuration { - mongodbClient: Promise - } -} -``` - -### Setup the Schema & Types - -To take full advantage of the new TypeScript features in Feathers v5, we can create schema for our service's data types. This example shows how to use `@feathersjs/typebox` to create schemas and types for data and query types. This is the same as generated by the CLI, but the resolvers have been removed for brevity. - -```ts -import { Type, querySyntax } from '@feathersjs/typebox' -import type { Static } from '@feathersjs/typebox' - -// Main data model schema -export const messagesSchema = Type.Object( - { - _id: Type.String(), - text: Type.String() - }, - { $id: 'Messages', additionalProperties: false } -) -export type Messages = Static - -// Schema for creating new entries -export const messagesDataSchema = Type.Pick(messagesSchema, ['name'], { - $id: 'MessagesData', - additionalProperties: false -}) -export type MessagesData = Static - -// Schema for allowed query properties -export const messagesQueryProperties = Type.Pick(messagesSchema, ['_id', 'name'], { - additionalProperties: false -}) -export const messagesQuerySchema = querySyntax(messagesQueryProperties) -export type MessagesQuery = Static -``` - -### Schemas vs MongoDB Validation - -In Feathers v5 (Dove) we added support for Feathers Schema, which performs validation and provides TypeScript types. Recent versions of MongoDB include support for JSON Schema validation at the database server. Most applications will benefit from using Feathers Schema for the following reasons. - -- Feathers Schema's TypeBox integration makes JSON Schema so much easier to read and write. -- You get TypeScript types for free once you've defined your validation rules, using `TypeBox` or `json-schema-to-ts` -- All configuration is done in code, reducing the time to prototype/setup/launch. With MongoDB's built-in validation, you essentially add another "DevOps" step before you can use the database. -- Support for JSON Schema draft 7. MongoDB's validation is based on version 4. -- Feathers Schema don't have to wait for a round-trip to the database to validate the data. -- Feathers Schema can be used in the browser or on the server. - -MongoDB's built-in validation does have built-in support for `bsonType` to force data to be stored as a specific BSON type once it passes validation. There's nothing keeping you from using both solutions together. It's not a use case that's documented, here. - ## API -### `service(options)` +### `MongoDBService(options)` -Returns a new service instance initialized with the given options. The following example extends the `MongoDBService` class using the schema examples from earlier on this page. It then uses the `mongodbClient` from the app configuration and provides it to the `Model` option, which is passed to the new `MessagesService`. +`new MongoDBService(options)` returns a new service instance initialized with the given options. The following example extends the `MongoDBService` and then uses the `mongodbClient` from the app configuration and provides it to the `Model` option, which is passed to the new `MessagesService`. ```ts import type { Params } from '@feathersjs/feathers' @@ -128,52 +50,149 @@ export const messages = (app: Application) => { paginate: app.get('paginate'), Model: app.get('mongodbClient').then((db) => db.collection('messages')) } - app.use('messages', new MessagesService(options), { - methods: ['find', 'get', 'create', 'update', 'patch', 'remove'], - events: [] - }) + app.use('messages', new MessagesService(options)) } ``` Here's an overview of the `options` object: -**Options:** +### Options + +MongoDB adapter specific options are: + +- `Model {Promise}` (**required**) - A Promise that resolves with the MongoDB collection instance. This can also be the return value of an `async` function without `await` +- `disableObjectify {boolean}` (_optional_, default `false`) - This will disable conversion of the id field to a MongoDB ObjectID if you want to e.g. use normal strings +- `useEstimatedDocumentCount {boolean}` (_optional_, default `false`) - If `true` document counting will rely on `estimatedDocumentCount` instead of `countDocuments` + +The [common API options](./common.md#options) are: -- `Model {Promise}` (**required**) - The MongoDB collection instance - `id {string}` (_optional_, default: `'_id'`) - The name of the id field property. By design, MongoDB will always add an `_id` property. -- `disableObjectify {boolean}` (_optional_, default `false`) - This will disable the objectify of the id field if you want to use normal strings - `events {string[]}` (_optional_) - A list of [custom service events](/api/events.html#custom-events) sent by this service - `paginate {Object}` (_optional_) - A [pagination object](/api/databases/common.html#pagination) containing a `default` and `max` page size -- `filters {Object}` (_optional_) - An object of additional filter parameters to allow (e..g `{ $customQueryOperator: true }`). See [Filters](/api/databases/querying.md#filters) -- `operators {string[]}` (_optional_) - A list of additional query parameters to allow (e..g `[ '$regex', '$geoNear' ]`) See [Operators](/api/databases/querying.md#operators) - `multi {string[]|true}` (_optional_) - Allow `create` with arrays and `update` and `remove` with `id` `null` to change multiple items. Can be `true` for all methods or an array of allowed methods (e.g. `[ 'remove', 'create' ]`) -- `useEstimatedDocumentCount {boolean}` (_optional_, default `false`) - If `true` document counting will rely on `estimatedDocumentCount` instead of `countDocuments` -### `aggregateRaw(params)` +### getModel() + +`getModel([params])` returns a Promise that resolves with the MongoDB collection object. The optional `params` is the service parameters which may allow to override the collection via [params.adapter](./common.md#paramsadapter). + +### aggregateRaw(params) The `find` method has been split into separate utilities for converting params into different types of MongoDB requests. By default, requests are processed by this method and are run through the MongoDB Aggregation Pipeline. This method returns a raw MongoDB Cursor object, which can be used to perform custom pagination or in custom server scripts, if desired. -### `findRaw(params)` +### findRaw(params) -The `find` method has been split into separate utilities for converting params into different types of MongoDB requests. When `params.mongodb` is used, the `findRaw` method is used to retrieve data using `params.mongodb` as the `FindOptions` object. This method returns a raw MongoDB Cursor object, which can be used to perform custom pagination or in custom server scripts, if desired. +`findRaw(params)` is used when `params.mongodb` is set to retrieve data using `params.mongodb` as the `FindOptions` object. This method returns a raw MongoDB Cursor object, which can be used to perform custom pagination or in custom server scripts, if desired. -### `makeFeathersPipeline(params)` +### makeFeathersPipeline(params) -`makeFeathersPipeline` takes a set of Feathers params and converts them to a pipeline array, ready to pass to `collection.aggregate`. This utility comprises the bulk of the `aggregateRaw` functionality, but does not use `params.pipeline`. +`makeFeathersPipeline(params)` takes a set of Feathers params and converts them to a pipeline array, ready to pass to `collection.aggregate`. This utility comprises the bulk of the `aggregateRaw` functionality, but does not use `params.pipeline`. ### Custom Params The `@feathersjs/mongodb` adapter utilizes two custom params which control adapter-specific features: `params.pipeline` and `params.mongodb`. +#### params.adapter + +Allows to dynamically set the [adapter options](#options) (like the `Model` collection) for a service method call. + #### params.pipeline -This is a test +Used for [aggregation pipelines](#aggregation-pipeline). #### params.mongodb -When making a [service method](/api/services.md) call, `params` can contain an `mongodb` property (for example, `{upsert: true}`) which allows modifying the options used to run the MongoDB query. +When making a [service method](/api/services.md) call, `params` can contain an`mongodb` property (for example, `{upsert: true}`) which allows modifying the options used to run the MongoDB query. The adapter will use the `collection.find` method and not the [aggregation pipeline](#aggregation-pipeline) when you use `params.mongodb`. + +## Transactions -The adapter will automatically switch to use the MongoClient's`collection.find` method when you use `params.mongodb`. +[MongoDB Transactions](https://docs.mongodb.com/manual/core/transactions/) can be used by passing a `session` in [params.mongodb](#paramsmongodb). For example in a [hook](../hooks.md): + +```ts +import { ObjectId } from 'mongodb' +import { HookContext } from '../declarations' + +export const myHook = async (context: HookContext) => { + const { app } = context + const session = app.get('mongoClient').startSession() + + try { + await session.withTransaction(async () => { + const fooData = { message: 'Data for foo' } + const barData = { text: 'Data for bar' } + + await app.service('fooService').create(fooData, { + mongodb: { session } + }) + await app.service('barService').create(barData, { + mongodb: { session } + }) + }) + } finally { + await session.endSession() + } +} +``` + +## Indexes + +Indexes and unique constraints can be added to the `Model` Promise, usually in the `getOptions` in `.class`: + +```ts +export const getOptions = (app: Application): MongoDBAdapterOptions => { + return { + paginate: app.get('paginate'), + Model: app + .get('mongodbClient') + .then((db) => db.collection('myservice')) + .then((collection) => { + collection.createIndex({ email: 1 }, { unique: true }) + + return collection + }) + } +} +``` + +
+ +Note that creating indexes for an existing collection with many entries should be done as a separate operation instead. See the [MongoDB createIndex documentation](https://www.mongodb.com/docs/manual/reference/method/db.collection.createIndex/) for more information. + +
+ +## Querying + +Additionally to the [common querying mechanism](./querying.md) this adapter also supports [MongoDB's query syntax](https://www.mongodb.com/docs/manual/tutorial/query-documents/) and the `update` method also supports MongoDB [update operators](https://www.mongodb.com/docs/manual/reference/operator/update/). + +## Search + +
+ +Note that in a normal application all MongoDB specific operators have to explicitly be added to the [TypeBox query schema](../schema/typebox.md#query-schemas) or [JSON query schema](../schema/schema.md#querysyntax). + +
+ +There are two ways to perform search queries with MongoDB: + +- Perform basic Regular Expression matches using the `$regex` filter. +- Perform full-text search using the `$search` filter. + +### Basic Regex Search + +You can perform basic search using regular expressions with the `$regex` operator. Here's an example query. + +```js +{ + text: { $regex: 'feathersjs', $options: 'igm' }, +} +``` + +### Full-Text Search + +See the MongoDB documentation for instructions on performing full-text search using the `$search` operator: + +- Perform [full-text queries on self-hosted MongoDB](https://www.mongodb.com/docs/manual/core/link-text-indexes/). +- Perform [full-text queries on MongoDB Atlas](https://www.mongodb.com/docs/atlas/atlas-search/) (MongoDB's first-party hosted database). +- Perform [full-text queries with the MongoDB Pipeline](https://www.mongodb.com/docs/manual/tutorial/text-search-in-aggregation/) ## Aggregation Pipeline @@ -343,192 +362,102 @@ Both examples look a bit complex, but te one using aggregation stages will be mu One more obstacle for using JavaScript this way is that if the user's query changed (from the front end), we would likely be required to edit multiple different parts of the JS logic in order to correctly display results. With the pipeline example, above, the query is very cleanly applied. -## Transactions - -You can utilize [MongoDB Transactions](https://docs.mongodb.com/manual/core/transactions/) by passing a `session` with the `params.mongodb`: - -```js -import { ObjectID } from 'mongodb' - -export default async app => { - app.use('/fooBarService', { - async create(data) { - // assumes you have access to the mongoClient via your app state - let session = app.mongoClient.startSession() - try { - await session.withTransaction(async () => { - let fooID = new ObjectID() - let barID = new ObjectID() - app.service('fooService').create( - { - ...data, - _id: fooID, - bar: barID, - }, - { mongodb: { session } }, - ) - app.service('barService').create( - { - ...data, - _id: barID - foo: fooID - }, - { mongodb: { session } }, - ) - }) - } finally { - await session.endSession() - } - } - }) -} -``` - ## Collation This adapter includes support for [collation and case insensitive indexes available in MongoDB v3.4](https://docs.mongodb.com/manual/release-notes/3.4/#collation-and-case-insensitive-indexes). Collation parameters may be passed using the special `collation` parameter to the `find()`, `remove()` and `patch()` methods. -### Example: Patch records with case-insensitive alphabetical ordering +**Example: Patch records with case-insensitive alphabetical ordering** The example below would patch all student records with grades of `'c'` or `'C'` and above (a natural language ordering). Without collations this would not be as simple, since the comparison `{ $gt: 'c' }` would not include uppercase grades of `'C'` because the code point of `'C'` is less than that of `'c'`. -```js -const patch = { shouldStudyMore: true }; -const query = { grade: { $gte: 'c' } }; -const collation = { locale: 'en', strength: 1 }; -students.patch(null, patch, { query, collation }).then( ... ); +```ts +const patch = { shouldStudyMore: true } +const query = { grade: { $gte: 'c' } } +const collation = { locale: 'en', strength: 1 } +const patchedStudent = await students.patch(null, patch, { query, collation }) ``` -### Example: Find records with a case-insensitive search +**Example: Find records with a case-insensitive search** Similar to the above example, this would find students with a grade of `'c'` or greater, in a case-insensitive manner. -```js -const query = { grade: { $gte: 'c' } }; -const collation = { locale: 'en', strength: 1 }; -students.find({ query, collation }).then( ... ); +```ts +const query = { grade: { $gte: 'c' } } +const collation = { locale: 'en', strength: 1 } + +const collatedStudents = await students.find({ query, collation }) ``` For more information on MongoDB's collation feature, visit the [collation reference page](https://docs.mongodb.com/manual/reference/collation/). -## Querying - -Additionally to the [common querying mechanism](https://docs.feathersjs.com/api/databases/querying.html) this adapter also supports [MongoDB's query syntax](https://docs.mongodb.com/v3.2/tutorial/query-documents/) and the `update` method also supports MongoDB [update operators](https://docs.mongodb.com/v3.2/reference/operator/update/). - -> **Important:** External query values through HTTP URLs may have to be converted to the same type stored in MongoDB in a before [hook](https://docs.feathersjs.com/api/hooks.html) otherwise no matches will be found. Websocket requests will maintain the correct format if it is supported by JSON (ObjectIDs and dates still have to be converted). - -For example, an `age` (which is a number) a hook like this can be used: +## ObjectIds -```js -const ObjectID = require('mongodb').ObjectID - -app.service('users').hooks({ - before: { - find(context) { - const { query = {} } = context.params +MongoDB uses [ObjectId](https://www.mongodb.com/docs/manual/reference/method/ObjectId/) object as primary keys. To store them in the right format they have to be converted from and to strings. - if (query.age !== undefined) { - query.age = parseInt(query.age, 10) - } +### AJV format - context.params.query = query +To validate an ObjectId via the `format` keyword, add the following to your `validators` file: - return Promise.resolve(context) +```ts +// `objectid` formatter +const formatObjectId = { + type: 'string', + validate: (id: string | ObjectId) => { + if (ObjectId.isValid(id)) { + if (String(new ObjectId(id)) === id) return true + return false } + return false } -}) -``` - -Which will allows queries like `/users?_id=507f1f77bcf86cd799439011&age=25`. - -## Validate Data - -There are two ways - -Since MongoDB uses special binary types for stored data, Feathers uses Schemas to validate MongoDB data and resolvers to convert types. Attributes on MongoDB services often require specitying two schema formats: - -- Object-type formats for data pulled from the database and passed between services on the API server. The MongoDB driver for Node converts binary data into objects, like `ObjectId` and `Date`. -- String-type formats for data passed from the client. - -You can convert values to their binary types to take advantage of performance enhancements in MongoDB. The following sections show how to use Schemas and Resolvers for each MongoDB binary format. - -### Shared Validators +} as const -The following example shows how to create two custom validator/type utilities: +dataValidator.addFormat('objectid', formatObjectId) +queryValidator.addFormat('objectid', formatObjectId) +``` -- An ObjectId validator which handles strings or objects. Place the following code inside `src/schema/shared.ts`. -- A Date validator +Now you can use it as a `format` option in your schema definitions: ```ts -import { Type } from '@feathersjs/typebox' - -export const ObjectId = () => Type.Union([ - Type.String({ format: 'objectid' }), - Type.Object({}, { additionalProperties: true }), -]) -export const NullableObjectId = () => Type.Union([ObjectId(), Type.Null()]) - -export const Date = () => Type.Union([ - Type.String({ format: 'date-time' }), - Type.Date(), -]) +// TypeBox +Type.String({ format: 'objectid' }) +// JSON schema +const schema = { + type: 'string', + format: 'objectid' +} ``` -Technically, `ObjectId` in the above example could be improved since it allows any object to pass validation. A query with a bogus id fail only when it reaches the database. Since objects can't be passed from the client, it's a situation which only occurs with our server code, so it will suffice. - -### ObjectIds +### Shared Validator -With the [shared validators](#shared-validators) in place, you can specify an `ObjectId` type in your TypeBox Schemas: +The following utility can be used to create a shared [TypeBox](../schema/typebox.md) type for object ids (e.g. in a `utilities` file): ```ts import { Type } from '@feathersjs/typebox' -import { ObjectId } from '../../schemas/shared' -const userSchema = Type.Object( - { - _id: ObjectId(), - orgIds: Type.Array( ObjectId() ), - }, - { $id: 'User', additionalProperties: false } -) +export const ObjectId = () => + Type.Union([Type.String({ format: 'objectid' }), Type.Object({}, { additionalProperties: true })]) ``` -### Dates - -The standard format for transmitting dates between client and server is [ISO8601](https://www.rfc-editor.org/rfc/rfc3339#section-5.6), which is a string representation of a date. - -While it's possible to use `$gt` (greater than) and `$lt` (less than) queries on string values, performance will be faster for dates stored as date objects. This means you'd use the same code as the previous example, followed by a resolver or a hook to convert the values in `context.data` and `context.query` into actual Dates. - -With the [shared validators](#shared-validators) in place, you can use the custom `Date` type in your TypeBox Schemas: +Which can then be used like this: ```ts import { Type } from '@feathersjs/typebox' -import { ObjectId, Date } from '../../schemas/shared' +import { ObjectId } from '../utilities' const userSchema = Type.Object( { _id: ObjectId(), - createdAt: Date(), + orgIds: Type.Array(ObjectId()) }, { $id: 'User', additionalProperties: false } ) ``` -## Convert Data - -It's possible to convert data by either customizing AJV or using resolvers. Converting with AJV is currently the simplest solution, but it uses extended AJV features, outside of the JSON Schema standard. - -### Convert With AJV - -The FeathersJS CLI creates validators in the `src/schema/validators` file. It's possible to enable the AJV validators to convert certain values using AJV keywords. This example works for both [TypeBox](/api/schema/typebox) and [JSON Schema](/api/schema/schema): +### AJV converter -#### Create Converters +To convert Object Ids, one option is to register an AJV specific converter by adding the following to the the `validators` file: ```ts -import type { AnySchemaObject } from 'ajv' -import { Ajv } from '@feathersjs/schema' -import { ObjectId } from 'mongodb' - // `convert` keyword. const keywordConvert = { keyword: 'convert', @@ -536,16 +465,7 @@ const keywordConvert = { compile(schemaVal: boolean, parentSchema: AnySchemaObject) { if (!schemaVal) return () => true - // Convert date-time string to Date - if (['date-time', 'date'].includes(parentSchema.format)) { - return function (value: string, obj: any) { - const { parentData, parentDataProperty } = obj - parentData[parentDataProperty] = new Date(value) - return true - } - } - // Convert objectid string to ObjectId - else if (parentSchema.format === 'objectid') { + if (parentSchema.format === 'objectid') { return function (value: string, obj: any) { const { parentData, parentDataProperty } = obj parentData[parentDataProperty] = new ObjectId(value) @@ -553,94 +473,26 @@ const keywordConvert = { } } return () => true - }, -} as const - -// `objectid` formatter -const formatObjectId = { - type: 'string', - validate: (id: string | ObjectId) => { - if (ObjectId.isValid(id)) { - if (String(new ObjectId(id)) === id) return true - return false - } - return false - }, + } } as const -export function addConverters(validator: Ajv) { - validator.addKeyword(keywordConvert) - validator.addFormat('objectid', formatObjectId) -} +dataValidator.addKeyword(keywordConvert) +queryValidator.addKeyword(keywordConvert) ``` -#### Apply to AJV - -You can then add converters to the generated validators by importing the `addConverters` utility in the `validators.ts` file: - -```ts{3,30-32} -import { Ajv, addFormats } from '@feathersjs/schema' -import type { FormatsPluginOptions } from '@feathersjs/schema' -import { addConverters } from './converters' - -const formats: FormatsPluginOptions = [ - 'date-time', - 'time', - 'date', - 'email', - 'hostname', - 'ipv4', - 'ipv6', - 'uri', - 'uri-reference', - 'uuid', - 'uri-template', - 'json-pointer', - 'relative-json-pointer', - 'regex', -] - -export const dataValidator = addFormats(new Ajv({}), formats) +And then update the shared `ObjectId` in `utilities`: -export const queryValidator = addFormats( - new Ajv({ - coerceTypes: true, - }), - formats, -) - -addConverters(dataValidator) -addConverters(queryValidator) -``` - -#### Update Shared Validators - -We can now update the [shared validators](#shared-validators) to also convert data. - -```ts{5} -import { Type } from '@feathersjs/typebox' - -export const ObjectId = () => Type.Union([ - Type.String({ format: 'objectid', convert: true }), - Type.Object({}, { additionalProperties: true }), -]) -export const NullableObjectId = () => Type.Union([ObjectId(), Type.Null()]) - -export const Date = () => Type.Union([ - Type.String({ format: 'date-time', convert: true }), - Type.Date(), -]) +```ts +export const ObjectId = () => + Type.Union([ + Type.String({ format: 'objectid', convert: true }), + Type.Object({}, { additionalProperties: true }) + ]) ``` -Now when we use the shared validators (as shown in the [ObjectIds](#objectids) and [Dates](#dates) sections) AJV will also convert the data to the correct type. +### ObjectId resolvers -### Convert with Resolvers - -In place of [converting data with AJV](#convert-with-ajv), you can also use resolvers to convert data. - -#### ObjectId Resolvers - -MongoDB uses object ids as primary keys and for references to other documents. To a client they are represented as strings and to convert between strings and object ids, the following [property resolver](../schema/resolvers.md) helpers can be used. +While the AJV format checks if an object id is valid, it still needs to be converted to the right type. An alternative the the [AJV converter](#ajv-converter) is to use [Feathers resolvers](../schema/resolvers.md). The following [property resolver](../schema/resolvers.md) helpers can be used. #### resolveObjectId @@ -683,48 +535,10 @@ export const messageQueryResolver = resolve({ }) ``` -## Common Mistakes - -Here are a couple of errors you might run into while using validators. - -### unknown keyword: "convert" - -You'll see an error like `"Error: strict mode: unknown keyword: "convert"` in a few scenarios: +## Dates -- You fail to [Pass the Custom AJV Instance to every `schema`](#pass-the-custom-ajv-instance-to-schema). If you're using a custom AJV instance, be sure to provide it to **every** place where you call `schema()`. -- You try to use custom keywords in your schema without registering them, first. -- You make a typo in your schema. For example, it's common to forget to accidentally mis-document arrays and collapse the item `properties` up one level. +While MongoDB has a native `Date` type, the most reliable way to deal with dates is to send and store them as UTC millisecond timestamps e.g. returned by [Date.now()](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Date/now) or [new Date().getTime()](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Date/getTime) which is also used in the [Feathers getting started guide](../../guides/basics/generator.md). This has a few advantages: -### unknown format "date-time" - -You'll see an error like `Error: unknown format "date-time" ignored in schema at path "#/properties/createdAt"` in a few scenarios. - -- You're attempting to use a formatter not built into AJV. -- You fail to [Pass the Custom AJV Instance to every `schema`](#pass-the-custom-ajv-instance-to-schema). If you're using a custom AJV instance, be sure to provide it to **every** place where you call `schema()`. - -## Search - -There are two ways to perform search queries with MongoDB: - -- Perform basic Regular Expression matches using the `$regex` filter. -- Perform full-text search using the `$search` filter. - -### Basic Regex Search - -You can perform basic search using regular expressions with the `$regex` operator. Here's an example query. - -```js -{ - text: { $regex: 'feathersjs', $options: 'igm' }, -} -``` - -TODO: Show how to customize the query syntax to allow the `$regex` and `$options` operators. - -### Full-Text Search - -See the MongoDB documentation for instructions on performing full-text search using the `$search` operator: - -- Perform [full-text queries on self-hosted MongoDB](https://www.mongodb.com/docs/manual/core/link-text-indexes/). -- Perform [full-text queries on MongoDB Atlas](https://www.mongodb.com/docs/atlas/atlas-search/) (MongoDB's first-party hosted database). -- Perform [full-text queries with the MongoDB Pipeline](https://www.mongodb.com/docs/manual/tutorial/text-search-in-aggregation/) +- No conversion between different string types +- No timezone and winter/summer time issues +- Easier calculations and query-ability diff --git a/docs/api/databases/querying.md b/docs/api/databases/querying.md index 4cafebb5c4..86250887c2 100644 --- a/docs/api/databases/querying.md +++ b/docs/api/databases/querying.md @@ -123,7 +123,7 @@ GET /messages?$or[0][archived][$ne]=true&$or[1][roomId]=2 ## Operators -Operators either query a property for a specific value or determine nested special properties (starting with a `$`) that allow querying the property for certain conditions. When multiple operators are set, conditions have to apply for a property to match. +Operators either query a property for a specific value or determine nested special properties (starting with a `$`) that allow querying the property for certain conditions. When multiple operators are set, all conditions have to apply for a property to match. ### Equality @@ -225,4 +225,4 @@ GET /messages?archived[$ne]=true ## Search -Searching is not part of the common querying syntax since it is very specific to the database you are using. For built in databases, see the [SQL `$like` and `$ilike`](./knex.md#like) and [MongoDb search](./mongodb.md#search) documentation. If you are using a community supported adapter its documentation may contain additional information on how to implement search functionality. +Searching is not part of the common querying syntax since it is very specific to the database you are using. For built in databases, see the [SQL search](./knex.md#search) and [MongoDb search](./mongodb.md#search) documentation. If you are using [a community supported adapter](/ecosystem/?cat=Database&sort=lastPublish) their documentation may contain additional information on how to implement search functionality. diff --git a/docs/guides/cli/databases.md b/docs/guides/cli/databases.md index b8f4d14f28..81c4620a15 100644 --- a/docs/guides/cli/databases.md +++ b/docs/guides/cli/databases.md @@ -48,7 +48,7 @@ KnexJS does not have a concept of models. Instead a new service is initialized w The collection for a MongoDB service can be accessed via ```ts -const userCollection = await app.service('users').Model +const userCollection = await app.service('users').getModel() ``` See the [MongoDB service API documentation](../../api/databases/mongodb.md) for more information. diff --git a/docs/guides/cli/service.schemas.md b/docs/guides/cli/service.schemas.md index 1326d72d56..a34b1827d2 100644 --- a/docs/guides/cli/service.schemas.md +++ b/docs/guides/cli/service.schemas.md @@ -90,3 +90,16 @@ export const messagesQueryResolver = resolve({ properties: {} }) ``` + +### Schemas vs MongoDB Validation + +In Feathers v5 (Dove) we added support for Feathers Schema, which performs validation and provides TypeScript types. Recent versions of MongoDB include support for JSON Schema validation at the database server. Most applications will benefit from using Feathers Schema for the following reasons. + +- Feathers Schema's TypeBox integration makes JSON Schema so much easier to read and write. +- You get TypeScript types for free once you've defined your validation rules, using `TypeBox` or `json-schema-to-ts` +- All configuration is done in code, reducing the time to prototype/setup/launch. With MongoDB's built-in validation, you essentially add another "DevOps" step before you can use the database. +- Support for JSON Schema draft 7. MongoDB's validation is based on version 4. +- Feathers Schema don't have to wait for a round-trip to the database to validate the data. +- Feathers Schema can be used in the browser or on the server. + +MongoDB's built-in validation does have built-in support for `bsonType` to force data to be stored as a specific BSON type once it passes validation. There's nothing keeping you from using both solutions together. It's not a use case that's documented, here. diff --git a/packages/adapter-tests/src/declarations.ts b/packages/adapter-tests/src/declarations.ts index d3d4f7cf65..4c3f5b43e5 100644 --- a/packages/adapter-tests/src/declarations.ts +++ b/packages/adapter-tests/src/declarations.ts @@ -55,6 +55,7 @@ export type AdapterMethodsTestName = | '.create' | '.create + $select' | '.create multi' + | '.create ignores query' | 'internal .find' | 'internal .get' | 'internal .create' diff --git a/packages/adapter-tests/src/methods.ts b/packages/adapter-tests/src/methods.ts index 9c1c72bff9..589f10dda5 100644 --- a/packages/adapter-tests/src/methods.ts +++ b/packages/adapter-tests/src/methods.ts @@ -591,6 +591,22 @@ export default (test: AdapterMethodsTest, app: any, _errors: any, serviceName: s await service.remove(data[idProp]) }) + test('.create ignores query', async () => { + const originalData = { + name: 'Billy', + age: 42 + } + const data = await service.create(originalData, { + query: { + name: 'Dave' + } + }) + + assert.strictEqual(data.name, 'Billy', 'data.name matches') + + await service.remove(data[idProp]) + }) + test('.create + $select', async () => { const originalData = { name: 'William', diff --git a/packages/knex/src/adapter.ts b/packages/knex/src/adapter.ts index 9c5d901417..5024877006 100644 --- a/packages/knex/src/adapter.ts +++ b/packages/knex/src/adapter.ts @@ -32,7 +32,6 @@ export class KnexAdapter< ServiceParams extends KnexAdapterParams = KnexAdapterParams, PatchData = Partial > extends AdapterBase { - table: string schema?: string constructor(options: KnexAdapterOptions) { @@ -53,28 +52,32 @@ export class KnexAdapter< }, operators: [...(options.operators || []), '$like', '$notlike', '$ilike', '$and', '$or'] }) + } - this.table = options.name - this.schema = options.schema + get fullName() { + const { name, schema } = this.getOptions({} as ServiceParams) + return schema ? `${schema}.${name}` : name } get Model() { - return this.options.Model + return this.getModel() } - get fullName() { - return this.schema ? `${this.schema}.${this.table}` : this.table + getModel(params?: ServiceParams) { + const { Model } = this.getOptions(params) + return Model } db(params?: ServiceParams) { - const { Model, table, schema } = this + const { Model, name, schema } = this.getOptions(params) if (params && params.transaction && params.transaction.trx) { const { trx } = params.transaction // debug('ran %s with transaction %s', fullName, id) - return schema ? (trx.withSchema(schema).table(table) as Knex.QueryBuilder) : trx(table) + return schema ? (trx.withSchema(schema).table(name) as Knex.QueryBuilder) : trx(name) } - return schema ? (Model.withSchema(schema).table(table) as Knex.QueryBuilder) : Model(table) + + return schema ? (Model.withSchema(schema).table(name) as Knex.QueryBuilder) : Model(name) } knexify(knexQuery: Knex.QueryBuilder, query: Query = {}, parentKey?: string): Knex.QueryBuilder { @@ -116,16 +119,16 @@ export class KnexAdapter< } createQuery(params: ServiceParams) { - const { table, id } = this + const { name, id } = this.getOptions(params) const { filters, query } = this.filterQuery(params) const builder = this.db(params) // $select uses a specific find syntax, so it has to come first. if (filters.$select) { // always select the id field, but make sure we only select it once - builder.select(...new Set([...filters.$select, `${table}.${id}`])) + builder.select(...new Set([...filters.$select, `${name}.${id}`])) } else { - builder.select(`${table}.*`) + builder.select(`${name}.*`) } // build up the knex query out of the query params, include $and and $or filters @@ -157,8 +160,9 @@ export class KnexAdapter< async _find(params?: ServiceParams): Promise | Result[]> async _find(params: ServiceParams = {} as ServiceParams): Promise | Result[]> { const { filters, paginate } = this.filterQuery(params) + const { name, id } = this.getOptions(params) const builder = params.knex ? params.knex.clone() : this.createQuery(params) - const countBuilder = builder.clone().clearSelect().clearOrder().count(`${this.table}.${this.id} as total`) + const countBuilder = builder.clone().clearSelect().clearOrder().count(`${name}.${id} as total`) // Handle $limit if (filters.$limit) { @@ -172,7 +176,7 @@ export class KnexAdapter< // provide default sorting if its not set if (!filters.$sort) { - builder.orderBy(`${this.table}.${this.id}`, 'asc') + builder.orderBy(`${name}.${id}`, 'asc') } const data = filters.$limit === 0 ? [] : await builder.catch(errorHandler) @@ -192,12 +196,13 @@ export class KnexAdapter< } async _findOrGet(id: NullableId, params?: ServiceParams) { + const { name, id: idField } = this.getOptions(params) const findParams = { ...params, paginate: false, query: { ...params?.query, - ...(id !== null ? { [`${this.table}.${this.id}`]: id } : {}) + ...(id !== null ? { [`${name}.${idField}`]: id } : {}) } } @@ -229,14 +234,17 @@ export class KnexAdapter< const client = this.db(params).client.config.client const returning = RETURNING_CLIENTS.includes(client as string) ? [this.id] : [] - const rows: any = await this.db(params).insert(data, returning).returning(this.id).catch(errorHandler) + const rows: any = await this.db(params).insert(data, returning).catch(errorHandler) const id = data[this.id] || rows[0][this.id] || rows[0] if (!id) { return rows as Result[] } - return this._get(id, params) + return this._get(id, { + ...params, + query: _.pick(params?.query || {}, '$select') + }) } async _patch(id: null, data: PatchData, params?: ServiceParams): Promise @@ -251,19 +259,20 @@ export class KnexAdapter< throw new MethodNotAllowed('Can not patch multiple entries') } + const { name, id: idField } = this.getOptions(params) const data = _.omit(raw, this.id) const results = await this._findOrGet(id, { ...params, query: { ...params?.query, - $select: [`${this.table}.${this.id}`] + $select: [`${name}.${idField}`] } }) - const idList = results.map((current: any) => current[this.id]) + const idList = results.map((current: any) => current[idField]) const updateParams = { ...params, query: { - [`${this.table}.${this.id}`]: { $in: idList }, + [`${name}.${idField}`]: { $in: idList }, ...(params?.query?.$select ? { $select: params?.query?.$select } : {}) } } diff --git a/packages/knex/src/hooks.ts b/packages/knex/src/hooks.ts index 1ef169570c..9502763401 100644 --- a/packages/knex/src/hooks.ts +++ b/packages/knex/src/hooks.ts @@ -8,7 +8,7 @@ const debug = createDebug('feathers-knex-transaction') const ROLLBACK = { rollback: true } export const getKnex = (context: HookContext): Knex => { - const knex = context.service.Model + const knex = typeof context.service.getModel === 'function' && context.service.getModel(context.params) return knex && typeof knex.transaction === 'function' ? knex : undefined } @@ -45,7 +45,6 @@ export const start = transaction.id = Date.now() context.params = { ...context.params, transaction } - debug('started a new transaction %s', transaction.id) resolve() diff --git a/packages/knex/test/index.test.ts b/packages/knex/test/index.test.ts index 715adde7ad..c805ec8abd 100644 --- a/packages/knex/test/index.test.ts +++ b/packages/knex/test/index.test.ts @@ -5,7 +5,8 @@ import adapterTests from '@feathersjs/adapter-tests' import { errors } from '@feathersjs/errors' import connection from './connection' -import { KnexService, transaction } from '../src/index' +import { ERROR, KnexAdapterParams, KnexService, transaction } from '../src/index' +import { AdapterQuery } from '@feathersjs/adapter-commons/lib' const testSuite = adapterTests([ '.options', @@ -45,6 +46,7 @@ const testSuite = adapterTests([ '.patch + query + NotFound', '.patch + id + query id', '.create', + '.create ignores query', '.create + $select', '.create multi', 'internal .find', @@ -86,39 +88,42 @@ const db = knex(connection(TYPE) as any) // Create a public database to mimic a "schema" const schemaName = 'public' -function clean() { - return Promise.all([ - db.schema.dropTableIfExists(people.fullName).then(() => { - return db.schema.createTable(people.fullName, (table) => { - table.increments('id') - table.string('name').notNullable() - table.integer('age') - table.integer('time') - table.boolean('created') - return table - }) - }), - db.schema.dropTableIfExists(peopleId.fullName).then(() => { - return db.schema.createTable(peopleId.fullName, (table) => { - table.increments('customid') - table.string('name') - table.integer('age') - table.integer('time') - table.boolean('created') - return table - }) - }), - db.schema.dropTableIfExists(users.fullName).then(() => { - return db.schema.createTable(users.fullName, (table) => { - table.increments('id') - table.string('name') - table.integer('age') - table.integer('time') - table.boolean('created') - return table - }) - }) - ]) +const clean = async () => { + await db.schema.dropTableIfExists('todos') + await db.schema.dropTableIfExists(people.fullName) + await db.schema.createTable(people.fullName, (table) => { + table.increments('id') + table.string('name').notNullable() + table.integer('age') + table.integer('time') + table.boolean('created') + return table + }) + await db.schema.createTable('todos', (table) => { + table.increments('id') + table.string('text') + table.integer('personId') + return table + }) + await db.schema.dropTableIfExists(peopleId.fullName) + await db.schema.createTable(peopleId.fullName, (table) => { + table.increments('customid') + table.string('name') + table.integer('age') + table.integer('time') + table.boolean('created') + return table + }) + + await db.schema.dropTableIfExists(users.fullName) + await db.schema.createTable(users.fullName, (table) => { + table.increments('id') + table.string('name') + table.integer('age') + table.integer('time') + table.boolean('created') + return table + }) } type Person = { @@ -129,10 +134,28 @@ type Person = { create: boolean } +type Todo = { + id: number + text: string + personId: number + personName: string +} + type ServiceTypes = { people: KnexService 'people-customid': KnexService users: KnexService + todos: KnexService +} + +class TodoService extends KnexService { + createQuery(params: KnexAdapterParams) { + const query = super.createQuery(params) + + query.join('people as person', 'todos.personId', 'person.id').select('person.name as personName') + + return query + } } const people = new KnexService({ @@ -154,6 +177,11 @@ const users = new KnexService({ events: ['testing'] }) +const todos = new TodoService({ + Model: db, + name: 'todos' +}) + describe('Feathers Knex Service', () => { const app = feathers() .hooks({ @@ -164,6 +192,7 @@ describe('Feathers Knex Service', () => { .use('people', people) .use('people-customid', peopleId) .use('users', users) + .use('todos', todos) const peopleService = app.service('people') before(() => { @@ -360,11 +389,19 @@ describe('Feathers Knex Service', () => { }) it('attaches the SQL error', async () => { - await assert.rejects(() => peopleService.create({})) + await assert.rejects( + () => peopleService.create({}), + (error: any) => { + assert.ok(error[ERROR]) + return true + } + ) }) }) describe('hooks', () => { + type ModelStub = { getModel: () => Knex } + afterEach(async () => { await db('people').truncate() }) @@ -406,7 +443,7 @@ describe('Feathers Knex Service', () => { it('does commit, rollback, nesting', async () => { const app = feathers<{ people: typeof people - test: Pick & { Model: Knex } + test: Pick & ModelStub }>() app.hooks({ @@ -418,7 +455,7 @@ describe('Feathers Knex Service', () => { app.use('people', people) app.use('test', { - Model: db, + getModel: () => db, create: async (data: any, params) => { const created = await app.service('people').create({ name: 'Foo' }, { ...params }) @@ -444,9 +481,9 @@ describe('Feathers Knex Service', () => { it('does use savepoints for nested calls', async () => { const app = feathers<{ people: typeof people - success: Pick & { Model: Knex } - fail: Pick & { Model: Knex } - test: Pick & { Model: Knex } + success: Pick & ModelStub + fail: Pick & ModelStub + test: Pick & ModelStub }>() app.hooks({ @@ -458,14 +495,14 @@ describe('Feathers Knex Service', () => { app.use('people', people) app.use('success', { - Model: db, + getModel: () => db, create: async (_data, params) => { return app.service('people').create({ name: 'Success' }, { ...params }) } }) app.use('fail', { - Model: db, + getModel: () => db, create: async (_data, params) => { await app.service('people').create({ name: 'Fail' }, { ...params }) throw new TypeError('Deliberate') @@ -473,7 +510,7 @@ describe('Feathers Knex Service', () => { }) app.use('test', { - Model: db, + getModel: () => db, create: async (_data, params) => { await app.service('success').create({}, { ...params }) await app @@ -496,7 +533,7 @@ describe('Feathers Knex Service', () => { it('allows waiting for transaction to complete', async () => { const app = feathers<{ people: typeof people - test: Pick & { Model: Knex } + test: Pick & ModelStub }>() let seq: string[] = [] @@ -531,7 +568,7 @@ describe('Feathers Knex Service', () => { app.use('people', people) app.use('test', { - Model: db, + getModel: () => db, create: async (data: any, params) => { const peeps = await app.service('people').create({ name: 'Foo' }, { ...params }) @@ -598,6 +635,36 @@ describe('Feathers Knex Service', () => { }) }) + describe('associations', () => { + const todoService = app.service('todos') + + it('create, query and get with associations', async () => { + const dave = await peopleService.create({ + name: 'Dave', + age: 133 + }) + const todo = await todoService.create({ + text: 'Do dishes', + personId: dave.id + }) + + const [found] = await todoService.find({ + paginate: false, + query: { + 'person.age': { $gt: 100 } + } + }) + const got = await todoService.get(todo.id) + + assert.strictEqual(got.personName, dave.name) + assert.deepStrictEqual(got, todo) + assert.deepStrictEqual(found, todo) + + peopleService.remove(dave.id) + todoService.remove(todo.id) + }) + }) + testSuite(app, errors, 'users') testSuite(app, errors, 'people') testSuite(app, errors, 'people-customid', 'customid') diff --git a/packages/mongodb/src/adapter.ts b/packages/mongodb/src/adapter.ts index b41c5125ba..7a88392590 100644 --- a/packages/mongodb/src/adapter.ts +++ b/packages/mongodb/src/adapter.ts @@ -93,7 +93,7 @@ export class MongoDbAdapter< } } - getModel(params: ServiceParams) { + getModel(params: ServiceParams = {} as ServiceParams) { const { Model } = this.getOptions(params) return Promise.resolve(Model) } diff --git a/packages/mongodb/test/index.test.ts b/packages/mongodb/test/index.test.ts index ddc1455e4c..9759b5e175 100644 --- a/packages/mongodb/test/index.test.ts +++ b/packages/mongodb/test/index.test.ts @@ -46,6 +46,7 @@ const testSuite = adapterTests([ '.patch + NotFound', '.patch + id + query id', '.create', + '.create ignores query', '.create + $select', '.create multi', 'internal .find',