Skip to content
New issue

Have a question about this project? # for a free GitHub account to open an issue and contact its maintainers and the community.

#

By clicking “#”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? # to your account

fix(logs): add more context #2 #2146

Merged
merged 6 commits into from
May 13, 2024
Merged

fix(logs): add more context #2 #2146

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
82552c2
fix(logs): add more context #2
bodinsamuel May 13, 2024
5aa75d9
Merge branch 'master' into fix/logs-context-2
bodinsamuel May 13, 2024
f781ddf
proper mock
bodinsamuel May 13, 2024
62b1322
clean up
bodinsamuel May 13, 2024
5e4acad
fix condition
bodinsamuel May 13, 2024
633c7b5
Merge branch 'master' into fix/logs-context-2
bodinsamuel May 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 8 additions & 7 deletions packages/jobs/lib/activities.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -172,8 +172,8 @@ export async function scheduleAndRouteSync(args: ContinuousSyncArgs): Promise<bo
{
account: { id: nangoConnection.account_id! },
environment: { id: nangoConnection.environment_id },
connection: { id: nangoConnection.id! },
sync: { id: syncId }
connection: { id: nangoConnection.id!, name: nangoConnection.connection_id },
sync: { id: syncId, name: syncName }
}
);
await logCtx.error('The continuous sync failed to run because of a failure to obtain the provider config', { error: err, syncName });
Expand Down Expand Up @@ -231,11 +231,11 @@ export async function syncProvider(
start: Date.now(),
end: Date.now(),
timestamp: Date.now(),
connection_id: nangoConnection?.connection_id,
provider_config_key: nangoConnection?.provider_config_key,
connection_id: nangoConnection.connection_id,
provider_config_key: nangoConnection.provider_config_key,
provider: syncConfig.provider,
session_id: syncJobId ? syncJobId?.toString() : '',
environment_id: nangoConnection?.environment_id,
environment_id: nangoConnection.environment_id,
operation_name: syncName
};
const activityLogId = (await createActivityLog(log)) as number;
Expand All @@ -245,8 +245,9 @@ export async function syncProvider(
{
account: { id: nangoConnection.account_id! },
environment: { id: nangoConnection.environment_id },
connection: { id: nangoConnection.id! },
sync: { id: syncId }
connection: { id: nangoConnection.id!, name: nangoConnection.connection_id },
config: { id: syncConfig.id!, name: syncConfig.unique_key },
sync: { id: syncId, name: syncName }
}
);

Expand Down
8 changes: 4 additions & 4 deletions packages/jobs/lib/crons/autoIdleDemo.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,10 +63,10 @@ export async function exec(): Promise<void> {
const logCtx = await logContextGetter.create(
{ id: String(activityLogId), operation: { type: 'sync', action: 'pause' }, message: 'Sync' },
{
account: { id: sync.account_id },
environment: { id: sync.environment_id },
connection: { id: sync.connection_unique_id },
sync: { id: sync.id }
account: { id: sync.account_id, name: sync.account_name },
environment: { id: sync.environment_id, name: sync.environment_name },
connection: { id: sync.connection_unique_id, name: sync.connection_id },
sync: { id: sync.id, name: sync.name }
}
);

Expand Down
6 changes: 3 additions & 3 deletions packages/logs/lib/models/helpers.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,9 @@ export interface FormatMessageData {
account?: { id: number; name?: string };
user?: { id: number } | undefined;
environment?: { id: number; name?: string } | undefined;
connection?: { id: number; name?: string } | undefined;
config?: { id: number; name?: string } | undefined;
sync?: { id: string; name?: string } | undefined;
connection?: { id: number; name: string } | undefined;
config?: { id: number; name: string } | undefined;
sync?: { id: string; name: string } | undefined;
}

export function getFormattedMessage(data: Partial<MessageRow>, { account, user, environment, config, connection, sync }: FormatMessageData = {}): MessageRow {
Expand Down
2 changes: 1 addition & 1 deletion packages/persist/lib/server.integration.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -252,7 +252,7 @@ const initDb = async () => {

await logContextGetter.create(
{ id: String(activityLogId), operation: { type: 'sync', action: 'run' }, message: 'Sync' },
{ account: { id: env.account_id }, environment: { id: env.id } }
{ account: { id: env.account_id, name: '' }, environment: { id: env.id, name: env.name } }
);

const connectionRes = await connectionService.upsertConnection(`conn-test`, `provider-test`, 'google', {} as AuthCredentials, {}, env.id, 0);
Expand Down
82 changes: 40 additions & 42 deletions packages/server/lib/controllers/apiAuth.controller.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,7 @@ import type { RequestLocals } from '../utils/express.js';

class ApiAuthController {
async apiKey(req: Request, res: Response<any, Required<RequestLocals>>, next: NextFunction) {
const accountId = res.locals['account'].id;
const environmentId = res.locals['environment'].id;
const { account, environment } = res.locals;
const { providerConfigKey } = req.params;
const connectionId = req.query['connection_id'] as string | undefined;
const connectionConfig = req.query['params'] != null ? getConnectionConfig(req.query['params']) : {};
Expand All @@ -44,7 +43,7 @@ class ApiAuthController {
timestamp: Date.now(),
connection_id: connectionId as string,
provider_config_key: providerConfigKey as string,
environment_id: environmentId
environment_id: environment.id
};

const activityLogId = await createActivityLog(log);
Expand All @@ -53,9 +52,9 @@ class ApiAuthController {
try {
logCtx = await logContextGetter.create(
{ id: String(activityLogId), operation: { type: 'auth' }, message: 'Authorization API Key' },
{ account: { id: accountId }, environment: { id: environmentId } }
{ account, environment }
);
void analytics.track(AnalyticsTypes.PRE_API_KEY_AUTH, accountId);
void analytics.track(AnalyticsTypes.PRE_API_KEY_AUTH, account.id);

if (!providerConfigKey) {
errorManager.errRes(res, 'missing_connection');
Expand All @@ -69,13 +68,13 @@ class ApiAuthController {
return;
}

const hmacEnabled = await hmacService.isEnabled(environmentId);
const hmacEnabled = await hmacService.isEnabled(environment.id);
if (hmacEnabled) {
const hmac = req.query['hmac'] as string | undefined;
if (!hmac) {
await createActivityLogMessageAndEnd({
level: 'error',
environment_id: environmentId,
environment_id: environment.id,
activity_log_id: activityLogId as number,
timestamp: Date.now(),
content: 'Missing HMAC in query params'
Expand All @@ -87,11 +86,11 @@ class ApiAuthController {

return;
}
const verified = await hmacService.verify(hmac, environmentId, providerConfigKey, connectionId);
const verified = await hmacService.verify(hmac, environment.id, providerConfigKey, connectionId);
if (!verified) {
await createActivityLogMessageAndEnd({
level: 'error',
environment_id: environmentId,
environment_id: environment.id,
activity_log_id: activityLogId as number,
timestamp: Date.now(),
content: 'Invalid HMAC'
Expand All @@ -105,12 +104,12 @@ class ApiAuthController {
}
}

const config = await configService.getProviderConfig(providerConfigKey, environmentId);
const config = await configService.getProviderConfig(providerConfigKey, environment.id);

if (config == null) {
await createActivityLogMessageAndEnd({
level: 'error',
environment_id: environmentId,
environment_id: environment.id,
activity_log_id: activityLogId as number,
content: `Error during API Key auth: config not found`,
timestamp: Date.now()
Expand All @@ -128,7 +127,7 @@ class ApiAuthController {
if (template.auth_mode !== AuthModes.ApiKey) {
await createActivityLogMessageAndEnd({
level: 'error',
environment_id: environmentId,
environment_id: environment.id,
activity_log_id: activityLogId as number,
timestamp: Date.now(),
content: `Provider ${config.provider} does not support API key auth`
Expand Down Expand Up @@ -163,15 +162,15 @@ class ApiAuthController {
credentials,
connectionId,
providerConfigKey,
environmentId,
environment.id,
connectionConfig,
tracer
);

if (connectionResponse.isErr()) {
await createActivityLogMessageAndEnd({
level: 'error',
environment_id: environmentId,
environment_id: environment.id,
activity_log_id: activityLogId as number,
content: `The credentials provided were not valid for the ${config.provider} provider`,
timestamp: Date.now()
Expand All @@ -186,7 +185,7 @@ class ApiAuthController {

await createActivityLogMessage({
level: 'info',
environment_id: environmentId,
environment_id: environment.id,
activity_log_id: activityLogId as number,
content: `API key auth creation was successful`,
timestamp: Date.now()
Expand All @@ -202,8 +201,8 @@ class ApiAuthController {
config.provider,
credentials,
connectionConfig,
environmentId,
accountId
environment.id,
account.id
);

if (updatedConnection) {
Expand All @@ -212,7 +211,7 @@ class ApiAuthController {
id: updatedConnection.id,
connection_id: connectionId,
provider_config_key: providerConfigKey,
environment_id: environmentId,
environment_id: environment.id,
auth_mode: AuthModes.ApiKey,
operation: updatedConnection.operation
},
Expand All @@ -230,7 +229,7 @@ class ApiAuthController {

await createActivityLogMessage({
level: 'error',
environment_id: environmentId,
environment_id: environment.id,
activity_log_id: activityLogId as number,
content: `Error during API key auth: ${prettyError}`,
timestamp: Date.now()
Expand All @@ -241,7 +240,7 @@ class ApiAuthController {
id: -1,
connection_id: connectionId as string,
provider_config_key: providerConfigKey as string,
environment_id: environmentId,
environment_id: environment.id,
auth_mode: AuthModes.ApiKey,
error: `Error during API key auth: ${prettyError}`,
operation: AuthOperation.UNKNOWN
Expand All @@ -257,7 +256,7 @@ class ApiAuthController {
errorManager.report(err, {
source: ErrorSourceEnum.PLATFORM,
operation: LogActionEnum.AUTH,
environmentId,
environmentId: environment.id,
metadata: {
providerConfigKey,
connectionId
Expand All @@ -269,8 +268,7 @@ class ApiAuthController {
}

async basic(req: Request, res: Response<any, Required<RequestLocals>>, next: NextFunction) {
const accountId = res.locals['account'].id;
const environmentId = res.locals['environment'].id;
const { account, environment } = res.locals;
const { providerConfigKey } = req.params;
const connectionId = req.query['connection_id'] as string | undefined;
const connectionConfig = req.query['params'] != null ? getConnectionConfig(req.query['params']) : {};
Expand All @@ -284,7 +282,7 @@ class ApiAuthController {
timestamp: Date.now(),
connection_id: connectionId as string,
provider_config_key: providerConfigKey as string,
environment_id: environmentId
environment_id: environment.id
};

const activityLogId = await createActivityLog(log);
Expand All @@ -293,9 +291,9 @@ class ApiAuthController {
try {
logCtx = await logContextGetter.create(
{ id: String(activityLogId), operation: { type: 'auth' }, message: 'Authorization Basic' },
{ account: { id: accountId }, environment: { id: environmentId } }
{ account, environment }
);
void analytics.track(AnalyticsTypes.PRE_BASIC_API_KEY_AUTH, accountId);
void analytics.track(AnalyticsTypes.PRE_BASIC_API_KEY_AUTH, account.id);

if (!providerConfigKey) {
errorManager.errRes(res, 'missing_connection');
Expand All @@ -309,13 +307,13 @@ class ApiAuthController {
return;
}

const hmacEnabled = await hmacService.isEnabled(environmentId);
const hmacEnabled = await hmacService.isEnabled(environment.id);
if (hmacEnabled) {
const hmac = req.query['hmac'] as string | undefined;
if (!hmac) {
await createActivityLogMessageAndEnd({
level: 'error',
environment_id: environmentId,
environment_id: environment.id,
activity_log_id: activityLogId as number,
timestamp: Date.now(),
content: 'Missing HMAC in query params'
Expand All @@ -327,11 +325,11 @@ class ApiAuthController {

return;
}
const verified = await hmacService.verify(hmac, environmentId, providerConfigKey, connectionId);
const verified = await hmacService.verify(hmac, environment.id, providerConfigKey, connectionId);
if (!verified) {
await createActivityLogMessageAndEnd({
level: 'error',
environment_id: environmentId,
environment_id: environment.id,
activity_log_id: activityLogId as number,
timestamp: Date.now(),
content: 'Invalid HMAC'
Expand All @@ -346,12 +344,12 @@ class ApiAuthController {

const { username = '', password = '' } = req.body;

const config = await configService.getProviderConfig(providerConfigKey, environmentId);
const config = await configService.getProviderConfig(providerConfigKey, environment.id);

if (config == null) {
await createActivityLogMessageAndEnd({
level: 'error',
environment_id: environmentId,
environment_id: environment.id,
activity_log_id: activityLogId as number,
content: `Error during basic API auth: config not found`,
timestamp: Date.now()
Expand All @@ -369,7 +367,7 @@ class ApiAuthController {
if (template.auth_mode !== AuthModes.Basic) {
await createActivityLogMessageAndEnd({
level: 'error',
environment_id: environmentId,
environment_id: environment.id,
activity_log_id: activityLogId as number,
timestamp: Date.now(),
content: `Provider ${config.provider} does not support Basic API auth`
Expand All @@ -394,15 +392,15 @@ class ApiAuthController {
credentials,
connectionId,
providerConfigKey,
environmentId,
environment.id,
connectionConfig,
tracer
);

if (connectionResponse.isErr()) {
await createActivityLogMessageAndEnd({
level: 'error',
environment_id: environmentId,
environment_id: environment.id,
activity_log_id: activityLogId as number,
content: `The credentials provided were not valid for the ${config.provider} provider`,
timestamp: Date.now()
Expand All @@ -420,7 +418,7 @@ class ApiAuthController {

await createActivityLogMessage({
level: 'info',
environment_id: environmentId,
environment_id: environment.id,
activity_log_id: activityLogId as number,
content: `Basic API key auth creation was successful with the username ${username}`,
timestamp: Date.now()
Expand All @@ -436,8 +434,8 @@ class ApiAuthController {
config.provider,
credentials,
connectionConfig,
environmentId,
accountId
environment.id,
account.id
);

if (updatedConnection) {
Expand All @@ -446,7 +444,7 @@ class ApiAuthController {
id: updatedConnection.id,
connection_id: connectionId,
provider_config_key: providerConfigKey,
environment_id: environmentId,
environment_id: environment.id,
auth_mode: AuthModes.Basic,
operation: updatedConnection.operation
},
Expand All @@ -464,7 +462,7 @@ class ApiAuthController {

await createActivityLogMessage({
level: 'error',
environment_id: environmentId,
environment_id: environment.id,
activity_log_id: activityLogId as number,
content: `Error during basic API auth: ${prettyError}`,
timestamp: Date.now()
Expand All @@ -475,7 +473,7 @@ class ApiAuthController {
id: -1,
connection_id: connectionId as string,
provider_config_key: providerConfigKey as string,
environment_id: environmentId,
environment_id: environment.id,
auth_mode: AuthModes.ApiKey,
error: `Error during basic API key auth: ${prettyError}`,
operation: AuthOperation.UNKNOWN
Expand All @@ -491,7 +489,7 @@ class ApiAuthController {
errorManager.report(err, {
source: ErrorSourceEnum.PLATFORM,
operation: LogActionEnum.AUTH,
environmentId,
environmentId: environment.id,
metadata: {
providerConfigKey,
connectionId
Expand Down
Loading
Loading