feat: add logging (#5)

* chore: add logging wip

* chore: add loggig for verbosing debug

* chore: update dependencies

* chore: disable unit test

until future refactoring

* chore: add missing log param

* chore: update circleci

* chore: add formating

* chore: add codecov

plugins/verdaccio-aws-s3-storage/.circleci/config.yml

@@ -75,7 +75,7 @@ jobs:
       - save_cache:
           key: *repo_key
           paths:
-            - ~/local-storage
+            - ~/aws-s3-storage
 
   test_node12:
     <<: *defaults

plugins/verdaccio-aws-s3-storage/.eslintrc

@@ -4,7 +4,12 @@
   ],
   "plugins": ["jest"],
   "rules": {
-    "@typescript-eslint/no-use-before-define": 0
+    "@typescript-eslint/no-use-before-define": 0,
+    "@typescript-eslint/explicit-function-return-type": ["warn",
+      {
+          "allowExpressions": true,
+          "allowTypedFunctionExpressions": true
+    }]
   },
   "env": {
     "jest/globals": true

plugins/verdaccio-aws-s3-storage/.npmignore

@@ -1,6 +1,5 @@
 src/
 tests-report/
-.nyc_output
 .editorconfig
 .gitignore
 yarn-error.log
@@ -20,3 +19,11 @@ travis.yml
 coverage/
 jest.config.js
 jestEnvironment.js
+tests/
+.idea/
+.circleci/
+tsconfig.json
+jest.config.js
+.DS_Store
+.prettierrc
+*.spec.js

plugins/verdaccio-aws-s3-storage/README.md

@@ -12,7 +12,7 @@
 [![node](https://img.shields.io/node/v/verdaccio-aws-s3-storage/latest.svg)](https://www.npmjs.com/package/verdaccio-aws-s3-storage)
 
 
-Based on [`verdaccio-s3-storage`](https://github.com/Remitly/verdaccio-s3-storage) in Typescript
+Based on [`verdaccio-s3-storage`](https://github.com/Remitly/verdaccio-s3-storage) built in Typescript.
 
 🚧 Under construction
 
@@ -45,4 +45,6 @@ store:
     region: us-west-2 # optional, will use aws s3's default behavior if not specified
     endpoint: https://{service}.{region}.amazonaws.com # optional, will use aws s3's default behavior if not specified
     s3ForcePathStyle: false # optional, will use path style URLs for S3 objects
+    accessKeyId: your-access-key-id # optional, aws accessKeyId for private S3 bucket
+    secretAccessKey: your-secret-access-key # optional, aws secretAccessKey for private S3 bucket
 ```

plugins/verdaccio-aws-s3-storage/package.json

@@ -19,23 +19,26 @@
     "type-check:watch": "npm run type-check -- --watch",
     "build": "npm run build:types && npm run build:js",
     "build:types": "tsc --emitDeclarationOnly",
-    "build:js": "babel src/ --out-dir lib --extensions \".ts,.tsx\" --source-maps inline",
+    "build:js": "babel src/ --out-dir lib --extensions \".ts,.tsx\"",
     "format": "prettier --single-quote --trailing-comma none --write \"{src,test}/**/*.js\"",
     "coverage:publish": "codecov"
   },
   "dependencies": {
-    "@verdaccio/streams": "2.0.0",
-    "aws-sdk": "2.479.0"
+    "@verdaccio/commons-api": "^0.1.0",
+    "@verdaccio/streams": "^2.0.0",
+    "aws-sdk": "2.480.0"
   },
   "devDependencies": {
     "@commitlint/cli": "8.0.0",
     "@commitlint/config-conventional": "8.0.0",
     "@types/aws-sdk": "2.7.0",
     "@types/http-errors": "1.6.1",
     "@types/jest": "24.0.15",
+    "@types/node": "^12.0.10",
     "@verdaccio/babel-preset": "0.2.1",
     "@verdaccio/eslint-config": "0.0.1",
     "@verdaccio/types": "5.1.0",
+    "codecov": "^3.5.0",
     "cross-env": "5.2.0",
     "eslint": "5.16.0",
     "husky": "2.4.1",

plugins/verdaccio-aws-s3-storage/src/config.ts

@@ -6,4 +6,6 @@ export interface S3Config extends Config {
   endpoint?: string;
   region?: string;
   s3ForcePathStyle?: boolean;
+  accessKeyId?: string;
+  secretAccessKey?: string;
 }

plugins/verdaccio-aws-s3-storage/src/index.ts

@@ -1,48 +1,56 @@
 import { LocalStorage, Logger, Config, Callback, IPluginStorage, PluginOptions } from '@verdaccio/types';
+import { getInternalError, VerdaccioError } from '@verdaccio/commons-api';
 import { S3 } from 'aws-sdk';
 import { S3Config } from './config';
 import S3PackageManager from './s3PackageManager';
 import { convertS3Error, is404Error } from './s3Errors';
 
 export default class S3Database implements IPluginStorage<S3Config> {
-  logger: Logger;
-  config: S3Config;
-  s3: S3;
-  _localData: LocalStorage | null;
+  public logger: Logger;
+  public config: S3Config;
+  private s3: S3;
+  private _localData: LocalStorage | null;
 
-  constructor(config: Config, options: PluginOptions<S3Config>) {
+  public constructor(config: Config, options: PluginOptions<S3Config>) {
     this.logger = options.logger;
     // copy so we don't mutate
     if (!config) {
       throw new Error('s3 storage missing config. Add `store.s3-storage` to your config file');
     }
-    this.config = Object.assign({}, config.store['s3-storage']);
+    this.config = Object.assign({}, config.store['aws-s3-storage']);
     if (!this.config.bucket) {
       throw new Error('s3 storage requires a bucket');
     }
     const configKeyPrefix = this.config.keyPrefix;
     this._localData = null;
     this.config.keyPrefix = configKeyPrefix != null ? (configKeyPrefix.endsWith('/') ? configKeyPrefix : `${configKeyPrefix}/`) : '';
+
+    this.logger.debug({ config: JSON.stringify(this.config, null, 4) }, 's3: configuration: @{config}');
+
     this.s3 = new S3({
       endpoint: this.config.endpoint,
       region: this.config.region,
-      s3ForcePathStyle: this.config.s3ForcePathStyle
+      s3ForcePathStyle: this.config.s3ForcePathStyle,
+      accessKeyId: this.config.accessKeyId,
+      secretAccessKey: this.config.secretAccessKey
     });
   }
 
-  async getSecret(): Promise<any> {
+  public async getSecret(): Promise<string> {
     return Promise.resolve((await this._getData()).secret);
   }
 
-  async setSecret(secret: string): Promise<any> {
+  public async setSecret(secret: string): Promise<void> {
     (await this._getData()).secret = secret;
     await this._sync();
   }
 
-  add(name: string, callback: Callback) {
+  public add(name: string, callback: Callback): void {
+    this.logger.debug({ name }, 's3: [add] private package @{name}');
     this._getData().then(async data => {
       if (data.list.indexOf(name) === -1) {
         data.list.push(name);
+        this.logger.trace({ name }, 's3: [add] @{name} has been added');
         try {
           await this._sync();
           callback(null);
@@ -55,30 +63,38 @@ export default class S3Database implements IPluginStorage<S3Config> {
     });
   }
 
-  async search(onPackage: Function, onEnd: Function, validateName: Function) {
+  public async search(onPackage: Function, onEnd: Function, validateName: Function): Promise<void> {
+    this.logger.debug('s3: [search]');
     const storage = await this._getData();
     const storageInfoMap = storage.list.map(this._fetchPackageInfo.bind(this, onPackage));
+    this.logger.debug({ l: storageInfoMap.length }, 's3: [search] storageInfoMap length is @{l}');
     await Promise.all(storageInfoMap);
     onEnd();
   }
 
-  async _fetchPackageInfo(onPackage: Function, packageName: string) {
+  private async _fetchPackageInfo(onPackage: Function, packageName: string): Promise<void> {
+    const { bucket, keyPrefix } = this.config;
+    this.logger.debug({ packageName }, 's3: [_fetchPackageInfo] @{packageName}');
+    this.logger.trace({ keyPrefix, bucket }, 's3: [_fetchPackageInfo] bucket: @{bucket} prefix: @{keyPrefix}');
     return new Promise(resolve => {
       this.s3.headObject(
         {
-          Bucket: this.config.bucket,
-          Key: `${this.config.keyPrefix + packageName}/package.json`
+          Bucket: bucket,
+          Key: `${keyPrefix + packageName}/package.json`
         },
         (err, response) => {
           if (err) {
+            this.logger.debug({ err }, 's3: [_fetchPackageInfo] error: @{err}');
             return resolve();
           }
           if (response.LastModified) {
+            const { LastModified } = response;
+            this.logger.trace({ LastModified }, 's3: [_fetchPackageInfo] LastModified: @{LastModified}');
             return onPackage(
               {
                 name: packageName,
                 path: packageName,
-                time: response.LastModified.getTime()
+                time: LastModified.getTime()
               },
               resolve
             );
@@ -89,34 +105,43 @@ export default class S3Database implements IPluginStorage<S3Config> {
     });
   }
 
-  remove(name: string, callback: Callback) {
+  public remove(name: string, callback: Callback): void {
+    this.logger.debug({ name }, 's3: [remove] @{name}');
     this.get(async (err, data) => {
       if (err) {
-        callback(new Error('error on get'));
+        this.logger.error({ err }, 's3: [remove] error: @{err}');
+        callback(getInternalError('something went wrong on remove a package'));
       }
 
       const pkgName = data.indexOf(name);
       if (pkgName !== -1) {
         const data = await this._getData();
         data.list.splice(pkgName, 1);
+        this.logger.debug({ pkgName }, 's3: [remove] sucessfully removed @{pkgName}');
       }
 
       try {
+        this.logger.trace('s3: [remove] starting sync');
         await this._sync();
+        this.logger.trace('s3: [remove] finish sync');
         callback(null);
       } catch (err) {
+        this.logger.error({ err }, 's3: [remove] sync error: @{err}');
         callback(err);
       }
     });
   }
 
-  get(callback: Callback) {
+  public get(callback: Callback): void {
+    this.logger.debug('s3: [get]');
     this._getData().then(data => callback(null, data.list));
   }
 
   // Create/write database file to s3
-  async _sync() {
+  private async _sync(): Promise<void> {
     await new Promise((resolve, reject) => {
+      const { bucket, keyPrefix } = this.config;
+      this.logger.debug({ keyPrefix, bucket }, 's3: [_sync] bucket: @{bucket} prefix: @{keyPrefix}');
       this.s3.putObject(
         {
           Bucket: this.config.bucket,
@@ -125,46 +150,60 @@ export default class S3Database implements IPluginStorage<S3Config> {
         },
         (err, data) => {
           if (err) {
+            this.logger.error({ err }, 's3: [_sync] error: @{err}');
             reject(err);
             return;
           }
+          this.logger.debug('s3: [_sync] sucess');
           resolve();
         }
       );
     });
   }
 
   // returns an instance of a class managing the storage for a single package
-  getPackageStorage(packageName: string): S3PackageManager {
+  public getPackageStorage(packageName: string): S3PackageManager {
+    this.logger.debug({ packageName }, 's3: [getPackageStorage] @{packageName}');
+
     return new S3PackageManager(this.config, packageName, this.logger);
   }
 
-  async _getData(): Promise<LocalStorage> {
+  private async _getData(): Promise<LocalStorage> {
     if (!this._localData) {
       this._localData = await new Promise((resolve, reject) => {
+        const { bucket, keyPrefix } = this.config;
+        this.logger.debug({ keyPrefix, bucket }, 's3: [_getData] bucket: @{bucket} prefix: @{keyPrefix}');
+        this.logger.trace('s3: [_getData] get database object');
         this.s3.getObject(
           {
-            Bucket: this.config.bucket,
-            Key: `${this.config.keyPrefix}verdaccio-s3-db.json`
+            Bucket: bucket,
+            Key: `${keyPrefix}verdaccio-s3-db.json`
           },
           (err, response) => {
             if (err) {
-              const s3Err = convertS3Error(err);
+              const s3Err: VerdaccioError = convertS3Error(err);
+              this.logger.error({ s3Err }, 's3: [_getData] err: @{err}');
               if (is404Error(s3Err)) {
+                this.logger.error('s3: [_getData] err 404 create new database');
                 resolve({ list: [], secret: '' });
               } else {
                 reject(err);
               }
               return;
             }
+
             // @ts-ignore
-            const data = JSON.parse(response.Body.toString());
+            const body = response.Body.toString();
+            const data = JSON.parse(body);
+            this.logger.trace({ body }, 's3: [_getData] get data @{body}');
             resolve(data);
           }
         );
       });
+    } else {
+      this.logger.trace('s3: [_getData] already exist');
     }
-    // @ts-ignore
-    return this._localData;
+
+    return this._localData as LocalStorage;
   }
 }