Fix ScanCodeSummarizer

lib/utils.js

@@ -338,7 +338,12 @@ function isDeclaredLicense(identifier) {
 }
 
 function getLicenseLocations(coordinates) {
-  const map = { npm: ['package/'], maven: ['meta-inf/'], pypi: [`${coordinates.name}-${coordinates.revision}/`], go: [goLicenseLocations(coordinates)] }
+  const map = {
+    npm: ['package/'],
+    maven: ['meta-inf/', 'META-INF/'],
+    pypi: [`${coordinates.name}-${coordinates.revision}/`],
+    go: [goLicenseLocations(coordinates)]
+  }
   return map[coordinates.type]
 }
 

providers/summary/scancode.js

@@ -5,6 +5,7 @@ const { get, flatten, uniq } = require('lodash')
 const SPDX = require('@clearlydefined/spdx')
 const {
   extractDate,
+  isDeclaredLicense,
   getLicenseLocations,
   isLicenseFile,
   setIfValue,
@@ -33,8 +34,11 @@ class ScanCodeSummarizer {
     const result = {}
     this.addDescribedInfo(result, harvested)
     let declaredLicense = this._readDeclaredLicense(scancodeVersion, harvested)
-    if (!declaredLicense || declaredLicense === 'NOASSERTION') {
-      declaredLicense = this._getDeclaredLicense(scancodeVersion, harvested, coordinates)
+    if (!isDeclaredLicense(declaredLicense)) {
+      declaredLicense = this._readLicenseExpression(harvested) || declaredLicense
+    }
+    if (!isDeclaredLicense(declaredLicense)) {
+      declaredLicense = this._getDeclaredLicense(scancodeVersion, harvested, coordinates) || declaredLicense
     }
     setIfValue(result, 'licensed.declared', declaredLicense)
     result.files = this._summarizeFileInfo(harvested.content.files, coordinates)
@@ -61,8 +65,9 @@ class ScanCodeSummarizer {
         // Some Maven packages have this value as an object rather than a string
         // Example: for maven/mavencentral/redis.clients/jedis/4.1.1
         // declared_license would be { "name": "MIT", "url": "http://github.com/redis/jedis/raw/master/LICENSE.txt", "comments": null, "distribution": "repo" }'
+        // Some pypi packages have this value as an object with a license field 
         if (typeof declared_license != 'string' && declared_license != undefined) {
-          declared_license = declared_license['name']
+          declared_license = declared_license['name'] || declared_license['license']
         }
 
         return SPDX.normalize(declared_license)
@@ -72,6 +77,11 @@ class ScanCodeSummarizer {
     }
   }
 
+  _readLicenseExpression(harvested) {
+    const licenseExpression = get(harvested, 'content.summary.packages[0].license_expression')
+    return licenseExpression && this._normalizeLicenseExpression(licenseExpression)
+  }
+
   // find and return the files that should be considered for as a license determinator for this summarization
   _getRootFiles(coordinates, files) {
     const roots = getLicenseLocations(coordinates) || []
@@ -110,20 +120,16 @@ class ScanCodeSummarizer {
   }
 
   _getLicenseByIsLicenseText(files) {
-    const fullLicenses = files
-      .filter(file => file.is_license_text && file.licenses)
-      .reduce((licenses, file) => {
-        file.licenses.forEach(license => {
-          licenses.add(this._createExpressionFromLicense(license))
-        })
-        return licenses
-      }, new Set())
-    return this._joinExpressions(fullLicenses)
+    return this._extractDeclaredLicenses(files, file => file.is_license_text && file.licenses)
   }
 
   _getLicenseByFileName(files, coordinates) {
+    return this._extractDeclaredLicenses(files, file => isLicenseFile(file.path, coordinates) && file.licenses)
+  }
+
+  _extractDeclaredLicenses(files, isValidLicenseFile) {
     const fullLicenses = files
-      .filter(file => isLicenseFile(file.path, coordinates) && file.licenses)
+      .filter(file => isValidLicenseFile(file))
       .reduce((licenses, file) => {
         file.licenses.forEach(license => {
           if (license.score >= 90) licenses.add(this._createExpressionFromLicense(license))
@@ -188,9 +194,13 @@ class ScanCodeSummarizer {
   _createExpressionFromLicense(license) {
     const rule = license.matched_rule
     if (!rule || !rule.license_expression) return SPDX.normalize(license.spdx_license_key)
-    const parsed = SPDX.parse(rule.license_expression, key => SPDX.normalizeSingle(scancodeMap.get(key) || key))
+    return this._normalizeLicenseExpression(rule.license_expression)
+  }
+
+  _normalizeLicenseExpression(licenseExpression) {
+    const parsed = SPDX.parse(licenseExpression, (key) => SPDX.normalizeSingle(scancodeMap.get(key) || key))
     const result = SPDX.stringify(parsed)
-    if (result === 'NOASSERTION') this.logger.info(`ScanCode NOASSERTION from ${rule.license_expression}`)
+    if (result === 'NOASSERTION') this.logger.info(`ScanCode NOASSERTION from ${licenseExpression}`)
     return result
   }
 }

test/business/definitionServiceTest.js

@@ -400,7 +400,9 @@ describe('Aggregation service', () => {
     const summaries = summaryService.summarizeAll(coords, raw)
     const { service } = setupAggregatorWithParams(coordSpec, tools)
     const aggregated = service.process(summaries, coords)
-    expect(aggregated.licensed.declared).to.be.equal('LGPL-2.1-only')
+    expect(aggregated.licensed.declared).to.be.ok
+    // package manifest: LGPL-2.0-or-later, license: LGPL-2.1-only
+    expect(aggregated.licensed.declared).to.be.not.equal('NOASSERTION')
   })
 })