updated for #2753

core/src/main/java/org/owasp/dependencycheck/data/nvdcve/CveDB.java

@@ -49,6 +49,7 @@
 import org.owasp.dependencycheck.data.nvd.json.ProblemtypeDatum;
 import org.owasp.dependencycheck.data.nvd.json.Reference;
 import static org.owasp.dependencycheck.data.nvdcve.CveDB.PreparedStatementCveDb.*;
+import org.owasp.dependencycheck.data.update.cpe.CpeEcosystemCache;
 import org.owasp.dependencycheck.data.update.cpe.CpePlus;
 import org.owasp.dependencycheck.dependency.CvssV2;
 import org.owasp.dependencycheck.dependency.CvssV3;
@@ -209,7 +210,15 @@ enum PreparedStatementCveDb {
         /**
          * Key for SQL Statement.
          */
-        UPDATE_VULNERABILITY
+        UPDATE_VULNERABILITY,
+        /**
+         * Key for SQL Statement.
+         */
+        SELECT_CPE_ECOSYSTEM,
+        /**
+         * Key for SQL Statement.
+         */
+        MERGE_CPE_ECOSYSTEM
     }
 
     /**
@@ -810,10 +819,11 @@ public synchronized Vulnerability getVulnerability(String cve) throws DatabaseEx
      *
      * @param cve the vulnerability from the NVD CVE Data Feed to add to the
      * database
-     * @param ecosystem the ecosystem the CVE belongs to
+     * @param baseEcosystem the ecosystem the CVE belongs to; this is based off
+     * of things like the CVE description
      * @throws DatabaseException is thrown if the database
      */
-    public void updateVulnerability(DefCveItem cve, String ecosystem) {
+    public void updateVulnerability(DefCveItem cve, String baseEcosystem) {
         clearCache();
         final String cveId = cve.getCve().getCVEDataMeta().getId();
         try {
@@ -827,7 +837,7 @@ public void updateVulnerability(DefCveItem cve, String ecosystem) {
 
                 //parse the CPEs outside of a synchronized method
                 final List<VulnerableSoftware> software = parseCpes(cve);
-                updateVulnerabilityInsertSoftware(vulnerabilityId, cveId, software, ecosystem);
+                updateVulnerabilityInsertSoftware(vulnerabilityId, cveId, software, baseEcosystem);
             }
 
         } catch (SQLException ex) {
@@ -841,6 +851,50 @@ public void updateVulnerability(DefCveItem cve, String ecosystem) {
         }
     }
 
+    private void loadCpeEcosystemCache() {
+        Map<Pair<String, String>, String> map = new HashMap<>();
+        ResultSet rs;
+        try (PreparedStatement ps = prepareStatement(SELECT_CPE_ECOSYSTEM)) {
+            rs = ps.executeQuery();
+            while (rs.next()) {
+                Pair<String, String> key = new Pair<>(rs.getString(1), rs.getString(1));
+                String value = rs.getString(3);
+                map.put(key, value);
+            }
+        } catch (SQLException ex) {
+            final String msg = String.format("Error loading the Cpe Ecosystem Cache: %s", ex.getMessage());
+            LOGGER.debug(msg, ex);
+            throw new DatabaseException(msg, ex);
+        }
+
+        CpeEcosystemCache.setCache(map);
+    }
+
+    private void saveCpeEcosystemCache() {
+        Map<Pair<String, String>, String> map = CpeEcosystemCache.getChanged();
+        if (map != null && !map.isEmpty()) {
+            try (PreparedStatement ps = prepareStatement(MERGE_CPE_ECOSYSTEM)) {
+                for (Map.Entry<Pair<String, String>, String> entry : map.entrySet()) {
+                    ps.setString(1, entry.getKey().getLeft());
+                    ps.setString(2, entry.getKey().getRight());
+                    ps.setString(3, entry.getValue());
+                    if (isBatchInsertEnabled()) {
+                        ps.addBatch();
+                    } else {
+                        ps.execute();
+                    }
+                }
+                if (isBatchInsertEnabled()) {
+                    ps.executeBatch();
+                }
+            } catch (SQLException ex) {
+                final String msg = String.format("Error saving the Cpe Ecosystem Cache: %s", ex.getMessage());
+                LOGGER.debug(msg, ex);
+                throw new DatabaseException(msg, ex);
+            }
+        }
+    }
+
     /**
      * Used when updating a vulnerability - this method inserts the
      * vulnerability entry itself.
@@ -850,6 +904,9 @@ public void updateVulnerability(DefCveItem cve, String ecosystem) {
      * @return the vulnerability ID
      */
     private synchronized int updateOrInsertVulnerability(DefCveItem cve, String description) {
+        if (CpeEcosystemCache.isEmpty()) {
+            loadCpeEcosystemCache();
+        }
         final int vulnerabilityId;
         try (PreparedStatement callUpdate = prepareStatement(UPDATE_VULNERABILITY)) {
             if (callUpdate == null) {
@@ -1032,7 +1089,9 @@ private synchronized void updateVulnerabilityInsertSoftware(int vulnerabilityId,
                 insertSoftware.setString(10, parsedCpe.getTargetSw());
                 insertSoftware.setString(11, parsedCpe.getTargetHw());
                 insertSoftware.setString(12, parsedCpe.getOther());
-                final String ecosystem = cveItemConverter.extractEcosystem(baseEcosystem, parsedCpe);
+                String ecosystem = CpeEcosystemCache.getEcosystem(parsedCpe.getVendor(), parsedCpe.getProduct(), 
+                        cveItemConverter.extractEcosystem(baseEcosystem, parsedCpe));
+
                 addNullableStringParameter(insertSoftware, 13, ecosystem);
                 addNullableStringParameter(insertSoftware, 14, parsedCpe.getVersionEndExcluding());
                 addNullableStringParameter(insertSoftware, 15, parsedCpe.getVersionEndIncluding());
@@ -1063,8 +1122,7 @@ private synchronized void updateVulnerabilityInsertSoftware(int vulnerabilityId,
 
     /**
      * Used when updating a vulnerability - this method inserts the list of
-     * references. In addition, this method attempts to determine the ecosystem
-     * based on the references information.
+     * references.
      *
      * @param vulnerabilityId the vulnerability id
      * @param cve the CVE entry that contains the list of references
@@ -1195,17 +1253,6 @@ private boolean isBatchInsertEnabled() {
         return batch;
     }
 
-    /**
-     * Generates a logging message for batch inserts.
-     *
-     * @param pCountReferences the number of batch statements executed
-     * @param pFormat a Java String.format string
-     * @return the formated string
-     */
-    private String getLogForBatchInserts(int pCountReferences, String pFormat) {
-        return String.format(pFormat, pCountReferences, new Date());
-    }
-
     /**
      * Executes batch inserts of vulnerabilities when property
      * database.batchinsert.maxsize is reached.
@@ -1273,6 +1320,7 @@ public synchronized boolean dataExists() {
     public synchronized void cleanupDatabase() {
         LOGGER.info("Begin database maintenance");
         final long start = System.currentTimeMillis();
+        saveCpeEcosystemCache();
         clearCache();
         try (PreparedStatement psOrphans = getPreparedStatement(CLEANUP_ORPHANS);
                 PreparedStatement psEcosystem = getPreparedStatement(UPDATE_ECOSYSTEM);

core/src/main/resources/data/dbStatements.properties

@@ -23,6 +23,7 @@ INSERT_REFERENCE=INSERT INTO reference (cveid, name, url, source) VALUES (?, ?,
 INSERT_SOFTWARE=CALL insert_software(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
 UPDATE_VULNERABILITY=CALL update_vulnerability(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
 
+SELECT_CPE_ECOSYSTEM=SELECT DISTINCT vendor, product, ecosystem FROM cpeEcosystemCache
 SELECT_CVE_FROM_SOFTWARE=SELECT cve, part, vendor, product, version, update_version, edition, lang, sw_edition, target_sw, target_hw, other, versionEndExcluding, versionEndIncluding, versionStartExcluding, versionStartIncluding, vulnerable FROM software INNER JOIN vulnerability ON vulnerability.id = software.cveId INNER JOIN cpeEntry ON cpeEntry.id = software.cpeEntryId WHERE vendor = ? AND product = ? ORDER BY cve, vendor, product, version, update_version
 SELECT_CPE_ENTRIES=SELECT part, vendor, product, version, update_version, edition, lang, sw_edition, target_sw, target_hw, other, ecosystem FROM cpeEntry WHERE vendor = ? AND product = ?
 SELECT_REFERENCES=SELECT source, name, url FROM reference WHERE cveid = ?
@@ -37,8 +38,8 @@ INSERT_PROPERTY=INSERT INTO properties (id, value) VALUES (?, ?)
 UPDATE_PROPERTY=UPDATE properties SET value = ? WHERE id = ?
 DELETE_PROPERTY=DELETE FROM properties WHERE id = ?
 
-UPDATE_ECOSYSTEM=UPDATE cpeEntry u SET u.ecosystem = (SELECT q.eco FROM cpeEntry e INNER JOIN (SELECT DISTINCT vendor, product, MIN(ecosystem) eco FROM cpeEntry WHERE ecosystem is not null GROUP BY vendor, product) q ON q.vendor = e.vendor AND q.product = e.product AND e.id = u.id) WHERE exists (SELECT * FROM cpeEntry e INNER JOIN (SELECT DISTINCT vendor, product, MIN(ecosystem) eco FROM cpeEntry WHERE ecosystem is not null GROUP BY vendor, product) q ON q.vendor = e.vendor AND q.product = e.product AND e.id = u.id) AND u.ecosystem is null
-UPDATE_ECOSYSTEM2=UPDATE cpeEntry SET ecosystem = NULL WHERE id IN (SELECT cpeEntry.id FROM cpeEntry INNER JOIN (SELECT DISTINCT r.vendor, r.product FROM vulnerability v INNER JOIN software s ON v.id = s.cveid INNER JOIN cpeEntry r ON s.cpeentryid=r.id WHERE description like '%bindings%') bindings ON bindings.vendor=cpeEntry.vendor AND bindings.product=cpeEntry.product UNION ALL SELECT z.id FROM cpeEntry z WHERE z.ecosystem is not null AND z.vendor = 'apache' AND z.product = 'zookeeper') AND ecosystem IS NOT NULL;
+UPDATE_ECOSYSTEM=UPDATE cpeEntry e SET e.ecosystem=(SELECT cpeEcosystemCache.ecosystem FROM cpeEcosystemCache WHERE cpeEcosystemCache.vendor=e.vendor AND cpeEcosystemCache.product=e.product AND e.ecosystem IS NULL AND cpeEcosystemCache.ecosystem<>'MULTIPLE') WHERE e.ecosystem IS NULL;
+UPDATE_ECOSYSTEM2=UPDATE cpeEntry e SET e.ecosystem=null WHERE e.ecosystem IS NOT NULL AND EXISTS(SELECT * FROM cpeEcosystemCache WHERE cpeEcosystemCache.vendor=e.vendor AND cpeEcosystemCache.product=e.product AND cpeEcosystemCache.ecosystem='MULTIPLE');
 
 #the following two statements are unused and are only referenced in dead code
 #DELETE_UNUSED_DICT_CPE=DELETE FROM cpeEntry WHERE dictionaryEntry=true AND id NOT IN (SELECT cpeEntryId FROM software)

core/src/main/resources/data/dbStatements_h2.properties

@@ -13,4 +13,5 @@
 # limitations under the License.
 
 MERGE_PROPERTY=MERGE INTO properties (id, value) KEY(id) VALUES(?, ?)
+MERGE_CPE_ECOSYSTEM=MERGE INTO cpeEcosystemCache (vendor, product, ecosystem) KEY(vendor, product) VALUES(?, ?, ?)
 CLEANUP_ORPHANS=DELETE FROM cpeEntry WHERE id IN (SELECT id FROM cpeEntry LEFT JOIN software ON cpeEntry.id = software.CPEEntryId WHERE software.CPEEntryId IS NULL)

core/src/main/resources/data/dbStatements_postgresql.properties

@@ -17,3 +17,12 @@ CLEANUP_ORPHANS=DELETE FROM cpeEntry WHERE NOT EXISTS (SELECT * FROM software WH
 UPDATE_ECOSYSTEM=UPDATE CPEENTRY SET ecosystem = (SELECT q.eco FROM CPEENTRY e INNER JOIN (SELECT DISTINCT vendor, product, MIN(ecosystem) eco FROM CPEENTRY WHERE ecosystem is not null GROUP BY vendor, product) q ON q.VENDOR = e.VENDOR AND q.PRODUCT = e.PRODUCT AND e.id = CPEENTRY.id) WHERE exists (SELECT * FROM CPEENTRY e INNER JOIN (SELECT DISTINCT vendor, product, MIN(ecosystem) eco FROM CPEENTRY WHERE ecosystem is not null GROUP BY vendor, product) q ON q.VENDOR = e.VENDOR AND q.PRODUCT = e.PRODUCT AND e.id = CPEENTRY.id) AND CPEENTRY.ecosystem is null;
 INSERT_SOFTWARE=SELECT insert_software(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
 UPDATE_VULNERABILITY=SELECT update_vulnerability(?::VARCHAR(20), ?::VARCHAR(8000), ?::VARCHAR(20), ?::DECIMAL(3,1), ?::DECIMAL(3,1), ?::BOOLEAN, ?::BOOLEAN, ?:: BOOLEAN, ?::BOOLEAN, ?::BOOLEAN, ?::DECIMAL(3,1), ?::VARCHAR(20), ?::VARCHAR(20), ?::VARCHAR(20), ?::VARCHAR(20), ?::VARCHAR(20), ?::VARCHAR(20), ?::VARCHAR(5),?::DECIMAL(3,1), ?::DECIMAL(3,1), ?::VARCHAR(20), ?::VARCHAR(20), ?::VARCHAR(20), ?::VARCHAR(20), ?::VARCHAR(20), ?::VARCHAR(20), ?::VARCHAR(20), ?::VARCHAR(20), ?::DECIMAL(3,1), ?::VARCHAR(20), ?::VARCHAR(5));
+
+MERGE_CPE_ECOSYSTEM=SELECT merge_ecosystem(?, ?, ?);
+
+
+UPDATE_ECOSYSTEM=UPDATE cpeEntry SET ecosystem=cpeEcosystemCache.ecosystem FROM cpeEcosystemCache WHERE cpeEcosystemCache.vendor=cpeEntry.vendor AND cpeEcosystemCache.product=cpeEntry.product AND cpeEntry.ecosystem IS NULL AND cpeEcosystemCache.ecosystem<>'MULTIPLE';
+UPDATE_ECOSYSTEM2=UPDATE cpeEntry SET ecosystem=null FROM cpeEcosystemCache WHERE cpeEcosystemCache.vendor=cpeEntry.vendor AND cpeEcosystemCache.product=cpeEntry.product AND cpeEcosystemCache.ecosystem='MULTIPLE' AND cpeEntry.ecosystem IS NOT NULL;
+
+
+

core/src/main/resources/data/initialize.sql

@@ -4,6 +4,7 @@ DROP TABLE IF EXISTS reference;
 DROP TABLE IF EXISTS vulnerability;
 DROP TABLE IF EXISTS properties;
 DROP TABLE IF EXISTS cweEntry;
+DROP TABLE IF EXISTS cpeEcosystemCache;
 DROP ALIAS IF EXISTS update_vulnerability;
 DROP ALIAS IF EXISTS insert_software;
 
@@ -35,13 +36,14 @@ CREATE TABLE software (cveid INT, cpeEntryId INT, versionEndExcluding VARCHAR(50
 CREATE TABLE cweEntry (cveid INT, cwe VARCHAR(20),
     CONSTRAINT fkCweEntry FOREIGN KEY (cveid) REFERENCES vulnerability(id) ON DELETE CASCADE);
 
+CREATE TABLE cpeEcosystemCache (vendor VARCHAR(255), product VARCHAR(255), ecosystem VARCHAR(255), PRIMARY KEY (vendor, product));
+INSERT INTO cpeEcosystemCache (vendor, product, ecosystem) VALUES ('apache', 'zookeeper', 'MULTIPLE');
+
 CREATE INDEX idxCwe ON cweEntry(cveid);
 CREATE INDEX idxVulnerability ON vulnerability(cve);
 CREATE INDEX idxReference ON reference(cveid);
-CREATE INDEX idxCpe ON cpeEntry(vendor, product);
 CREATE INDEX idxSoftwareCve ON software(cveid);
 CREATE INDEX idxSoftwareCpe ON software(cpeEntryId);
-
 CREATE INDEX idxCpeEntry ON cpeEntry(part, vendor, product, version, update_version, edition, lang, sw_edition, target_sw, target_hw, other);
 
 CREATE ALIAS update_vulnerability FOR "org.owasp.dependencycheck.data.nvdcve.H2Functions.updateVulnerability";

core/src/main/resources/data/initialize_postgres.sql

@@ -2,15 +2,17 @@
 --\c dependencycheck;
 --CREATE USER dcuser WITH PASSWORD 'DC-Pass1337!';
 
-DROP FUNCTION IF EXISTS dependencycheck.save_property;
-DROP FUNCTION IF EXISTS dependencycheck.update_vulnerability;
-DROP FUNCTION IF EXISTS dependencycheck.insert_software;
+DROP FUNCTION IF EXISTS public.save_property;
+DROP FUNCTION IF EXISTS public.update_vulnerability;
+DROP FUNCTION IF EXISTS public.insert_software;
+DROP FUNCTION IF EXISTS public.merge_ecosystem;
 DROP TABLE IF EXISTS software;
 DROP TABLE IF EXISTS cpeEntry;
 DROP TABLE IF EXISTS reference;
 DROP TABLE IF EXISTS properties;
 DROP TABLE IF EXISTS cweEntry;
 DROP TABLE IF EXISTS vulnerability;
+DROP TABLE IF EXISTS cpeEcosystemCache;
 
 CREATE TABLE vulnerability (id SERIAL PRIMARY KEY, cve VARCHAR(20) UNIQUE,
     description VARCHAR(8000), v2Severity VARCHAR(20), v2ExploitabilityScore DECIMAL(3,1), 
@@ -36,6 +38,9 @@ CREATE TABLE software (cveid INT, cpeEntryId INT, versionEndExcluding VARCHAR(50
     , CONSTRAINT fkSoftwareCve FOREIGN KEY (cveid) REFERENCES vulnerability(id) ON DELETE CASCADE
     , CONSTRAINT fkSoftwareCpeProduct FOREIGN KEY (cpeEntryId) REFERENCES cpeEntry(id));
 
+CREATE TABLE cpeEcosystemCache (vendor VARCHAR(255), product VARCHAR(255), ecosystem VARCHAR(255), PRIMARY KEY (vendor, product));
+INSERT INTO cpeEcosystemCache (vendor, product, ecosystem) VALUES ('apache', 'zookeeper', 'MULTIPLE');
+
 CREATE TABLE cweEntry (cveid INT, cwe VARCHAR(20),
     CONSTRAINT fkCweEntry FOREIGN KEY (cveid) REFERENCES vulnerability(id) ON DELETE CASCADE);
 
@@ -46,15 +51,15 @@ CREATE INDEX idxCpe ON cpeEntry(vendor, product);
 CREATE INDEX idxSoftwareCve ON software(cveid);
 CREATE INDEX idxSoftwareCpe ON software(cpeEntryId);
 CREATE INDEX idxCpeEntry ON cpeEntry(part, vendor, product, version, update_version, edition, lang, sw_edition, target_sw, target_hw, other);
-CREATE INDEX idxCpeEntryEco ON cpeEntry(id, vendor, product);
+
 
 
 CREATE TABLE properties (id varchar(50) PRIMARY KEY, value varchar(500));
 
 GRANT SELECT, INSERT, DELETE, UPDATE ON ALL TABLES IN SCHEMA public TO dcuser;
 GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public to dcuser;
 
-DROP FUNCTION IF EXISTS save_property(varchar(50),varchar(500));
+
 
 CREATE FUNCTION save_property (IN prop varchar(50), IN val varchar(500))
 RETURNS void
@@ -69,6 +74,20 @@ $$ LANGUAGE sql;
 GRANT EXECUTE ON FUNCTION public.save_property(varchar(50),varchar(500)) TO dcuser;
 
 
+CREATE FUNCTION merge_ecosystem (IN p_vendor VARCHAR(255), IN p_product VARCHAR(255), IN p_ecosystem varchar(255))
+RETURNS void
+AS $$
+BEGIN
+IF EXISTS(SELECT * FROM cpeEcosystemCache WHERE vendor=p_vendor AND product=p_product) THEN
+    UPDATE cpeEcosystemCache SET ecosystem=p_ecosystem WHERE vendor=p_vendor AND product=p_product;
+ELSE
+    INSERT INTO cpeEcosystemCache (vendor, product, ecosystem) VALUES (p_vendor, p_product, p_ecosystem);
+END IF;
+END
+$$ LANGUAGE plpgsql;
+
+GRANT EXECUTE ON FUNCTION public.save_property(varchar(50),varchar(500)) TO dcuser;
+
 CREATE FUNCTION update_vulnerability (
     IN p_cveId VARCHAR(20), IN p_description VARCHAR(8000), IN p_v2Severity VARCHAR(20), 
     IN p_v2ExploitabilityScore DECIMAL(3,1), IN p_v2ImpactScore DECIMAL(3,1), IN p_v2AcInsufInfo BOOLEAN, 

core/src/test/resources/data/upgrade_4.2.sql

@@ -28,6 +28,9 @@ ALTER TABLE vulnerability ADD   COLUMN v3ExploitabilityScore DECIMAL(3,1);
 ALTER TABLE vulnerability ADD   COLUMN v3ImpactScore DECIMAL(3,1);
 ALTER TABLE vulnerability ADD   COLUMN v3Version VARCHAR(5);
 
+CREATE TABLE cpeEcosystemCache (vendor VARCHAR(255), product VARCHAR(255), ecosystem VARCHAR(255), PRIMARY KEY (vendor, product));
+INSERT INTO cpeEcosystemCache (vendor, product, ecosystem) VALUES ('apache', 'zookeeper', 'MULTIPLE');
+
 CREATE ALIAS update_vulnerability FOR "org.owasp.dependencycheck.data.nvdcve.H2Functions.updateVulnerability";
 CREATE ALIAS insert_software FOR "org.owasp.dependencycheck.data.nvdcve.H2Functions.insertSoftware";