Merge pull request #96 from spdx/fixes

Check for duplicate licenses for single file publishing

Test/org/spdx/licensexml/LicenseXmlDocumentTest.java

@@ -22,6 +22,7 @@
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.List;
+import java.util.regex.Pattern;
 
 import javax.xml.parsers.DocumentBuilder;
 import javax.xml.parsers.DocumentBuilderFactory;
@@ -53,32 +54,32 @@ public class LicenseXmlDocumentTest {
 	private static final String TEST_LICENSE_ID = "test-id";
 	private static final String TEST_LICENSE_TEXT = "Test Copyright\n\nparagraph 1" +
 			"\n\n   1.\n\n   List item 1\n\n   2.\n\n   List item 2\n\n" +
-			"Last Paragraph Alternate Text Non matching line. Optional text";
+			"Last Paragraph Alternate Text Non matching line. Optional text\n\n";
 	private static final String TEST_LICENSE_NAME = "Test License";
 	private static final List<String>  TEST_LICENSE_URLS = Arrays.asList(new String[] {"http://test/url1","http://test/url2"});
 	private static final String TEST_LICENSE_HEADER = "Test header optional var";
 	private static final String TEST_LICENSE_HEADER_TEMPLATE = "Test header<<beginOptional>> optional<<endOptional>> <<var;name=\"h1test\";original=\"var\";match=\".+\">>";
-	private static final String TEST_LICENSE_TEMPLATE = "<<var;name=\"copyright\";original=\"Test Copyright\";match=\".{0,1000}\">>\n\nparagraph 1" +
+	private static final String TEST_LICENSE_TEMPLATE = "<<var;name=\"copyright\";original=\"Test Copyright  \";match=\".{0,1000}\">>\n\nparagraph 1" +
 			"\n\n   <<var;name=\"bullet\";original=\"1.\";match=\".{0,20}\">>\n\n   List item 1\n\n   <<var;name=\"bullet\";original=\"2.\";match=\".{0,20}\">>\n\n   List item 2\n\n" +
-			"Last Paragraph <<var;name=\"alttest\";original=\"Alternate Text\";match=\".+\">> Non matching line.<<beginOptional>> Optional text<<endOptional>>";
+			"Last Paragraph <<var;name=\"alttest\";original=\"Alternate Text\";match=\".+\">> Non matching line.<<beginOptional>> Optional text<<endOptional>>\n\n";
 
 	private static final String TEST_DEP_LICENSE_COMMENT = "Test dep note";
 	private static final String TEST_DEP_LICENSE_ID = "test-dep";
 	private static final String TEST_DEP_LICENSE_TEXT = "Test Copyright dep\n\nparagraph 1d" +
 			"\n\n   1.d\n\n   List item 1d\n\n   2.d\n\n   List item 2d\n\n" +
-			"Last Paragraph dep Alternate Text dep Non matching line dep. Optional text dep";
+			"Last Paragraph dep Alternate Text dep Non matching line dep. Optional text dep\n\n";
 	private static final String TEST_DEP_LICENSE_NAME = "Test Deprecated License";
 	private static final List<String> TEST_DEP_LICENSE_URLS = Arrays.asList(new String[] {"http://test/url1d","http://test/url2d"});
 	private static final String TEST_DEP_LICENSE_HEADER = "Test header dep";
-	private static final String TEST_DEP_LICENSE_TEMPLATE = "<<var;name=\"copyright\";original=\"Test Copyright dep\";match=\".{0,1000}\">>\n\nparagraph 1d" +
+	private static final String TEST_DEP_LICENSE_TEMPLATE = "<<var;name=\"copyright\";original=\"Test Copyright dep  \";match=\".{0,1000}\">>\n\nparagraph 1d" +
 			"\n\n   <<var;name=\"bullet\";original=\"1.d\";match=\".{0,20}\">>\n\n   List item 1d\n\n   <<var;name=\"bullet\";original=\"2.d\";match=\".{0,20}\">>\n\n   List item 2d\n\n" +
-			"Last Paragraph dep <<var;name=\"alttestd\";original=\"Alternate Text dep\";match=\".+\">> Non matching line dep.<<beginOptional>> Optional text dep<<endOptional>>";
+			"Last Paragraph dep <<var;name=\"alttestd\";original=\"Alternate Text dep\";match=\".+\">> Non matching line dep.<<beginOptional>> Optional text dep<<endOptional>>\n\n";
 
 	private static final String TEST_EXCEPTION_COMMENT = "Test note exception";
 	private static final String TEST_EXCEPTION_ID = "test-ex";
 	private static final String TEST_EXCEPTION_TEXT = "Test Copyrighte\n\nparagraph 1e" +
 			"\n\n   1.e\n\n   List item 1e\n\n   2.e\n\n   List item 2e\n\n" +
-			"Last Paragraph exc Alternate Text exc Non matching line. e Optional text exc";
+			"Last Paragraph exc Alternate Text exc Non matching line. e Optional text exc\n\n";
 	private static final String TEST_EXCEPTION_NAME = "Test Exception";
 	private static final List<String> TEST_EXCEPTION_URLS = Arrays.asList(new String[] {"http://test/url1e","http://test/url2e"});
 	@SuppressWarnings("unused")
@@ -87,7 +88,7 @@ public class LicenseXmlDocumentTest {
 			"Last Paragraph exc <<var;name=\"altteste\";original=\"Alternate Text exc\";match=\".+\">> Non matching line. e<<beginOptional>> Optional text exc<<endOptional>>";
 	private static final String TEST_DEP_LICENSE_VERSION = "2.2";
 	private static final String AGPL3ONLY_FILE_PATH = "TestFiles" + File.separator + "AGPL-3.0-only.xml";
-
+	private static final String BSD_PROTECTION_FILE_PATH = "TestFiles" + File.separator + "BSD-Protection.xml";
 
 
 	/**
@@ -200,4 +201,16 @@ public void testRegressionAgpl3Only() throws LicenseXmlException, InvalidSPDXAna
 		List<SpdxListedLicense> licenses = doc.getListedLicenses();
 		assertEquals(1, licenses.size());
 	}
+
+	@Test
+	public void testRegressionBsdProtection() throws LicenseXmlException, InvalidSPDXAnalysisException, InvalidLicenseTemplateException {
+        File licenseFile = new File(BSD_PROTECTION_FILE_PATH);
+        LicenseXmlDocument doc = new LicenseXmlDocument(licenseFile);
+        List<SpdxListedLicense> licenses = doc.getListedLicenses();
+        assertEquals(1, licenses.size());
+        SpdxListedLicense result = licenses.get(0);
+        String template = result.getStandardLicenseTemplate();
+        Pattern matchingModificationLine = Pattern.compile("<<beginOptional>>\\s?----------------------------------------------------------------<<endOptional>>",Pattern.MULTILINE);
+        assertTrue(matchingModificationLine.matcher(template).find());       
+	}
 }

TestFiles/BSD-Protection.xml

@@ -0,0 +1,168 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<SPDXLicenseCollection xmlns="http://www.spdx.org/license">
+   <license isOsiApproved="false" licenseId="BSD-Protection" name="BSD Protection License">
+      <crossRefs>
+         <crossRef>https://fedoraproject.org/wiki/Licensing/BSD_Protection_License</crossRef>
+      </crossRefs>
+    <text>
+      <titleText>
+         <p>BSD Protection License
+        <br/>February 2002
+      </p>
+      </titleText>
+      <optional>
+         <p>Preamble
+        <br/>--------
+      </p>
+         <p>The Berkeley Software Distribution ("BSD") license has proven very effective over the years at
+         allowing for a wide spread of work throughout both commercial and non-commercial products. For
+         programmers whose primary intention is to improve the general quality of available software, it is
+         arguable that there is no better license than the BSD license, as it permits improvements to be used
+         wherever they will help, without idealogical or metallic constraint.</p>
+         <p>This is of particular value to those who produce reference implementations of proposed standards: The
+         case of TCP/IP clearly illustrates that freely and universally available implementations leads the
+         rapid acceptance of standards -- often even being used instead of a de jure standard (eg, OSI network
+         models).</p>
+         <p>With the rapid proliferation of software licensed under the GNU General Public License, however, the
+         continued success of this role is called into question. Given that the inclusion of a few lines of
+         "GPL-tainted" work into a larger body of work will result in restricted distribution -- and
+         given that further work will likely build upon the "tainted" portions, making them difficult
+         to remove at a future date -- there are inevitable circumstances where authors would, in order to
+         protect their goal of providing for the widespread usage of their work, wish to guard against such
+         "GPL-taint".</p>
+         <p>In addition, one can imagine that companies which operate by producing and selling (possibly
+         closed-source) code would wish to protect themselves against the rise of a GPL-licensed competitor.
+         While under existing licenses this would mean not releasing their code under any form of open license,
+         if a license existed under which they could incorporate any improvements back into their own
+         (commercial) products then they might be far more willing to provide for non-closed distribution.</p>
+         <p>For the above reasons, we put forth this "BSD Protection License": A license designed to retain
+         the freedom granted by the BSD license to use licensed works in a wide variety of settings, both
+         non-commercial and commercial, while protecting the work from having future contributors restrict that
+         freedom.</p>
+         <p>The precise terms and conditions for copying, distribution, and modification follow.</p>
+      </optional>
+
+      <p>BSD PROTECTION LICENSE TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION, AND MODIFICATION</p>
+      <optional>----------------------------------------------------------------</optional>
+      <list>
+        <item>
+            <bullet>0.</bullet>
+          Definitions.
+        </item>
+        <list>
+            <item>
+               <bullet>a)</bullet>
+            "Program", below, refers to any program or work distributed under the terms of this
+               license.
+          </item>
+            <item>
+               <bullet>b)</bullet>
+            A "work based on the Program", below, refers to either the Program or any
+               derivative work under copyright law.
+          </item>
+            <item>
+               <bullet>c)</bullet>
+            "Modification", below, refers to the act of creating derivative works.
+          </item>
+            <item>
+               <bullet>d)</bullet>
+            "You", below, refers to each licensee.
+          </item>
+        </list>
+        <item>
+            <bullet>1.</bullet>
+          Scope.
+            <br/>This license governs the copying, distribution, and modification of the Program. Other
+                 activities are outside the scope of this license; The act of running the Program is
+                 not restricted, and the output from the Program is covered only if its contents
+                 constitute a work based on the Program.
+
+        </item>
+        <item>
+            <bullet>2.</bullet>
+          Verbatim copies.
+            <br/>You may copy and distribute verbatim copies of the Program as you receive it, in any
+                 medium, provided that you conspicuously and appropriately publish on each copy an
+                 appropriate copyright notice; keep intact all the notices that refer to this License
+                 and to the absence of any warranty; and give any other recipients of the Program a
+                 copy of this License along with the Program.
+
+        </item>
+        <item>
+            <bullet>3.</bullet>
+          Modification and redistribution under closed license.
+            <br/>You may modify your copy or copies of the Program, and distribute the resulting derivative
+                 works, provided that you meet the following conditions:
+
+        </item>
+        <list>
+            <item>
+               <bullet>a)</bullet>
+            The copyright notice and disclaimer on the Program must be reproduced and included in the
+               source code, documentation, and/or other materials provided in a manner in which such
+               notices are normally distributed.
+          </item>
+            <item>
+               <bullet>b)</bullet>
+            The derivative work must be clearly identified as such, in order that it may not be confused
+               with the original work.
+          </item>
+            <item>
+               <bullet>c)</bullet>
+            The license under which the derivative work is distributed must expressly prohibit the
+               distribution of further derivative works.
+          </item>
+        </list>
+        <item>
+            <bullet>4.</bullet>
+          Modification and redistribution under open license.
+            <br/>You may modify your copy or copies of the Program, and distribute the resulting derivative
+                 works, provided that you meet the following conditions:
+
+        </item>
+        <list>
+            <item>
+               <bullet>a)</bullet>
+            The copyright notice and disclaimer on the Program must be reproduced and included in the
+               source code, documentation, and/or other materials provided in a manner in which such
+               notices are normally distributed.
+          </item>
+            <item>
+               <bullet>b)</bullet>
+            You must clearly indicate the nature and date of any changes made to the Program. The full
+               details need not necessarily be included in the individual modified files, provided that
+               each modified file is clearly marked as such and instructions are included on where the
+               full details of the modifications may be found.
+          </item>
+            <item>
+               <bullet>c)</bullet>
+            You must cause any work that you distribute or publish, that in whole or in part contains or
+               is derived from the Program or any part thereof, to be licensed as a whole at no charge to
+               all third parties under the terms of this License.
+          </item>
+        </list>
+        <item>
+            <bullet>5.</bullet>
+          Implied acceptance.
+            <br/>You may not copy or distribute the Program or any derivative works except as expressly
+                 provided under this license. Consequently, any such action will be taken as implied
+                 acceptance of the terms of this license.
+
+        </item>
+        <item>
+            <bullet>6.</bullet>
+          NO WARRANTY.
+        </item>
+      </list>
+      <p>THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+         LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+         DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+         REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+         EXEMPLARY, OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING,
+         BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+         BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+         LIABILITY, OR TORT, EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
+         DAMAGES.</p>
+    </text>
+  </license>
+</SPDXLicenseCollection>

pom.xml

@@ -3,7 +3,7 @@
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>org.spdx</groupId>
 	<artifactId>licenseListPublisher</artifactId>
-	<version>2.2.0-SNAPSHOT</version>
+	<version>2.2.0</version>
 	<name>License List Publisher</name>
 	<description>Tool that generates license data found in the license-list-data repository from the license-list-XML source</description>
 	<licenses>
@@ -33,11 +33,11 @@
 		<developerConnection>scm:git:ssh://git@github.com:spdx/LicenseListPublisher.git</developerConnection>
 	</scm>
 	<distributionManagement>
-		<repository>
-			<id>bintray-spdx-spdx-LicenseListPublisher</id>
-			<name>spdx-spdx-LicenseListPublisher</name>
-        <url>https://api.bintray.com/maven/spdx/spdx-tools/LicenseListPublisher/;publish=1</url>
-    </repository>
+	  <repository>
+        <id>ossrh</id>
+        <name>spdx-spdx-tools</name>
+        <url>https://oss.sonatype.org/service/local/staging/deploy/maven2/</url>
+      </repository>
 	</distributionManagement>
 	  <properties>
 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
@@ -77,7 +77,7 @@
 		<dependency>
 			<groupId>org.spdx</groupId>
 			<artifactId>java-spdx-library</artifactId>
-			<version>0.0.5</version>
+			<version>1.0.1</version>
 		</dependency>
 		<dependency>
 			<groupId>com.github.spullara.mustache.java</groupId>
@@ -97,7 +97,7 @@
 		<dependency>
 			<groupId>org.spdx</groupId>
 			<artifactId>spdx-rdf-store</artifactId>
-			<version>0.0.5</version>
+			<version>1.0.0</version>
 		</dependency>
 	</dependencies>
 	<profiles>

src/org/spdx/licenselistpublisher/LicenseRDFAGenerator.java

@@ -522,6 +522,24 @@ private static Set<String> writeLicenseList(String version, String releaseDate,
 					}
 				}
 			}
+			if (addedLicIdTextMap.size() == 1) {
+			    // Since we are only creating a single file, we should check the listed licenses for duplicates
+			    addedLicIdTextMap.entrySet().forEach(entry -> {
+		             String[] matchingLicenseIds;
+                    try {
+                        matchingLicenseIds = LicenseCompareHelper.matchingStandardLicenseIds(entry.getValue());
+                        for (String matchingId:matchingLicenseIds) {
+                            if (!entry.getKey().equals(matchingId)) {
+                                warnings.add("Duplicates licenses: "+entry.getKey()+", "+matchingId);
+                            }
+                        }
+                    } catch (InvalidSPDXAnalysisException e) {
+                        warnings.add("Error comparing single license to existing listed licenses: "+e.getMessage());
+                    } catch (SpdxCompareException e) {
+                        warnings.add("Error comparing single license to existing listed licenses: "+e.getMessage());
+                    }
+			    });
+			}
 			return addedLicIdTextMap.keySet();
 		} finally {
 			if (licenseIter instanceof Closeable) {

src/org/spdx/licensexml/LicenseXmlHelper.java

@@ -161,6 +161,8 @@ private static boolean appendNodeText(Node node, boolean useTemplateFormat, Stri
 				appendElementChildrenText(element, useTemplateFormat, sb, indentCount, unprocessedTags, includeHtmlTags);
 				if (includeHtmlTags) {
 					sb.append("</p>\n");
+				} else {
+				    addNewline(sb, indentCount);
 				}
 			} else if (SpdxConstants.LICENSEXML_ELEMENT_TITLE_TEXT.equals(tagName)) {
 				if (!inALtBlock(element)) {
@@ -431,7 +433,7 @@ private static boolean appendAltText(Element element, String altName, String mat
 			sb.append("<<var;name=\"");
 			sb.append(altName);
 			sb.append("\";original=\"");
-			sb.append(originalSb);
+			sb.append(originalSb.toString().replaceAll("\n", " "));  // Remove any new lines
 			sb.append("\";match=\"");
 			sb.append(match);
 			sb.append("\">>");
@@ -638,7 +640,9 @@ public static Object getHeaderTextHtml(Element headerElement) throws LicenseXmlE
 	 * @return Text normalized for different character variations
 	 */
 	private static String fixUpText(String string) {
-		return string.replaceAll(DOUBLE_QUOTES_REGEX, "\"").replaceAll(SINGLE_QUOTES_REGEX, "'");
+		return string.replaceAll(DOUBLE_QUOTES_REGEX, "\"")
+		        .replaceAll(SINGLE_QUOTES_REGEX, "'")
+		        .replaceAll("\\n\\s*\\n\\s*\\n", "\n\n");
 	}
 
 	/**