diff --git a/src/org/spdx/licenselistpublisher/LicenseSingleHTMLGenerator.java b/src/org/spdx/licenselistpublisher/LicenseSingleHTMLGenerator.java new file mode 100644 index 0000000..0cb3f5b --- /dev/null +++ b/src/org/spdx/licenselistpublisher/LicenseSingleHTMLGenerator.java @@ -0,0 +1,324 @@ +/** + * Copyright (c) 2011 Source Auditor Inc. + * Copyright (c) 2024 Steve Winslow + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.spdx.licenselistpublisher; + +import java.io.BufferedReader; +import java.io.Closeable; +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.StringReader; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Map.Entry; +import java.util.Set; + +import org.apache.commons.io.FileUtils; +import org.spdx.library.InvalidSPDXAnalysisException; +import org.spdx.library.model.license.LicenseException; +import org.spdx.library.model.license.ListedLicenseException; +import org.spdx.library.model.license.SpdxListedLicense; +import org.spdx.library.model.license.SpdxListedLicenseException; +import org.spdx.licenseTemplate.InvalidLicenseTemplateException; +import org.spdx.licensexml.XmlLicenseProviderSingleFile; +import org.spdx.licensexml.XmlLicenseProviderWithCrossRefDetails; +import org.spdx.utility.compare.LicenseCompareHelper; +import org.spdx.utility.compare.SpdxCompareException; +import org.spdx.licenselistpublisher.licensegenerator.FsfLicenseDataParser; +import org.spdx.licenselistpublisher.licensegenerator.ILicenseFormatWriter; +import org.spdx.licenselistpublisher.licensegenerator.ILicenseTester; +import org.spdx.licenselistpublisher.licensegenerator.LicenseHtmlFormatWriter; +import org.spdx.licenselistpublisher.licensegenerator.LicenseJsonFormatWriter; +import org.spdx.licenselistpublisher.licensegenerator.LicenseMarkdownFormatWriter; +import org.spdx.licenselistpublisher.licensegenerator.LicenseRdfFormatWriter; +import org.spdx.licenselistpublisher.licensegenerator.LicenseRdfaFormatWriter; +import org.spdx.licenselistpublisher.licensegenerator.LicenseTemplateFormatWriter; +import org.spdx.licenselistpublisher.licensegenerator.LicenseTextFormatWriter; +import org.spdx.licenselistpublisher.licensegenerator.SimpleLicenseTester; +import org.spdx.licenselistpublisher.licensegenerator.SpdxWebsiteFormatWriter; + +import au.com.bytecode.opencsv.CSVReader; + +/** + * Converts input license text and metadata for a single license into a single HTML file. + * + * Supported input formats: + * - License XML file - File following the SPDX legal team license format + * + * Supported output formats: + * - Website - the content for the website available at https://spdx.org/licenses + * + * @author Gary O'Neall and Steve Winslow + * + */ +public class LicenseSingleHTMLGenerator { + + static final Set INVALID_TEXT_CHARS = new HashSet<>(); + + static { + INVALID_TEXT_CHARS.add('\uFFFD'); + } + static final int ERROR_STATUS = 1; + static final int WARNING_STATUS = 64; + private static final String LICENSE_XML_FOLDER_NAME = "license-list-XML"; + + /** + * @param args Arg 0 is a license XML file in XML format, arg 1 is the directory for the output file + */ + public static void main(String[] args) { + if (args == null || args.length != 2) { + System.out.println("Invalid arguments"); + usage(); + System.exit(ERROR_STATUS); + } + File licenseXmlFile = new File(args[0]); + if (!licenseXmlFile.exists()) { + System.out.println("License XML "+licenseXmlFile.getName()+" does not exist"); + usage(); + System.exit(ERROR_STATUS); + } + File dir = new File(args[1]); + if (!dir.exists()) { + System.out.println("Output directory "+dir.getName()+" does not exist"); + usage(); + System.exit(ERROR_STATUS); + } + if (!dir.isDirectory()) { + System.out.println("Output directory "+dir.getName()+" is not a directory"); + usage(); + System.exit(ERROR_STATUS); + } + + try { + List warnings = generateLicenseData(licenseXmlFile, dir); + if (warnings != null && warnings.size() > 0) { + System.exit(WARNING_STATUS); + } + } catch (LicenseGeneratorException e) { + System.out.println(e.getMessage()); + System.exit(ERROR_STATUS); + } + } + /** + * Generate license data + * @param licenseXml License XML file or directory containing license XML files + * @param dir Output directory for the generated results + * @return warnings + * @throws LicenseGeneratorException + */ + public static List generateLicenseData(File licenseXml, File dir) throws LicenseGeneratorException { + List warnings = new ArrayList<>(); + List writers = new ArrayList<>(); + ISpdxListedLicenseProvider licenseProvider = null; + try { + licenseProvider = new XmlLicenseProviderSingleFile(licenseXml); + File website = new File(dir.getPath()); + if (!website.isDirectory() && !website.mkdir()) { + throw new LicenseGeneratorException("Error: Website folder is not a directory"); + } + writers.add(new SpdxWebsiteFormatWriter(null, null, website)); + System.out.print("Processing License List"); + Set licenseIds = writeLicenseList(licenseProvider, warnings, writers); + System.out.println(); + System.out.print("Processing Exceptions"); + writeExceptionList(licenseProvider, warnings, writers, licenseIds); + System.out.println(); + warnings.addAll(licenseProvider.getWarnings()); + if (warnings.size() > 0) { + System.out.println("The following warning(s) were identified:"); + for (String warning : warnings) { + System.out.println("\t"+warning); + } + } + System.out.println("Completed processing licenses"); + return warnings; + } catch (SpdxListedLicenseException e) { + throw new LicenseGeneratorException("\nError reading standard licenses: "+e.getMessage(),e); + } catch (LicenseGeneratorException e) { + throw(e); + } catch (Exception e) { + throw new LicenseGeneratorException("\nUnhandled exception generating html: "+e.getMessage(),e); + } + } + + /** + * @param version License list version + * @param releaseDate release date for the license list + * @param licenseProvider Provides the licensing information + * @param warnings Populated with any warnings if they occur + * @param writers License Format Writers to handle the writing for the different formats + * @param tester License tester used to test the results of licenses + * @param licenseIds license IDs + * @param useTestText use the text file from the testFileDir for the verbatim text rather than the text from the XML document + * @throws IOException + * @throws SpreadsheetException + * @throws LicenseRestrictionException + * @throws LicenseGeneratorException + * @throws InvalidLicenseTemplateException + * @throws InvalidSPDXAnalysisException + */ + private static void writeExceptionList(ISpdxListedLicenseProvider licenseProvider, List warnings, List writers, + Set licenseIds) throws IOException, LicenseGeneratorException, InvalidLicenseTemplateException, InvalidSPDXAnalysisException { + Iterator exceptionIter = licenseProvider.getExceptionIterator(); + Map addedExceptionsMap = new HashMap<>(); + while (exceptionIter.hasNext()) { + System.out.print("."); + ListedLicenseException nextException = exceptionIter.next(); + if (nextException.getLicenseExceptionId() != null && !nextException.getLicenseExceptionId().isEmpty()) { + checkText(nextException.getLicenseExceptionText(), + "License Exception Text for "+nextException.getLicenseExceptionId(), warnings); + for (ILicenseFormatWriter writer:writers) { + writer.writeException(nextException); + } + } + } + } + + /** + * Check text for invalid characters + * @param text Text to check + * @param textDescription Description of the text being check (this will be used to form warning messages) + * @param warnings Array list of warnings to add to if an problem is found with the text + */ + private static void checkText(String text, String textDescription, + List warnings) { + BufferedReader reader = new BufferedReader(new StringReader(text)); + try { + int lineNumber = 1; + String line = reader.readLine(); + while (line != null) { + for (int i = 0; i < line.length(); i++) { + if (INVALID_TEXT_CHARS.contains(line.charAt(i))) { + warnings.add("Invalid character in " + textDescription + + " at line number " + String.valueOf(lineNumber) + + " \"" +line + "\" at character location "+String.valueOf(i)); + } + } + lineNumber++; + line = reader.readLine(); + } + } catch (IOException e) { + warnings.add("IO error reading text"); + } finally { + try { + reader.close(); + } catch (IOException e) { + warnings.add("IO Error closing string reader"); + } + } + } + + /** + * Formats and writes the license list data + * @param licenseProvider Provides the licensing information + * @param warnings Populated with any warnings if they occur + * @param writers License Format Writers to handle the writing for the different formats + * @return list of license ID's which have been added + * @throws LicenseGeneratorException + * @throws InvalidSPDXAnalysisException + * @throws IOException + * @throws SpdxListedLicenseException + * @throws SpdxCompareException + * @throws InvalidLicenseTemplateException + */ + private static Set writeLicenseList(ISpdxListedLicenseProvider licenseProvider, List warnings, + List writers) throws LicenseGeneratorException, InvalidSPDXAnalysisException, IOException, SpdxListedLicenseException, SpdxCompareException, InvalidLicenseTemplateException { + Iterator licenseIter = licenseProvider.getLicenseIterator(); + try { + Map addedLicIdTextMap = new HashMap<>(); + while (licenseIter.hasNext()) { + System.out.print("."); + SpdxListedLicense license = licenseIter.next(); + addExternalMetaData(license); + if (license.getLicenseId() != null && !license.getLicenseId().isEmpty()) { + checkText(license.getLicenseText(), "License text for "+license.getLicenseId(), warnings); + for (ILicenseFormatWriter writer : writers) { + if (writer instanceof LicenseTextFormatWriter) { + ((LicenseTextFormatWriter)(writer)).writeLicense(license, license.isDeprecated(), license.getDeprecatedVersion(), true); + } else { + writer.writeLicense(license, license.isDeprecated(), license.getDeprecatedVersion()); + } + } + } + } + return addedLicIdTextMap.keySet(); + } finally { + if (licenseIter instanceof Closeable) { + ((Closeable)licenseIter).close(); + //TODO: Is there a cleaner way to handle this? The XmlLicenseProviderWithCrossRefDetails uses executorService which must be closed + } + } + } + + /** + * Update license fields based on information from external metadata + * @param license + * @throws LicenseGeneratorException + * @throws InvalidSPDXAnalysisException + */ + private static void addExternalMetaData(SpdxListedLicense license) throws LicenseGeneratorException, InvalidSPDXAnalysisException { + license.setFsfLibre(FsfLicenseDataParser.getFsfLicenseDataParser().isSpdxLicenseFsfLibre(license.getLicenseId())); + } + + /** + * Copy a file from the resources directory to a destination file + * @param resourceFileName filename of the file in the resources directory + * @param destination target file - warning, this will be overwritten + * @throws IOException + */ + private static void copyResourceFile(String resourceFileName, File destination) throws IOException { + File resourceFile = new File(resourceFileName); + if (resourceFile.exists()) { + Files.copy(resourceFile.toPath(), destination.toPath()); + } else { + InputStream is = LicenseRDFAGenerator.class.getClassLoader().getResourceAsStream(resourceFileName); + InputStreamReader reader = new InputStreamReader(is); + FileWriter writer = new FileWriter(destination); + try { + char[] buf = new char[2048]; + int len = reader.read(buf); + while (len > 0) { + writer.write(buf, 0, len); + len = reader.read(buf); + } + } finally { + if (writer != null) { + writer.close(); + } + reader.close(); + } + } + } + + private static void usage() { + System.out.println("Usage:"); + System.out.println("LicenseSingleHTMLGenerator licenseXmlFile outputDirectory"); + System.out.println(" licencenseXmlFile - a license XML file"); + System.out.println(" outputDirectory - Directory to store the output from the license generator"); + } + +} diff --git a/src/org/spdx/licenselistpublisher/Main.java b/src/org/spdx/licenselistpublisher/Main.java index ada4e6e..4bc24a0 100644 --- a/src/org/spdx/licenselistpublisher/Main.java +++ b/src/org/spdx/licenselistpublisher/Main.java @@ -23,6 +23,8 @@ public static void main(String[] args) { args = ArrayUtils.removeElement(args, args[0]); if ("LicenseRDFAGenerator".equals(spdxTool)) { LicenseRDFAGenerator.main(args); + } else if ("LicenseSingleHTMLGenerator".equals(spdxTool)) { + LicenseSingleHTMLGenerator.main(args); } else if ("TestLicenseXML".equals(spdxTool)) { LicenseXmlTester.main(args); } else { @@ -33,6 +35,7 @@ public static void main(String[] args) { private static void usage() { System.out.println("Usage: java -jar spdx-tools-jar-with-dependencies.jar "); System.out.println("LicenseRDFAGenerator - Generates license data"); + System.out.println("LicenseSingleHTMLGenerator - Generates license data as single HTML file"); System.out.println("TestLicenseXML - Tests a license XML file"); } }