From 8af0e0ac08c035f0a72d5682972e55d33c8952a3 Mon Sep 17 00:00:00 2001 From: Giuseppe Villani Date: Thu, 16 Feb 2023 14:31:31 +0100 Subject: [PATCH] [TOQo1n9M] apoc-hadoop dependency is conflicting (#3450) --- test-startup/src/test/java/StartupTest.java | 66 +++++++++++++++++++ .../java/apoc/util/TestContainerUtil.java | 64 +++++++++++++----- 2 files changed, 112 insertions(+), 18 deletions(-) diff --git a/test-startup/src/test/java/StartupTest.java b/test-startup/src/test/java/StartupTest.java index 0cb5c32634..c4bbe924af 100644 --- a/test-startup/src/test/java/StartupTest.java +++ b/test-startup/src/test/java/StartupTest.java @@ -2,13 +2,20 @@ import apoc.util.Neo4jContainerExtension; import apoc.util.TestContainerUtil; import apoc.util.TestUtil; +import org.apache.commons.io.FileUtils; import apoc.util.TestContainerUtil.Neo4jVersion; import apoc.util.TestContainerUtil.ApocPackage; import org.junit.Test; import org.neo4j.driver.Session; +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Paths; import java.util.List; +import java.util.function.Consumer; +import java.util.function.Supplier; import java.util.stream.Collectors; import static apoc.util.TestContainerUtil.createDB; @@ -22,6 +29,9 @@ */ public class StartupTest { + private static final File APOC_FULL; + private static final File APOC_CORE; + @Test public void check_basic_deployment() { for (var version: Neo4jVersion.values()) { @@ -80,6 +90,62 @@ public void compare_with_sources() { } } + @Test + public void checkFullWithExtraDependenciesJars() throws IOException { + + // we retrieve every full procedure and function via the extended.txt file + final File extendedFile = new File(APOC_FULL, "src/main/resources/extended.txt"); + final List expectedFullProcAndFunNames = FileUtils.readLines(extendedFile, StandardCharsets.UTF_8); + + // we check that with apoc-full jar and all extra-dependencies jars every procedure/function is detected + startNeo4jContainerSession(() -> createEnterpriseDB(APOC_FULL, !TestUtil.isRunningInCI(), true), + session -> { + checkCoreProcsAndFuncsExistence(session); + + // all full procedures and functions are present, also the ones which require extra-deps, e.g. the apoc.export.xls.* + final List actualFullProcAndFunNames = session.run("CALL apoc.help('') YIELD core, type, name WHERE core = false RETURN name").list(i -> i.get("name").asString()); + assertEquals(sorted(expectedFullProcAndFunNames), sorted(actualFullProcAndFunNames)); + }); + } + + @Test + public void checkCoreWithExtraDependenciesJars() { + // we check that with apoc-core jar and all extra-dependencies jars every procedure/function is detected + startNeo4jContainerSession(() -> createEnterpriseDB(APOC_CORE, !TestUtil.isRunningInCI(), true), + this::checkCoreProcsAndFuncsExistence); + + } + + private void startNeo4jContainerSession(Supplier neo4jContainerCreation, + Consumer sessionConsumer) { + try (final Neo4jContainerExtension neo4jContainer = neo4jContainerCreation.get()) { + neo4jContainer.start(); + assertTrue("Neo4j Instance should be up-and-running", neo4jContainer.isRunning()); + + final Session session = neo4jContainer.getSession(); + + sessionConsumer.accept(session); + } catch (Exception ex) { + // if Testcontainers wasn't able to retrieve the docker image we ignore the test + if (TestContainerUtil.isDockerImageAvailable(ex)) { + ex.printStackTrace(); + fail("Should not have thrown exception when trying to start Neo4j: " + ex); + } else { + fail( "The docker image could not be loaded. Check whether it's available locally / in the CI. Exception:" + ex); + } + } + } + + private void checkCoreProcsAndFuncsExistence(Session session) { + final List functionNames = session.run("CALL apoc.help('') YIELD core, type, name WHERE core = true and type = 'function' RETURN name") + .list(record -> record.get("name").asString()); + final List procedureNames = session.run("CALL apoc.help('') YIELD core, type, name WHERE core = true and type = 'procedure' RETURN name") + .list(record -> record.get("name").asString()); + + assertEquals(sorted(ApocSignatures.PROCEDURES), procedureNames); + assertEquals(sorted(ApocSignatures.FUNCTIONS), functionNames); + } + private List sorted(List signatures) { return signatures.stream().sorted().collect(Collectors.toList()); } diff --git a/test-utils/src/main/java/apoc/util/TestContainerUtil.java b/test-utils/src/main/java/apoc/util/TestContainerUtil.java index 6996e51866..43c7f522bf 100644 --- a/test-utils/src/main/java/apoc/util/TestContainerUtil.java +++ b/test-utils/src/main/java/apoc/util/TestContainerUtil.java @@ -2,6 +2,8 @@ import com.github.dockerjava.api.exception.NotFoundException; import org.apache.commons.io.FileUtils; +import org.apache.commons.io.filefilter.IOFileFilter; +import org.apache.commons.io.filefilter.TrueFileFilter; import org.apache.commons.io.filefilter.WildcardFileFilter; import org.apache.commons.lang3.exception.ExceptionUtils; import org.gradle.tooling.BuildLauncher; @@ -46,6 +48,7 @@ public enum ApocPackage { private TestContainerUtil() {} + private static File pluginsFolder; private static File baseDir = Paths.get("..").toFile(); private static File coreDir = new File(baseDir, "core"); private static File extendedDir = new File(baseDir, "extended"); @@ -68,8 +71,24 @@ public static Neo4jContainerExtension createEnterpriseDB(List apocP public static Neo4jContainerExtension createCommunityDB(List apocPackages, boolean withLogging) { return createNeo4jContainer(apocPackages, withLogging, Neo4jVersion.COMMUNITY); } + + private static void addExtraDependencies() { + final File projectRootDir = Paths.get("..").toFile(); + File extraDepsDir = new File(projectRootDir, "extra-dependencies"); + // build the extra-dependencies + executeGradleTasks(extraDepsDir, "buildDependencies"); + + // add all extra deps to the plugin docker folder + final File directory = new File(extraDepsDir, "build/allJars"); + final IOFileFilter instance = TrueFileFilter.TRUE; + copyFilesToPlugin(directory, instance); + } + + public static Neo4jContainerExtension createEnterpriseDB(File baseDir, boolean withLogging, Neo4jVersion version) { + return createEnterpriseDB(baseDir, withLogging, version, false); + } - private static Neo4jContainerExtension createNeo4jContainer(List apocPackages, boolean withLogging, Neo4jVersion version) { + private static Neo4jContainerExtension createNeo4jContainer(List apocPackages, boolean withLogging, Neo4jVersion version, boolean withExtraDeps) { String dockerImage; if (version == Neo4jVersion.ENTERPRISE) { dockerImage = neo4jEnterpriseDockerImageVersion; @@ -88,6 +107,7 @@ private static Neo4jContainerExtension createNeo4jContainer(List ap File importFolder = new File("import"); importFolder.mkdirs(); // use a separate folder for mounting plugins jar - build/libs might contain other jars as well. + pluginsFolder = new File(baseDir, "build/plugins"); pluginsFolder.mkdirs(); String canonicalPath = null; @@ -97,23 +117,20 @@ private static Neo4jContainerExtension createNeo4jContainer(List ap e.printStackTrace(); } - for (ApocPackage apocPackage: apocPackages) { - if (apocPackage == ApocPackage.CORE) { - projectDir = coreDir; - } else { - projectDir = extendedDir; - } - - executeGradleTasks(projectDir, "shadowJar"); - - Collection files = FileUtils.listFiles(new File(projectDir, "build/libs"), new WildcardFileFilter(Arrays.asList("*-extended.jar", "*-core.jar")), null); - for (File file: files) { - try { - FileUtils.copyFileToDirectory(file, pluginsFolder); - } catch (IOException e) { - throw new RuntimeException(e); - } - } + final File directory = new File(baseDir, "build/libs"); + final IOFileFilter fileFilter = new WildcardFileFilter(Arrays.asList("*-extended.jar", "*-core.jar")); + + copyFilesToPlugin(directory, fileFilter); + + if (withExtraDeps) { + addExtraDependencies(); + } + + String canonicalPath = null; + try { + canonicalPath = importFolder.getCanonicalPath(); + } catch (IOException e) { + e.printStackTrace(); } System.out.println("neo4jDockerImageVersion = " + dockerImage); @@ -156,6 +173,17 @@ private static Neo4jContainerExtension createNeo4jContainer(List ap return neo4jContainer.withWaitForNeo4jDatabaseReady(password, version); } + private static void copyFilesToPlugin(File directory, IOFileFilter instance) { + Collection files = FileUtils.listFiles(directory, instance, null); + for (File file: files) { + try { + FileUtils.copyFileToDirectory(file, pluginsFolder); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + public static void executeGradleTasks(File baseDir, String... tasks) { try (ProjectConnection connection = GradleConnector.newConnector() .forProjectDirectory(baseDir)