diff --git a/scg-system/src/test/files/yaml/config-abac-tdb2.yaml b/scg-system/src/test/files/yaml/config-abac-tdb2.yaml new file mode 100644 index 0000000..0baa996 --- /dev/null +++ b/scg-system/src/test/files/yaml/config-abac-tdb2.yaml @@ -0,0 +1,24 @@ +version: "1.0" +prefixes: + - prefix: "authz" + namespace: "http://telicent.io/security#" +server: + name: "Fuseki server simple" +services: + - name: "ds" + endpoints: + - operation: query + settings: + - name: "upload" + operation: upload + database: "abac-tdb2-db" + +databases: + - name: "abac-tdb2-db" + dbtype: ABAC + dataset: "dataset-under" + attributes: "attribute-store.ttl" + + - name: "dataset-under" + dbtype: TDB2 + location: "target/test-db" \ No newline at end of file diff --git a/scg-system/src/test/java/io/telicent/TestYamlConfigParserAuthz.java b/scg-system/src/test/java/io/telicent/TestYamlConfigParserAuthz.java index 82b036f..f36b91f 100644 --- a/scg-system/src/test/java/io/telicent/TestYamlConfigParserAuthz.java +++ b/scg-system/src/test/java/io/telicent/TestYamlConfigParserAuthz.java @@ -16,29 +16,51 @@ package io.telicent; +import io.telicent.jena.abac.core.Attributes; +import io.telicent.jena.abac.core.AttributesStore; import io.telicent.jena.abac.fuseki.SysFusekiABAC; +import io.telicent.jena.abac.services.SimpleAttributesStore; import io.telicent.smart.cache.configuration.Configurator; +import org.apache.jena.atlas.io.IO; +import org.apache.jena.atlas.lib.FileOps; import org.apache.jena.datatypes.xsd.XSDDatatype; +import org.apache.jena.fuseki.kafka.lib.FKLib; import org.apache.jena.fuseki.main.FusekiServer; import org.apache.jena.fuseki.system.FusekiLogging; +import org.apache.jena.graph.Graph; +import org.apache.jena.http.HttpOp; +import org.apache.jena.query.Dataset; import org.apache.jena.query.Query; import org.apache.jena.query.QueryFactory; import org.apache.jena.rdf.model.*; +import org.apache.jena.riot.Lang; +import org.apache.jena.riot.RDFDataMgr; +import org.apache.jena.riot.RDFParser; +import org.apache.jena.riot.WebContent; import org.apache.jena.sparql.core.DatasetGraphFactory; import org.apache.jena.sparql.exec.QueryExec; import org.apache.jena.sparql.exec.RowSet; import org.apache.jena.sparql.exec.RowSetOps; import org.apache.jena.sparql.exec.RowSetRewindable; import org.apache.jena.sparql.exec.http.DSP; +import org.apache.jena.sparql.exec.http.QueryExecHTTP; import org.apache.jena.sparql.exec.http.QueryExecHTTPBuilder; import org.apache.jena.sparql.resultset.ResultSetCompare; +import org.apache.jena.tdb2.TDB2Factory; import org.junit.jupiter.api.*; +import org.testcontainers.shaded.org.apache.commons.lang3.StringUtils; +import yamlconfig.ConfigConstants; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; import java.util.List; +import java.util.Properties; import static io.telicent.LibTestsSCG.*; import static io.telicent.TestSmartCacheGraphIntegration.launchServer; import static io.telicent.core.SmartCacheGraph.construct; +import static io.telicent.core.SmartCacheGraph.log; import static org.junit.jupiter.api.Assertions.*; /* @@ -48,9 +70,23 @@ class TestYamlConfigParserAuthz { private static final String DIR = "src/test/files"; private FusekiServer server; + private static MockKafka mock; - private static final String serviceName = "/ds"; + private static final String serviceName = "ds"; public static RowSetRewindable expectedRSR; + public static RowSetRewindable expectedRSRtdl; + public final String sparqlUpdateConnector = """ + PREFIX : + INSERT DATA { + :s :p4 12355 . + :s :p5 45655 . + }"""; + + Properties producerProps() { + Properties producerProps = new Properties(); + producerProps.put("bootstrap.servers", mock.getServer()); + return producerProps; + } @BeforeAll public static void before() { @@ -64,11 +100,24 @@ public static void before() { Literal l2 = comparisonModel.createTypedLiteral(789, XSDDatatype.XSDinteger); comparisonModel.add(s1, p1, l1); comparisonModel.add(s2, p2, l2); - String queryString = "SELECT ?s ?p ?o WHERE { ?s ?p ?o }"; Query query = QueryFactory.create(queryString); QueryExec qExec = QueryExec.dataset(DatasetGraphFactory.create(comparisonModel.getGraph())).query(query).build(); expectedRSR = qExec.select().rewindable(); + Resource s3 = comparisonModel.createResource(baseURI + "s"); + Property p3 = comparisonModel.createProperty(baseURI + "q"); + Literal l3 = comparisonModel.createLiteral("No label"); + comparisonModel.add(s3, p3, l3); + QueryExec qExec2 = QueryExec.dataset(DatasetGraphFactory.create(comparisonModel.getGraph())).query(query).build(); + expectedRSRtdl = qExec2.select().rewindable(); + + mock = new MockKafka(); + Properties consumerProps = new Properties(); + consumerProps.put("bootstrap.servers", mock.getServer()); + Properties producerProps = new Properties(); + producerProps.put("bootstrap.servers", mock.getServer()); + + mock.createTopic("RDF0"); } @@ -79,6 +128,7 @@ void setUp() throws Exception { SysFusekiABAC.init(); LibTestsSCG.setupAuthentication(); LibTestsSCG.disableInitialCompaction(); + expectedRSR.reset(); } @AfterEach @@ -90,92 +140,221 @@ void clearDown() throws Exception { LibTestsSCG.teardownAuthentication(); } + @Test + void yaml_config_abac_tim() { + List arguments = List.of("--conf",DIR + "/yaml/config-abac-tim.yaml"); + server = construct(arguments.toArray(new String[0])).start(); + RowSetRewindable actualResponseRSR; + String validToken = tokenForUser("u1"); + //doesn't pass unless I upload! Even though there's a data field in the file + LibTestsSCG.uploadFile(server.serverURL() + serviceName + "/upload", DIR + "/yaml/data-and-labels.trig");//load(server); + actualResponseRSR = QueryExecHTTPBuilder.service(server.serverURL() + serviceName) + .query("SELECT * { ?s ?p ?o }") + .httpHeader(LibTestsSCG.tokenHeader(), + LibTestsSCG.tokenHeaderValue(validToken)) + .select().rewindable(); + RowSetOps.out(System.out, actualResponseRSR); + boolean equals = ResultSetCompare.isomorphic(expectedRSR, actualResponseRSR); + assertTrue(equals); + } @Test - void yaml_config_abac() { - // given - List arguments = List.of("--conf",DIR + "/yaml/config-abac-multiple.yaml"); - // when + void yaml_config_abac_tdb2() { + List arguments = List.of("--conf",DIR + "/yaml/config-abac-tdb2.yaml"); server = construct(arguments.toArray(new String[0])).start(); RowSetRewindable actualResponseRSR; + String validToken = tokenForUser("u1"); + //doesn't pass unless I upload! Even though there's a data field in the file LibTestsSCG.uploadFile(server.serverURL() + serviceName + "/upload", DIR + "/yaml/data-and-labels.trig");//load(server); + //uploading this way doesn't seem to work. Could this also be the reason for the labelsStore failures? + /*Dataset dataset = TDB2Factory.connectDataset("target/test-db"); + dataset.begin(org.apache.jena.query.ReadWrite.WRITE); + try { + Model databaseModel = dataset.getDefaultModel(); + RDFDataMgr.read(databaseModel, "src/test/files/yaml/data-and-labels.trig", Lang.TURTLE); + dataset.commit(); + } catch (Exception e) { + ConfigConstants.log.error(e.getMessage()); + dataset.abort(); + } finally { + dataset.end(); + }*/ + actualResponseRSR = QueryExecHTTPBuilder.service(server.serverURL() + serviceName) + .query("SELECT * { ?s ?p ?o }") + .httpHeader(LibTestsSCG.tokenHeader(), + LibTestsSCG.tokenHeaderValue(validToken)) + .select().rewindable(); + RowSetOps.out(System.out, actualResponseRSR); + boolean equals = ResultSetCompare.isomorphic(expectedRSR, actualResponseRSR); + assertTrue(equals); + } + + @Test + void yaml_config_abac_labels_store() { + List arguments = List.of("--conf",DIR + "/yaml/config-abac-labels-store.yaml"); + server = construct(arguments.toArray(new String[0])).start(); + RowSetRewindable actualResponseRSR; String validToken = tokenForUser("u1"); + //doesn't pass unless I upload! Even though there's a data field in the file + LibTestsSCG.uploadFile(server.serverURL() + serviceName + "/upload", DIR + "/yaml/data-no-labels.trig");//load(server); + //uploading this way doesn't seem to work. Could this also be the reason for the labelsStore failures? + /*Dataset dataset = TDB2Factory.connectDataset("target/test-db"); + dataset.begin(org.apache.jena.query.ReadWrite.WRITE); + try { + Model databaseModel = dataset.getDefaultModel(); + RDFDataMgr.read(databaseModel, "src/test/files/yaml/data-and-labels.trig", Lang.TURTLE); + dataset.commit(); + } catch (Exception e) { + ConfigConstants.log.error(e.getMessage()); + dataset.abort(); + } finally { + dataset.end(); + }*/ + Dataset dataset = TDB2Factory.connectDataset("target/labels-test"); + dataset.begin(org.apache.jena.query.ReadWrite.WRITE); + try { + Model labelsStoreModel = dataset.getDefaultModel(); + RDFDataMgr.read(labelsStoreModel, "src/test/files/yaml/labels.ttl", Lang.TURTLE); + dataset.commit(); + } catch (Exception e) { + ConfigConstants.log.error(e.getMessage()); + dataset.abort(); + } finally { + dataset.end(); + } actualResponseRSR = QueryExecHTTPBuilder.service(server.serverURL() + serviceName) .query("SELECT * { ?s ?p ?o }") - .httpHeader(tokenHeader(), - tokenHeaderValue("u1")) + .httpHeader(LibTestsSCG.tokenHeader(), + LibTestsSCG.tokenHeaderValue(validToken)) .select().rewindable(); - actualResponseRSR = LibTestsSCG.queryWithToken(server.serverURL() + serviceName, "SELECT * { ?s ?p ?o }", "u1").rewindable();//query(server, "u1"); + RowSetOps.out(System.out, actualResponseRSR); + boolean equals = ResultSetCompare.isomorphic(expectedRSR, actualResponseRSR); + assertTrue(equals); + } + + @Test + void yaml_config_abac_attributes_store() { + Graph g = RDFParser.source(DIR+"/yaml/attribute-store.ttl").toGraph(); + AttributesStore attrStore = Attributes.buildStore(g); + String mockServerURL = SimpleAttributesStore.run(3132, attrStore); - // then + List arguments = List.of("--conf",DIR + "/yaml/config-abac-remote-attributes.yaml"); + server = construct(arguments.toArray(new String[0])).start(); + String validToken = tokenForUser("u1"); + //doesn't pass unless I upload! Even though there's a data field in the file + LibTestsSCG.uploadFile(server.serverURL() + serviceName + "/upload", DIR + "/yaml/data-and-labels.trig");//load(server); + + RowSetRewindable actualResponseRSR = QueryExecHTTPBuilder.service(server.serverURL() + serviceName) + .query("SELECT * { ?s ?p ?o }") + .httpHeader(LibTestsSCG.tokenHeader(), + LibTestsSCG.tokenHeaderValue(validToken)) + .select().rewindable(); boolean equals = ResultSetCompare.isomorphic(expectedRSR, actualResponseRSR); assertTrue(equals); } @Test - void yaml_config_abac_2() { - // given - List arguments = List.of("--conf",DIR + "/yaml/config-abac-tim.yaml"); - // when + void yaml_config_abac_labels() { + List arguments = List.of("--conf",DIR + "/yaml/config-abac-labels.yaml"); server = construct(arguments.toArray(new String[0])).start(); RowSetRewindable actualResponseRSR; - LibTestsSCG.uploadFile(server.serverURL() + serviceName + "/upload", DIR + "/yaml/data-and-labels.trig");//load(server); - actualResponseRSR = LibTestsSCG.queryWithToken(server.serverURL() + serviceName, "SELECT * { ?s ?p ?o }", "u1").rewindable();//query(server, "u1"); - - // then + String validToken = tokenForUser("u1"); + //doesn't pass unless I upload! Even though there's a data field in the file + LibTestsSCG.uploadFile(server.serverURL() + serviceName + "/upload", DIR + "/yaml/data-and-labels.trig");//load(server); + actualResponseRSR = QueryExecHTTPBuilder.service(server.serverURL() + serviceName) + .query("SELECT * { ?s ?p ?o }") + .httpHeader(LibTestsSCG.tokenHeader(), + LibTestsSCG.tokenHeaderValue(validToken)) + .select().rewindable(); + RowSetOps.out(System.out, actualResponseRSR); boolean equals = ResultSetCompare.isomorphic(expectedRSR, actualResponseRSR); assertTrue(equals); } @Test - void givenValidToken_whenMakingARequest_thenSuccess() { - server = launchServer("yaml/config-abac-tim.ttl"); - // given + void yaml_config_abac_triple_default_labels() { + List arguments = List.of("--conf",DIR + "/yaml/config-abac-tdl.yaml"); + server = construct(arguments.toArray(new String[0])).start(); + RowSetRewindable actualResponseRSR; String validToken = tokenForUser("u1"); - // when - RowSet rowSet = QueryExecHTTPBuilder.service(server.serverURL() + serviceName) + //doesn't pass unless I upload! Even though there's a data field in the file + LibTestsSCG.uploadFile(server.serverURL() + serviceName + "/upload", DIR + "/yaml/data-no-labels.trig");//load(server); + actualResponseRSR = QueryExecHTTPBuilder.service(server.serverURL() + serviceName) .query("SELECT * { ?s ?p ?o }") - .httpHeader(tokenHeader(), - tokenHeaderValue(validToken)) - .select(); - // then - assertNotNull(rowSet); - assertEquals(0, RowSetOps.count(rowSet)); + .httpHeader(LibTestsSCG.tokenHeader(), + LibTestsSCG.tokenHeaderValue(validToken)) + .select().rewindable(); + RowSetOps.out(System.out, actualResponseRSR); + boolean equals = ResultSetCompare.isomorphic(expectedRSRtdl, actualResponseRSR); + assertTrue(equals); } + @Test + void yaml_config_kafka_connector() { + String TOPIC = "RDF0"; + String STATE_DIR = "target/state"; + FileOps.ensureDir(STATE_DIR); + FileOps.clearDirectory(STATE_DIR); - private static void load(FusekiServer server) { - String URL = "http://localhost:" + server.getPort() + serviceName; - String uploadURL = URL + "/upload"; - load(uploadURL, DIR + "/yaml/data-and-labels.trig"); - } + File originalConfig = new File(DIR + "/yaml/config-connector-integration-test-1.yaml"); + File actualConfig = replacePlaceholder(originalConfig, "localhost:9092", mock.getServer()); + List arguments = List.of("--conf", actualConfig.getAbsolutePath()); - private static RowSetRewindable query(FusekiServer server, String user) { - String URL = "http://localhost:" + server.getPort() + serviceName; - return query(URL, user); + server = construct(arguments.toArray(new String[0])); + FKLib.sendFiles(producerProps(), TOPIC, List.of("src/test/files/yaml/data-no-labels.trig")); + server.start(); + String validToken = tokenForUser("u1"); + try { + String URL = "http://localhost:"+server.getHttpPort()+"/ds"; + RowSet rowSet = QueryExecHTTPBuilder.service(server.serverURL() + serviceName) + .query("SELECT (count(*) AS ?C) {?s ?p ?o}") + .httpHeader(LibTestsSCG.tokenHeader(), + LibTestsSCG.tokenHeaderValue(validToken)) + .select();//QueryExecHTTP.service(URL).query("SELECT (count(*) AS ?C) {?s ?p ?o}").select(); + int count = ((Number)rowSet.next().get("C").getLiteralValue()).intValue(); + Assertions.assertEquals(6, count); + HttpOp.httpPost(URL + "/update", WebContent.contentTypeSPARQLUpdate, sparqlUpdateConnector); + RowSet rowSet2 = QueryExecHTTPBuilder.service(server.serverURL() + serviceName) + .query("SELECT (count(*) AS ?C) {?s ?p ?o}") + .httpHeader(LibTestsSCG.tokenHeader(), + LibTestsSCG.tokenHeaderValue(validToken)) + .select();//QueryExecHTTP.service(URL).query("SELECT (count(*) AS ?C) {?s ?p ?o}").select(); + int count2 = ((Number)rowSet2.next().get("C").getLiteralValue()).intValue(); + Assertions.assertEquals(8, count2); + } catch (Exception e) { + throw new RuntimeException(e); + } } - private static void load(String uploadURL, String filename) { - DSP.service(uploadURL).POST(filename); + @Test + void yaml_config_custom_prefix() { + List arguments = List.of("--conf",DIR + "/yaml/config-prefixes-1.yaml"); + server = construct(arguments.toArray(new String[0])).start(); + RowSetRewindable actualResponseRSR; + String validToken = tokenForUser("u1"); + //doesn't pass unless I upload! Even though there's a data field in the file + LibTestsSCG.uploadFile(server.serverURL() + serviceName + "/upload", DIR + "/yaml/data-and-labels.trig");//load(server); + actualResponseRSR = QueryExecHTTPBuilder.service(server.serverURL() + serviceName) + .query("SELECT * { ?s ?p ?o }") + .httpHeader(LibTestsSCG.tokenHeader(), + LibTestsSCG.tokenHeaderValue(validToken)) + .select().rewindable(); + RowSetOps.out(System.out, actualResponseRSR); + boolean equals = ResultSetCompare.isomorphic(expectedRSR, actualResponseRSR); + assertTrue(equals); } - private static RowSetRewindable query(String url, String user) { - String queryString = "SELECT * { ?s ?p ?o }"; - return query(url, user, queryString); - } - private static RowSetRewindable query(String url, String user, String queryString) { - System.out.println("Query: " + user); - RowSetRewindable rowSet = - QueryExecHTTPBuilder.service(url) - .query(queryString) - .httpHeader("Authorization", "Bearer user:" + user) - .select() - .rewindable(); - long x = RowSetOps.count(rowSet); - System.out.printf("User = %s ; returned %d results\n", user, x); - rowSet.reset(); - RowSetOps.out(System.out, rowSet); - return rowSet; + public static File replacePlaceholder(File input, String find, String replace) { + try { + File output = + Files.createTempFile("temp", input.getName().substring(input.getName().lastIndexOf('.'))).toFile(); + String contents = IO.readWholeFileAsUTF8(input.getAbsolutePath()); + contents = StringUtils.replace(contents, find, replace); + IO.writeStringAsUTF8(output.getAbsolutePath(), contents); + return output; + } catch (IOException e) { + throw new RuntimeException(e); + } } }