Skip to content
New issue

Have a question about this project? # for a free GitHub account to open an issue and contact its maintainers and the community.

#

By clicking “#”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? # to your account

add api for fetching federated query configs #339

Merged
merged 5 commits into from
Jul 10, 2024
Merged

add api for fetching federated query configs #339

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
38b0fef
add api for fetching federated query configs
tiana528 Jul 8, 2024
8f67232
fix
tiana528 Jul 9, 2024
49987af
fix
tiana528 Jul 9, 2024
f22aef7
fix
tiana528 Jul 9, 2024
756c684
fix format
tiana528 Jul 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions src/main/java/com/treasuredata/client/TDClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
import com.treasuredata.client.model.TDDatabase;
import com.treasuredata.client.model.TDExportJobRequest;
import com.treasuredata.client.model.TDExportResultJobRequest;
import com.treasuredata.client.model.TDFederatedQueryConfig;
import com.treasuredata.client.model.TDImportResult;
import com.treasuredata.client.model.TDJob;
import com.treasuredata.client.model.TDJobList;
Expand Down Expand Up @@ -1137,4 +1138,11 @@ public TDImportResult importBytes(String databaseName, String tableName, byte[]
{
return doPut(buildUrl(String.format("/v3/table/import_with_id/%s/%s/%s/%s", databaseName, tableName, id, "msgpack.gz")), Collections.emptyMap(), content, offset, length, TDImportResult.class);
}

@Override
public List<TDFederatedQueryConfig> getFederatedQueryConfigs()
throws TDClientException
{
return doGet("/v4/federated_query_configs", new TypeReference<List<TDFederatedQueryConfig>>() {});
}
}
7 changes: 7 additions & 0 deletions src/main/java/com/treasuredata/client/TDClientApi.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
import com.treasuredata.client.model.TDDatabase;
import com.treasuredata.client.model.TDExportJobRequest;
import com.treasuredata.client.model.TDExportResultJobRequest;
import com.treasuredata.client.model.TDFederatedQueryConfig;
import com.treasuredata.client.model.TDImportResult;
import com.treasuredata.client.model.TDJob;
import com.treasuredata.client.model.TDJobList;
Expand Down Expand Up @@ -501,4 +502,10 @@ default <Result> Result getBulkImportErrorRecords(String sessionName, com.google
* @return TDImportResult which contains a unique import id and md5
*/
TDImportResult importBytes(String database, String table, byte[] content, int offset, int length, String id);

/**
* Fetch a list of federated query configurations
* @return List<TDFederatedQueryConfig> which contains a list of federated query configs
*/
List<TDFederatedQueryConfig> getFederatedQueryConfigs();
}
20 changes: 20 additions & 0 deletions ...in/java/com/treasuredata/client/deserialize/FederatedQueryConfigSettingsDeserializer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
package com.treasuredata.client.deserialize;

import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.databind.DeserializationContext;
import com.fasterxml.jackson.databind.JsonDeserializer;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;

import java.io.IOException;

public class FederatedQueryConfigSettingsDeserializer extends JsonDeserializer<String>
{
@Override
public String deserialize(JsonParser p, DeserializationContext ctxt) throws IOException
{
ObjectMapper mapper = (ObjectMapper) p.getCodec();
ObjectNode root = mapper.readTree(p);
return root.toString();
}
}
145 changes: 145 additions & 0 deletions src/main/java/com/treasuredata/client/model/TDFederatedQueryConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,145 @@
package com.treasuredata.client.model;

import java.util.Objects;

import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;

public class TDFederatedQueryConfig
{
private final int id;
private final String type;
private final int userId;
private final int accountId;
private final String name;
private final int connectionId;
private final String createdAt;
private final String updatedAt;
@JsonDeserialize(using = com.treasuredata.client.deserialize.FederatedQueryConfigSettingsDeserializer.class)
private final String settings;

@JsonCreator
public TDFederatedQueryConfig(
@JsonProperty("id") int id,
@JsonProperty("type") String type,
@JsonProperty("user_id") int userId,
@JsonProperty("account_id") int accountId,
@JsonProperty("name") String name,
@JsonProperty("connection_id") int connectionId,
@JsonProperty("created_at") String createdAt,
@JsonProperty("updated_at") String updatedAt,
@JsonProperty("settings") String settings)
{
this.id = id;
this.type = type;
this.userId = userId;
this.accountId = accountId;
this.name = name;
this.connectionId = connectionId;
this.createdAt = createdAt;
this.updatedAt = updatedAt;
this.settings = settings;
}

@JsonProperty
public int getId()
{
return id;
}

@JsonProperty
public String getType()
{
return type;
}

@JsonProperty("user_id")
public int getUserId()
{
return userId;
}

@JsonProperty("account_id")
public int getAccountId()
{
return accountId;
}

@JsonProperty
public String getName()
{
return name;
}

@JsonProperty("connection_id")
public int getConnectionId()
{
return connectionId;
}

@JsonProperty("created_at")
public String getCreatedAt()
{
return createdAt;
}

@JsonProperty("updated_at")
public String getUpdatedAt()
{
return updatedAt;
}

@JsonProperty
public String getSettings()
{
return settings;
}

@Override
public final boolean equals(Object o)
{
if (this == o) {
return true;
}
if (!(o instanceof TDFederatedQueryConfig)) {
return false;
}
TDFederatedQueryConfig that = (TDFederatedQueryConfig) o;
return id == that.id && userId == that.userId && accountId == that.accountId
&& connectionId == that.connectionId && Objects.equals(type, that.type) && Objects.equals(
name, that.name) && Objects.equals(createdAt, that.createdAt) && Objects.equals(updatedAt,
that.updatedAt) && Objects.equals(settings, that.settings);
}

@Override
public int hashCode()
{
int result = id;
result = 31 * result + Objects.hashCode(type);
result = 31 * result + userId;
result = 31 * result + accountId;
result = 31 * result + Objects.hashCode(name);
result = 31 * result + connectionId;
result = 31 * result + Objects.hashCode(createdAt);
result = 31 * result + Objects.hashCode(updatedAt);
result = 31 * result + Objects.hashCode(settings);
return result;
}

@Override
public String toString()
{
return "TDFederatedQueryConfig{" +
"id=" + id +
", type='" + type + '\'' +
", userId=" + userId +
", accountId=" + accountId +
", name='" + name + '\'' +
", connectionId=" + connectionId +
", createdAt='" + createdAt + '\'' +
", updatedAt='" + updatedAt + '\'' +
", settings='" + settings + '\'' +
'}';
}
}
45 changes: 45 additions & 0 deletions src/test/java/com/treasuredata/client/TestTDClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
import com.treasuredata.client.model.TDExportFileFormatType;
import com.treasuredata.client.model.TDExportJobRequest;
import com.treasuredata.client.model.TDExportResultJobRequest;
import com.treasuredata.client.model.TDFederatedQueryConfig;
import com.treasuredata.client.model.TDImportResult;
import com.treasuredata.client.model.TDJob;
import com.treasuredata.client.model.TDJob.EngineVersion;
Expand Down Expand Up @@ -1958,6 +1959,50 @@ public void testImportBytesWithId()
assertEquals(result.getUniqueId(), "4288048cf8f811e88b560a87157ac806");
}

@Test
public void testGetFederatedQueryConfigsWhenEmpty()
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could you add test cases that the endpoint return 4xx status code?
The endpoint returns 401 and 403.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It is not very convenient to simulate 401 and 403. I added a test case of 401, while it depends on the internal behaviour.
To support 401 and 403 in the test case, we need to configure the account of staging-aws used by the unit tests in a way to trigger 401 and 403 (Some junit tests in this class actually sends real request to staging-aws api3), which I feel might be a little too much.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

sure, I see.

throws Exception
{
client = mockClient();
server.enqueue(new MockResponse().setBody("[]"));

List<TDFederatedQueryConfig> result = client.getFederatedQueryConfigs();

assertTrue(result.isEmpty());
}

@Test
public void testGetFederatedQueryConfigsWhenOnlyOne()
throws Exception
{
client = mockClient();
final String federatedQueryConfigs = "[{\"id\":12345,\"type\":\"custom_type\",\"user_id\":67890,\"account_id\":54321,\"name\":\"test_name\",\"connection_id\":112233,\"created_at\":\"2023-06-01T00:00:00Z\",\"updated_at\":\"2023-06-15T00:00:00Z\",\"settings\":{\"account_name\":\"snowflake_account\",\"auth_method\":\"password\",\"private_key\":\"encrypted_key\",\"passphrase\":\"secret_phrase\",\"options\":{\"key1\":\"val1\",\"key2\":\"val2\"},\"user\":\"snowflake_user\",\"password\":\"snowflake_password\",\"database\":\"sample_database\"}}]";
server.enqueue(new MockResponse().setBody(federatedQueryConfigs));

List<TDFederatedQueryConfig> result = client.getFederatedQueryConfigs();

List<TDFederatedQueryConfig> expected = Arrays.asList(new TDFederatedQueryConfig(12345, "custom_type", 67890, 54321, "test_name", 112233, "2023-06-01T00:00:00Z", "2023-06-15T00:00:00Z", "{\"account_name\":\"snowflake_account\",\"auth_method\":\"password\",\"private_key\":\"encrypted_key\",\"passphrase\":\"secret_phrase\",\"options\":{\"key1\":\"val1\",\"key2\":\"val2\"},\"user\":\"snowflake_user\",\"password\":\"snowflake_password\",\"database\":\"sample_database\"}"));
assertEquals(result, expected);
}

@Test
public void testGetFederatedQueryConfigsWhenMultiple()
throws Exception
{
client = mockClient();
final String federatedQueryConfigs = "[{\"id\":12345,\"type\":\"custom_type\",\"user_id\":67890,\"account_id\":54321,\"name\":\"test_name\",\"connection_id\":112233,\"created_at\":\"2023-06-01T00:00:00Z\",\"updated_at\":\"2023-06-15T00:00:00Z\",\"settings\":{\"account_name\":\"snowflake_account\",\"auth_method\":\"password\",\"private_key\":\"encrypted_key\",\"passphrase\":\"secret_phrase\",\"options\":{\"key1\":\"val1\",\"key2\":\"val2\"},\"user\":\"snowflake_user\",\"password\":\"snowflake_password\",\"database\":\"sample_database\"}},"
+ "{\"id\":67890,\"type\":\"another_type\",\"user_id\":12345,\"account_id\":98765,\"name\":\"another_test_name\",\"connection_id\":445566,\"created_at\":\"2023-07-01T00:00:00Z\",\"updated_at\":\"2023-07-15T00:00:00Z\",\"settings\":{\"account_name\":\"another_snowflake_account\",\"auth_method\":\"oauth\",\"private_key\":\"another_encrypted_key\",\"passphrase\":\"another_secret_phrase\",\"options\":{\"keyA\":\"valA\",\"keyB\":\"valB\"},\"user\":\"another_snowflake_user\",\"password\":\"another_snowflake_password\",\"database\":\"another_sample_database\"}}]";
server.enqueue(new MockResponse().setBody(federatedQueryConfigs));

List<TDFederatedQueryConfig> result = client.getFederatedQueryConfigs();

List<TDFederatedQueryConfig> expected = Arrays.asList(
new TDFederatedQueryConfig(12345, "custom_type", 67890, 54321, "test_name", 112233, "2023-06-01T00:00:00Z", "2023-06-15T00:00:00Z", "{\"account_name\":\"snowflake_account\",\"auth_method\":\"password\",\"private_key\":\"encrypted_key\",\"passphrase\":\"secret_phrase\",\"options\":{\"key1\":\"val1\",\"key2\":\"val2\"},\"user\":\"snowflake_user\",\"password\":\"snowflake_password\",\"database\":\"sample_database\"}"),
new TDFederatedQueryConfig(67890, "another_type", 12345, 98765, "another_test_name", 445566, "2023-07-01T00:00:00Z", "2023-07-15T00:00:00Z", "{\"account_name\":\"another_snowflake_account\",\"auth_method\":\"oauth\",\"private_key\":\"another_encrypted_key\",\"passphrase\":\"another_secret_phrase\",\"options\":{\"keyA\":\"valA\",\"keyB\":\"valB\"},\"user\":\"another_snowflake_user\",\"password\":\"another_snowflake_password\",\"database\":\"another_sample_database\"}")
);
assertEquals(result, expected);
}

private File createTempMsgpackGz(String prefix, int numRows)
throws IOException
{
Expand Down
Loading