From 1f640cb08e3a833e4bd7858e3b5042ceb4c7f140 Mon Sep 17 00:00:00 2001 From: agelostsal Date: Tue, 27 Jun 2023 10:49:30 +0300 Subject: [PATCH 1/9] AM-326 ams:Introduce trace id identifier in logs --- auth/acl.go | 25 +- auth/auth_test.go | 225 ++++++------ auth/users.go | 174 +++++----- brokers/broker.go | 10 +- brokers/kafka.go | 24 +- brokers/mock.go | 12 +- handlers/errors.go | 42 +-- handlers/handlers.go | 70 ++-- handlers/metrics.go | 107 +++--- handlers/projects.go | 196 ++++++----- handlers/projects_test.go | 7 +- handlers/registrations.go | 95 +++-- handlers/registrations_test.go | 3 +- handlers/schemas.go | 87 +++-- handlers/subscriptions.go | 409 +++++++++++++--------- handlers/subscriptions_test.go | 47 +-- handlers/topics.go | 199 ++++++----- handlers/topics_test.go | 13 +- handlers/users.go | 129 ++++--- metrics/metrics_test.go | 21 +- metrics/operational.go | 57 ++- metrics/queries.go | 69 ++-- projects/project.go | 57 +-- projects/project_test.go | 44 +-- push/push.go | 16 +- schemas/schema.go | 31 +- schemas/schema_test.go | 24 +- stores/mock.go | 165 ++++----- stores/mongo.go | 535 ++++++++++------------------- stores/store.go | 135 ++++---- stores/store_test.go | 285 +++++++-------- subscriptions/subscription.go | 102 ++++-- subscriptions/subscription_test.go | 67 ++-- topics/topic.go | 51 +-- topics/topic_test.go | 41 ++- 35 files changed, 1888 insertions(+), 1686 deletions(-) diff --git a/auth/acl.go b/auth/acl.go index 0097608e..f88c6ad0 100644 --- a/auth/acl.go +++ b/auth/acl.go @@ -1,6 +1,7 @@ package auth import ( + "context" "encoding/json" "errors" @@ -32,55 +33,55 @@ func GetACLFromJSON(input []byte) (ACL, error) { } // ModACL is called to modify an acl -func ModACL(projectUUID string, resourceType string, resourceName string, acl []string, store stores.Store) error { +func ModACL(ctx context.Context, projectUUID string, resourceType string, resourceName string, acl []string, store stores.Store) error { // Transform user name to user uuid userUUIDs := []string{} for _, username := range acl { - userUUID := GetUUIDByName(username, store) + userUUID := GetUUIDByName(ctx, username, store) userUUIDs = append(userUUIDs, userUUID) } - return store.ModACL(projectUUID, resourceType, resourceName, userUUIDs) + return store.ModACL(ctx, projectUUID, resourceType, resourceName, userUUIDs) } // AppendToACL is used to append unique users to a topic's or sub's ACL -func AppendToACL(projectUUID string, resourceType string, resourceName string, acl []string, store stores.Store) error { +func AppendToACL(ctx context.Context, projectUUID string, resourceType string, resourceName string, acl []string, store stores.Store) error { // Transform user name to user uuid userUUIDs := []string{} for _, username := range acl { - userUUID := GetUUIDByName(username, store) + userUUID := GetUUIDByName(ctx, username, store) userUUIDs = append(userUUIDs, userUUID) } - return store.AppendToACL(projectUUID, resourceType, resourceName, userUUIDs) + return store.AppendToACL(ctx, projectUUID, resourceType, resourceName, userUUIDs) } // AppendToACL is used to remove users from a topic's or sub's acl -func RemoveFromACL(projectUUID string, resourceType string, resourceName string, acl []string, store stores.Store) error { +func RemoveFromACL(ctx context.Context, projectUUID string, resourceType string, resourceName string, acl []string, store stores.Store) error { // Transform user name to user uuid userUUIDs := []string{} for _, username := range acl { - userUUID := GetUUIDByName(username, store) + userUUID := GetUUIDByName(ctx, username, store) userUUIDs = append(userUUIDs, userUUID) } - return store.RemoveFromACL(projectUUID, resourceType, resourceName, userUUIDs) + return store.RemoveFromACL(ctx, projectUUID, resourceType, resourceName, userUUIDs) } // GetACL returns an authorized list of user for the resource (topic or subscription) -func GetACL(projectUUID string, resourceType string, resourceName string, store stores.Store) (ACL, error) { +func GetACL(ctx context.Context, projectUUID string, resourceType string, resourceName string, store stores.Store) (ACL, error) { result := ACL{} - acl, err := store.QueryACL(projectUUID, resourceType, resourceName) + acl, err := store.QueryACL(ctx, projectUUID, resourceType, resourceName) if err != nil { return result, err } for _, item := range acl.ACL { // Get Username from user uuid - username := GetNameByUUID(item, store) + username := GetNameByUUID(ctx, item, store) // if username is empty, meaning that the user with this id probably doesn't exists // skip it and don't pollute the acl with empty "" if username == "" { diff --git a/auth/auth_test.go b/auth/auth_test.go index f95162af..a53eb4d3 100644 --- a/auth/auth_test.go +++ b/auth/auth_test.go @@ -1,6 +1,7 @@ package auth import ( + "context" "errors" "io/ioutil" "testing" @@ -16,9 +17,11 @@ import ( type AuthTestSuite struct { suite.Suite cfgStr string + ctx context.Context } func (suite *AuthTestSuite) SetupTest() { + suite.ctx = context.Background() suite.cfgStr = `{ "broker_host":"localhost:9092", "store_host":"localhost", @@ -30,21 +33,21 @@ func (suite *AuthTestSuite) SetupTest() { func (suite *AuthTestSuite) TestAuth() { store := stores.NewMockStore("mockhost", "mockbase") - authen01, user01 := Authenticate("argo_uuid", "S3CR3T1", store) - authen02, user02 := Authenticate("argo_uuid", "falseSECRET", store) + authen01, user01 := Authenticate(suite.ctx, "argo_uuid", "S3CR3T1", store) + authen02, user02 := Authenticate(suite.ctx, "argo_uuid", "falseSECRET", store) suite.Equal("UserA", user01) suite.Equal("", user02) suite.Equal([]string{"consumer", "publisher"}, authen01) suite.Equal([]string{}, authen02) - suite.Equal(true, Authorize("topics:list_all", []string{"admin"}, store)) - suite.Equal(true, Authorize("topics:list_all", []string{"admin", "reader"}, store)) - suite.Equal(true, Authorize("topics:list_all", []string{"admin", "foo"}, store)) - suite.Equal(false, Authorize("topics:list_all", []string{"foo"}, store)) - suite.Equal(false, Authorize("topics:publish", []string{"reader"}, store)) - suite.Equal(true, Authorize("topics:list_all", []string{"admin"}, store)) - suite.Equal(true, Authorize("topics:list_all", []string{"publisher"}, store)) - suite.Equal(true, Authorize("topics:publish", []string{"publisher"}, store)) + suite.Equal(true, Authorize(suite.ctx, "topics:list_all", []string{"admin"}, store)) + suite.Equal(true, Authorize(suite.ctx, "topics:list_all", []string{"admin", "reader"}, store)) + suite.Equal(true, Authorize(suite.ctx, "topics:list_all", []string{"admin", "foo"}, store)) + suite.Equal(false, Authorize(suite.ctx, "topics:list_all", []string{"foo"}, store)) + suite.Equal(false, Authorize(suite.ctx, "topics:publish", []string{"reader"}, store)) + suite.Equal(true, Authorize(suite.ctx, "topics:list_all", []string{"admin"}, store)) + suite.Equal(true, Authorize(suite.ctx, "topics:list_all", []string{"publisher"}, store)) + suite.Equal(true, Authorize(suite.ctx, "topics:publish", []string{"publisher"}, store)) // Check user authorization per topic // @@ -53,24 +56,24 @@ func (suite *AuthTestSuite) TestAuth() { // topic3: userC // Check authorization per topic for userA - suite.Equal(true, PerResource("argo_uuid", "topics", "topic1", "uuid1", store)) - suite.Equal(true, PerResource("argo_uuid", "topics", "topic2", "uuid1", store)) - suite.Equal(false, PerResource("argo_uuid", "topics", "topic3", "uuid1", store)) + suite.Equal(true, PerResource(suite.ctx, "argo_uuid", "topics", "topic1", "uuid1", store)) + suite.Equal(true, PerResource(suite.ctx, "argo_uuid", "topics", "topic2", "uuid1", store)) + suite.Equal(false, PerResource(suite.ctx, "argo_uuid", "topics", "topic3", "uuid1", store)) // Check authorization per topic for userB - suite.Equal(true, PerResource("argo_uuid", "topics", "topic1", "uuid2", store)) - suite.Equal(true, PerResource("argo_uuid", "topics", "topic2", "uuid2", store)) - suite.Equal(false, PerResource("argo_uuid", "topics", "topic3", "uuid2", store)) + suite.Equal(true, PerResource(suite.ctx, "argo_uuid", "topics", "topic1", "uuid2", store)) + suite.Equal(true, PerResource(suite.ctx, "argo_uuid", "topics", "topic2", "uuid2", store)) + suite.Equal(false, PerResource(suite.ctx, "argo_uuid", "topics", "topic3", "uuid2", store)) // Check authorization per topic for userC - suite.Equal(false, PerResource("argo_uuid", "topics", "topic1", "uuid3", store)) - suite.Equal(false, PerResource("argo_uuid", "topics", "topic2", "uuid3", store)) - suite.Equal(true, PerResource("argo_uuid", "topics", "topic3", "uuid3", store)) + suite.Equal(false, PerResource(suite.ctx, "argo_uuid", "topics", "topic1", "uuid3", store)) + suite.Equal(false, PerResource(suite.ctx, "argo_uuid", "topics", "topic2", "uuid3", store)) + suite.Equal(true, PerResource(suite.ctx, "argo_uuid", "topics", "topic3", "uuid3", store)) // Check authorization per topic for userD - suite.Equal(false, PerResource("argo_uuid", "topics", "topic1", "uuid4", store)) - suite.Equal(true, PerResource("argo_uuid", "topics", "topic2", "uuid4", store)) - suite.Equal(false, PerResource("argo_uuid", "topics", "topic3", "uuid4", store)) + suite.Equal(false, PerResource(suite.ctx, "argo_uuid", "topics", "topic1", "uuid4", store)) + suite.Equal(true, PerResource(suite.ctx, "argo_uuid", "topics", "topic2", "uuid4", store)) + suite.Equal(false, PerResource(suite.ctx, "argo_uuid", "topics", "topic3", "uuid4", store)) // Check user authorization per subscription // @@ -80,26 +83,26 @@ func (suite *AuthTestSuite) TestAuth() { // sub4: userB, userD // Check authorization per subscription for userA - suite.Equal(true, PerResource("argo_uuid", "subscriptions", "sub1", "uuid1", store)) - suite.Equal(true, PerResource("argo_uuid", "subscriptions", "sub2", "uuid1", store)) - suite.Equal(true, PerResource("argo_uuid", "subscriptions", "sub3", "uuid1", store)) - suite.Equal(false, PerResource("argo_uuid", "subscriptions", "sub4", "uuid1", store)) + suite.Equal(true, PerResource(suite.ctx, "argo_uuid", "subscriptions", "sub1", "uuid1", store)) + suite.Equal(true, PerResource(suite.ctx, "argo_uuid", "subscriptions", "sub2", "uuid1", store)) + suite.Equal(true, PerResource(suite.ctx, "argo_uuid", "subscriptions", "sub3", "uuid1", store)) + suite.Equal(false, PerResource(suite.ctx, "argo_uuid", "subscriptions", "sub4", "uuid1", store)) // Check authorization per subscription for userB - suite.Equal(true, PerResource("argo_uuid", "subscriptions", "sub1", "uuid2", store)) - suite.Equal(false, PerResource("argo_uuid", "subscriptions", "sub2", "uuid2", store)) - suite.Equal(true, PerResource("argo_uuid", "subscriptions", "sub3", "uuid2", store)) - suite.Equal(true, PerResource("argo_uuid", "subscriptions", "sub4", "uuid2", store)) + suite.Equal(true, PerResource(suite.ctx, "argo_uuid", "subscriptions", "sub1", "uuid2", store)) + suite.Equal(false, PerResource(suite.ctx, "argo_uuid", "subscriptions", "sub2", "uuid2", store)) + suite.Equal(true, PerResource(suite.ctx, "argo_uuid", "subscriptions", "sub3", "uuid2", store)) + suite.Equal(true, PerResource(suite.ctx, "argo_uuid", "subscriptions", "sub4", "uuid2", store)) // Check authorization per subscription for userC - suite.Equal(false, PerResource("argo_uuid", "subscriptions", "sub1", "uuid3", store)) - suite.Equal(true, PerResource("argo_uuid", "subscriptions", "sub2", "uuid3", store)) - suite.Equal(false, PerResource("argo_uuid", "subscriptions", "sub3", "uuid3", store)) - suite.Equal(false, PerResource("argo_uuid", "subscriptions", "sub4", "uuid3", store)) + suite.Equal(false, PerResource(suite.ctx, "argo_uuid", "subscriptions", "sub1", "uuid3", store)) + suite.Equal(true, PerResource(suite.ctx, "argo_uuid", "subscriptions", "sub2", "uuid3", store)) + suite.Equal(false, PerResource(suite.ctx, "argo_uuid", "subscriptions", "sub3", "uuid3", store)) + suite.Equal(false, PerResource(suite.ctx, "argo_uuid", "subscriptions", "sub4", "uuid3", store)) // Check authorization per subscription for userD - suite.Equal(false, PerResource("argo_uuid", "subscriptions", "sub1", "uuid4", store)) - suite.Equal(false, PerResource("argo_uuid", "subscriptions", "sub2", "uuid4", store)) - suite.Equal(true, PerResource("argo_uuid", "subscriptions", "sub3", "uuid4", store)) - suite.Equal(true, PerResource("argo_uuid", "subscriptions", "sub4", "uuid4", store)) + suite.Equal(false, PerResource(suite.ctx, "argo_uuid", "subscriptions", "sub1", "uuid4", store)) + suite.Equal(false, PerResource(suite.ctx, "argo_uuid", "subscriptions", "sub2", "uuid4", store)) + suite.Equal(true, PerResource(suite.ctx, "argo_uuid", "subscriptions", "sub3", "uuid4", store)) + suite.Equal(true, PerResource(suite.ctx, "argo_uuid", "subscriptions", "sub4", "uuid4", store)) suite.Equal(true, IsConsumer([]string{"consumer"})) suite.Equal(true, IsConsumer([]string{"consumer", "publisher"})) @@ -126,12 +129,12 @@ func (suite *AuthTestSuite) TestAuth() { suite.Equal(false, IsAdminViewer([]string{"publisher"})) // Check ValidUsers mechanism - v, err := AreValidUsers("ARGO", []string{"UserA", "foo", "bar"}, store) + v, err := AreValidUsers(suite.ctx, "ARGO", []string{"UserA", "foo", "bar"}, store) suite.Equal(false, v) suite.Equal("User(s): foo, bar do not exist", err.Error()) // Check ValidUsers mechanism - v, err = AreValidUsers("ARGO", []string{"UserA", "UserB"}, store) + v, err = AreValidUsers(suite.ctx, "ARGO", []string{"UserA", "UserB"}, store) suite.Equal(true, v) suite.Equal(nil, err) @@ -313,7 +316,7 @@ func (suite *AuthTestSuite) TestAuth() { ] }` - users, _ := FindUsers("argo_uuid", "", "", true, store) + users, _ := FindUsers(suite.ctx, "argo_uuid", "", "", true, store) outUserList, _ := users.ExportJSON() suite.Equal(expUserList, outUserList) @@ -345,26 +348,26 @@ func (suite *AuthTestSuite) TestAuth() { }` // Test GetUserByToken - userTk, _ := GetUserByToken("S3CR3T4", store) + userTk, _ := GetUserByToken(suite.ctx, "S3CR3T4", store) usrTkJSON, _ := userTk.ExportJSON() suite.Equal(expUsrTkJSON, usrTkJSON) - suite.Equal(true, ExistsWithName("UserA", store)) - suite.Equal(false, ExistsWithName("userA", store)) - suite.Equal(true, ExistsWithName("UserB", store)) - suite.Equal(true, ExistsWithUUID("uuid1", store)) - suite.Equal(false, ExistsWithUUID("foouuuid", store)) - suite.Equal(true, ExistsWithUUID("uuid2", store)) + suite.Equal(true, ExistsWithName(suite.ctx, "UserA", store)) + suite.Equal(false, ExistsWithName(suite.ctx, "userA", store)) + suite.Equal(true, ExistsWithName(suite.ctx, "UserB", store)) + suite.Equal(true, ExistsWithUUID(suite.ctx, "uuid1", store)) + suite.Equal(false, ExistsWithUUID(suite.ctx, "foouuuid", store)) + suite.Equal(true, ExistsWithUUID(suite.ctx, "uuid2", store)) - suite.Equal("UserA", GetNameByUUID("uuid1", store)) - suite.Equal("UserB", GetNameByUUID("uuid2", store)) - suite.Equal("UserX", GetNameByUUID("uuid3", store)) - suite.Equal("UserZ", GetNameByUUID("uuid4", store)) + suite.Equal("UserA", GetNameByUUID(suite.ctx, "uuid1", store)) + suite.Equal("UserB", GetNameByUUID(suite.ctx, "uuid2", store)) + suite.Equal("UserX", GetNameByUUID(suite.ctx, "uuid3", store)) + suite.Equal("UserZ", GetNameByUUID(suite.ctx, "uuid4", store)) - suite.Equal("uuid1", GetUUIDByName("UserA", store)) - suite.Equal("uuid2", GetUUIDByName("UserB", store)) - suite.Equal("uuid3", GetUUIDByName("UserX", store)) - suite.Equal("uuid4", GetUUIDByName("UserZ", store)) + suite.Equal("uuid1", GetUUIDByName(suite.ctx, "UserA", store)) + suite.Equal("uuid2", GetUUIDByName(suite.ctx, "UserB", store)) + suite.Equal("uuid3", GetUUIDByName(suite.ctx, "UserX", store)) + suite.Equal("uuid4", GetUUIDByName(suite.ctx, "UserZ", store)) // Test GetUserByUUID expUsrUUIDJSON := `{ @@ -394,20 +397,20 @@ func (suite *AuthTestSuite) TestAuth() { "created_by": "UserA" }` // normal use case - expUsrUUID, expNilErr := GetUserByUUID("uuid4", store) + expUsrUUID, expNilErr := GetUserByUUID(suite.ctx, "uuid4", store) usrUUIDJson, _ := expUsrUUID.ExportJSON() suite.Equal(usrUUIDJson, expUsrUUIDJSON) suite.Nil(expNilErr) // different users have the same uuid - expUsrMultipleUUID, expErrMultipleUUIDS := GetUserByUUID("same_uuid", store) + expUsrMultipleUUID, expErrMultipleUUIDS := GetUserByUUID(suite.ctx, "same_uuid", store) suite.Equal("multiple uuids", expErrMultipleUUIDS.Error()) suite.Equal(User{}, expUsrMultipleUUID) // user with given uuid doesn't exist - expUsrNotFoundUUID, expErrNotFoundUUID := GetUserByUUID("uuid10", store) + expUsrNotFoundUUID, expErrNotFoundUUID := GetUserByUUID(suite.ctx, "uuid10", store) suite.Equal("not found", expErrNotFoundUUID.Error()) suite.Equal(User{}, expUsrNotFoundUUID) @@ -450,14 +453,14 @@ func (suite *AuthTestSuite) TestAuth() { tm := time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC) // Test Create - CreateUser("uuid12", "johndoe", "firstdoe", "lastdoe", "orgdoe", "descdoe", []ProjectRoles{ProjectRoles{Project: "ARGO", Roles: []string{"consumer"}}}, "johndoe@fake.email.foo", "TOK3N", []string{"service_admin"}, tm, "", store) - usrs, _ := FindUsers("", "uuid12", "", true, store) + CreateUser(suite.ctx, "uuid12", "johndoe", "firstdoe", "lastdoe", "orgdoe", "descdoe", []ProjectRoles{ProjectRoles{Project: "ARGO", Roles: []string{"consumer"}}}, "johndoe@fake.email.foo", "TOK3N", []string{"service_admin"}, tm, "", store) + usrs, _ := FindUsers(suite.ctx, "", "uuid12", "", true, store) usrJSON, _ := usrs.List[0].ExportJSON() suite.Equal(expUsrJSON, usrJSON) // Test Create with empty project list - CreateUser("uuid13", "empty-proj", "", "", "", "", []ProjectRoles{{Project: "", Roles: []string{"consumer"}}}, "TOK3N", "johndoe@fake.email.foo", []string{"service_admin"}, tm, "", store) - usrs2, _ := FindUsers("", "uuid13", "", true, store) + CreateUser(suite.ctx, "uuid13", "empty-proj", "", "", "", "", []ProjectRoles{{Project: "", Roles: []string{"consumer"}}}, "TOK3N", "johndoe@fake.email.foo", []string{"service_admin"}, tm, "", store) + usrs2, _ := FindUsers(suite.ctx, "", "uuid13", "", true, store) expusrs2 := Users{List: []User{{UUID: "uuid13", Projects: []ProjectRoles{}, Name: "empty-proj", Token: "TOK3N", Email: "johndoe@fake.email.foo", ServiceRoles: []string{"service_admin"}, CreatedOn: "2009-11-10T23:00:00Z", ModifiedOn: "2009-11-10T23:00:00Z", CreatedBy: ""}}} suite.Equal(expusrs2, usrs2) @@ -488,24 +491,24 @@ func (suite *AuthTestSuite) TestAuth() { "created_on": "2009-11-10T23:00:00Z", "modified_on": "2009-11-10T23:00:00Z" }` - UpdateUser("uuid12", "firstdoe2", "lastdoe2", "orgdoe2", "descdoe2", "johnny_doe", nil, "", []string{"consumer", "producer"}, tm, false, store) - usrUpd, _ := FindUsers("", "uuid12", "", true, store) + UpdateUser(suite.ctx, "uuid12", "firstdoe2", "lastdoe2", "orgdoe2", "descdoe2", "johnny_doe", nil, "", []string{"consumer", "producer"}, tm, false, store) + usrUpd, _ := FindUsers(suite.ctx, "", "uuid12", "", true, store) usrUpdJSON, _ := usrUpd.List[0].ExportJSON() suite.Equal(expUpdate, usrUpdJSON) // reflect obj true - usrUpd2, _ := UpdateUser("uuid12", "", "", "", "", "johnny_doe", nil, "", []string{"consumer", "producer"}, tm, true, store) + usrUpd2, _ := UpdateUser(suite.ctx, "uuid12", "", "", "", "", "johnny_doe", nil, "", []string{"consumer", "producer"}, tm, true, store) usrUpdJSON2, _ := usrUpd2.ExportJSON() suite.Equal(expUpdate, usrUpdJSON2) // Test update with empty project - UpdateUser("uuid13", "", "", "", "", "empty-proj", []ProjectRoles{{Project: "", Roles: []string{"consumer"}}}, "johndoe@fake.email.foo", []string{"service_admin"}, tm, false, store) - usrs2, _ = FindUsers("", "uuid13", "", true, store) + UpdateUser(suite.ctx, "uuid13", "", "", "", "", "empty-proj", []ProjectRoles{{Project: "", Roles: []string{"consumer"}}}, "johndoe@fake.email.foo", []string{"service_admin"}, tm, false, store) + usrs2, _ = FindUsers(suite.ctx, "", "uuid13", "", true, store) expusrs2 = Users{List: []User{{UUID: "uuid13", Projects: []ProjectRoles{}, Name: "empty-proj", Token: "TOK3N", Email: "johndoe@fake.email.foo", ServiceRoles: []string{"service_admin"}, CreatedOn: "2009-11-10T23:00:00Z", ModifiedOn: "2009-11-10T23:00:00Z", CreatedBy: ""}}} suite.Equal(expusrs2, usrs2) - RemoveUser("uuid12", store) - _, err = FindUsers("", "uuid12", "", true, store) + RemoveUser(suite.ctx, "uuid12", store) + _, err = FindUsers(suite.ctx, "", "uuid12", "", true, store) suite.Equal(errors.New("not found"), err) store2 := stores.NewMockStore("", "") @@ -534,7 +537,7 @@ func (suite *AuthTestSuite) TestAuth() { qUsers1 = append(qUsers1, User{"uuid1", []ProjectRoles{{"ARGO", []string{"consumer", "publisher"}, []string{"topic1", "topic2"}, []string{"sub1", "sub2", "sub3"}}}, "UserA", "FirstA", "LastA", "OrgA", "DescA", "S3CR3T1", "foo-email", []string{}, created, modified, ""}) qUsers1 = append(qUsers1, User{"uuid0", []ProjectRoles{{"ARGO", []string{"consumer", "publisher"}, []string{}, []string{}}}, "Test", "", "", "", "", "S3CR3T", "Test@test.com", []string{}, created, modified, ""}) // return all users - pu1, e1 := PaginatedFindUsers("", 0, "", true, true, store2) + pu1, e1 := PaginatedFindUsers(suite.ctx, "", 0, "", true, true, store2) var qUsers2 []User qUsers2 = append(qUsers2, User{"uuid8", []ProjectRoles{{"ARGO2", []string{"consumer", "publisher"}, []string{}, []string{}}}, "UserZ", "", "", "", "", "S3CR3T1", "foo-email", []string{}, created, modified, ""}) @@ -552,21 +555,21 @@ func (suite *AuthTestSuite) TestAuth() { qUsers2 = append(qUsers2, User{"same_uuid", []ProjectRoles{{"ARGO", []string{"publisher", "consumer"}, []string{}, []string{}}}, "UserSame2", "", "", "", "", "S3CR3T42", "foo-email", []string{}, created, modified, "UserA"}) // return the first page with 2 users - pu2, e2 := PaginatedFindUsers("", 3, "", true, true, store2) + pu2, e2 := PaginatedFindUsers(suite.ctx, "", 3, "", true, true, store2) var qUsers3 []User qUsers3 = append(qUsers3, User{"uuid4", []ProjectRoles{{"ARGO", []string{"publisher", "consumer"}, []string{"topic2"}, []string{"sub3", "sub4"}}}, "UserZ", "", "", "", "", "S3CR3T4", "foo-email", []string{}, created, modified, "UserA"}) qUsers3 = append(qUsers3, User{"uuid3", []ProjectRoles{{"ARGO", []string{"publisher", "consumer"}, []string{"topic3"}, []string{"sub2"}}}, "UserX", "", "", "", "", "S3CR3T3", "foo-email", []string{}, created, modified, "UserA"}) // return the next 2 users - pu3, e3 := PaginatedFindUsers("NA==", 2, "", true, true, store2) + pu3, e3 := PaginatedFindUsers(suite.ctx, "NA==", 2, "", true, true, store2) // empty collection store3 := stores.NewMockStore("", "") store3.UserList = []stores.QUser{} - pu4, e4 := PaginatedFindUsers("", 0, "", true, true, store3) + pu4, e4 := PaginatedFindUsers(suite.ctx, "", 0, "", true, true, store3) // invalid id - _, e5 := PaginatedFindUsers("invalid", 0, "", true, true, store2) + _, e5 := PaginatedFindUsers(suite.ctx, "invalid", 0, "", true, true, store2) // check user list by project var qUsersB []User @@ -592,10 +595,10 @@ func (suite *AuthTestSuite) TestAuth() { ModifiedOn: modified, CreatedBy: ""}) - ndu, _ := PaginatedFindUsers("", 1, "", true, false, store2) + ndu, _ := PaginatedFindUsers(suite.ctx, "", 1, "", true, false, store2) suite.Equal(ndUser, ndu.Users) - puC, e1 := PaginatedFindUsers("", 1, "argo_uuid2", false, true, store2) + puC, e1 := PaginatedFindUsers(suite.ctx, "", 1, "argo_uuid2", false, true, store2) suite.Equal(qUsersC, puC.Users) suite.Equal(int64(1), puC.TotalSize) suite.Equal("", puC.NextPageToken) @@ -629,8 +632,8 @@ func (suite *AuthTestSuite) TestAppendToUserProjects() { store.ProjectList = append(store.ProjectList, stores.QProject{UUID: "append_uuid", Name: "append_project"}) store.UserList = append(store.UserList, stores.QUser{UUID: "append_uuid"}) - err1 := AppendToUserProjects("append_uuid", "append_uuid", store, "publisher") - u, _ := store.QueryUsers("append_uuid", "append_uuid", "") + err1 := AppendToUserProjects(suite.ctx, "append_uuid", "append_uuid", store, "publisher") + u, _ := store.QueryUsers(suite.ctx, "append_uuid", "append_uuid", "") suite.Equal([]stores.QProjectRoles{ { ProjectUUID: "append_uuid", @@ -640,11 +643,11 @@ func (suite *AuthTestSuite) TestAppendToUserProjects() { suite.Nil(err1) // invalid project - err2 := AppendToUserProjects("", "unknown", store) + err2 := AppendToUserProjects(suite.ctx, "", "unknown", store) suite.Equal("invalid project unknown", err2.Error()) // invalid role - err3 := AppendToUserProjects("append_uuid", "append_uuid", store, "r1") + err3 := AppendToUserProjects(suite.ctx, "append_uuid", "append_uuid", store, "r1") suite.Equal("invalid role r1", err3.Error()) } @@ -695,19 +698,19 @@ func (suite *AuthTestSuite) TestSubACL() { store := stores.NewMockStore(APIcfg.StoreHost, APIcfg.StoreDB) - sACL, _ := GetACL("argo_uuid", "subscriptions", "sub1", store) + sACL, _ := GetACL(suite.ctx, "argo_uuid", "subscriptions", "sub1", store) outJSON, _ := sACL.ExportJSON() suite.Equal(expJSON01, outJSON) - sACL2, _ := GetACL("argo_uuid", "subscriptions", "sub2", store) + sACL2, _ := GetACL(suite.ctx, "argo_uuid", "subscriptions", "sub2", store) outJSON2, _ := sACL2.ExportJSON() suite.Equal(expJSON02, outJSON2) - sACL3, _ := GetACL("argo_uuid", "subscriptions", "sub3", store) + sACL3, _ := GetACL(suite.ctx, "argo_uuid", "subscriptions", "sub3", store) outJSON3, _ := sACL3.ExportJSON() suite.Equal(expJSON03, outJSON3) - sACL4, _ := GetACL("argo_uuid", "subscriptions", "sub4", store) + sACL4, _ := GetACL(suite.ctx, "argo_uuid", "subscriptions", "sub4", store) outJSON4, _ := sACL4.ExportJSON() suite.Equal(expJSON04, outJSON4) @@ -716,8 +719,8 @@ func (suite *AuthTestSuite) TestSubACL() { suite.Equal(expJSON05, outJSON5) // make sure that the acl doesn't contain empty "" in the spot of the deleted user - store.RemoveUser("uuid1") - dACL, _ := GetACL("argo_uuid", "subscriptions", "sub1", store) + store.RemoveUser(suite.ctx, "uuid1") + dACL, _ := GetACL(suite.ctx, "argo_uuid", "subscriptions", "sub1", store) outJSONd, _ := dACL.ExportJSON() suite.Equal(expJSON01deleted, outJSONd) @@ -760,15 +763,15 @@ func (suite *AuthTestSuite) TestTopicACL() { store := stores.NewMockStore(APIcfg.StoreHost, APIcfg.StoreDB) - tACL, _ := GetACL("argo_uuid", "topics", "topic1", store) + tACL, _ := GetACL(suite.ctx, "argo_uuid", "topics", "topic1", store) outJSON, _ := tACL.ExportJSON() suite.Equal(expJSON01, outJSON) - tACL2, _ := GetACL("argo_uuid", "topics", "topic2", store) + tACL2, _ := GetACL(suite.ctx, "argo_uuid", "topics", "topic2", store) outJSON2, _ := tACL2.ExportJSON() suite.Equal(expJSON02, outJSON2) - tACL3, _ := GetACL("argo_uuid", "topics", "topic3", store) + tACL3, _ := GetACL(suite.ctx, "argo_uuid", "topics", "topic3", store) outJSON3, _ := tACL3.ExportJSON() suite.Equal(expJSON03, outJSON3) @@ -777,8 +780,8 @@ func (suite *AuthTestSuite) TestTopicACL() { suite.Equal(expJSON04, outJSON4) // make sure that the acl doesn't contain empty "" in the spot of the deleted user - store.RemoveUser("uuid1") - dACL, _ := GetACL("argo_uuid", "topics", "topic1", store) + store.RemoveUser(suite.ctx, "uuid1") + dACL, _ := GetACL(suite.ctx, "argo_uuid", "topics", "topic1", store) outJSONd, _ := dACL.ExportJSON() suite.Equal(expJSON01deleted, outJSONd) } @@ -787,19 +790,19 @@ func (suite *AuthTestSuite) TestModACL() { store := stores.NewMockStore("", "") - e1 := ModACL("argo_uuid", "topics", "topic1", []string{"UserX", "UserZ"}, store) + e1 := ModACL(suite.ctx, "argo_uuid", "topics", "topic1", []string{"UserX", "UserZ"}, store) suite.Nil(e1) tACL1, _ := store.TopicsACL["topic1"] suite.Equal([]string{"uuid3", "uuid4"}, tACL1.ACL) - e2 := ModACL("argo_uuid", "subscriptions", "sub1", []string{"UserX", "UserZ"}, store) + e2 := ModACL(suite.ctx, "argo_uuid", "subscriptions", "sub1", []string{"UserX", "UserZ"}, store) suite.Nil(e2) sACL1, _ := store.SubsACL["sub1"] suite.Equal([]string{"uuid3", "uuid4"}, sACL1.ACL) - e3 := ModACL("argo_uuid", "mistype", "sub1", []string{"UserX", "UserZ"}, store) + e3 := ModACL(suite.ctx, "argo_uuid", "mistype", "sub1", []string{"UserX", "UserZ"}, store) suite.Equal("wrong resource type", e3.Error()) } @@ -807,19 +810,19 @@ func (suite *AuthTestSuite) TestAppendToACL() { store := stores.NewMockStore("", "") - e1 := AppendToACL("argo_uuid", "topics", "topic1", []string{"UserX", "UserZ", "UserZ"}, store) + e1 := AppendToACL(suite.ctx, "argo_uuid", "topics", "topic1", []string{"UserX", "UserZ", "UserZ"}, store) suite.Nil(e1) tACL1, _ := store.TopicsACL["topic1"] suite.Equal([]string{"uuid1", "uuid2", "uuid3", "uuid4"}, tACL1.ACL) - e2 := AppendToACL("argo_uuid", "subscriptions", "sub1", []string{"UserX", "UserZ", "UserZ"}, store) + e2 := AppendToACL(suite.ctx, "argo_uuid", "subscriptions", "sub1", []string{"UserX", "UserZ", "UserZ"}, store) suite.Nil(e2) sACL1, _ := store.SubsACL["sub1"] suite.Equal([]string{"uuid1", "uuid2", "uuid3", "uuid4"}, sACL1.ACL) - e3 := AppendToACL("argo_uuid", "mistype", "sub1", []string{"UserX", "UserZ"}, store) + e3 := AppendToACL(suite.ctx, "argo_uuid", "mistype", "sub1", []string{"UserX", "UserZ"}, store) suite.Equal("wrong resource type", e3.Error()) } @@ -827,33 +830,33 @@ func (suite *AuthTestSuite) TestRemoveFromACL() { store := stores.NewMockStore("", "") - e1 := RemoveFromACL("argo_uuid", "topics", "topic1", []string{"UserA", "UserK"}, store) + e1 := RemoveFromACL(suite.ctx, "argo_uuid", "topics", "topic1", []string{"UserA", "UserK"}, store) suite.Nil(e1) tACL1, _ := store.TopicsACL["topic1"] suite.Equal([]string{"uuid2"}, tACL1.ACL) - e2 := RemoveFromACL("argo_uuid", "subscriptions", "sub1", []string{"UserA", "UserK"}, store) + e2 := RemoveFromACL(suite.ctx, "argo_uuid", "subscriptions", "sub1", []string{"UserA", "UserK"}, store) suite.Nil(e2) sACL1, _ := store.SubsACL["sub1"] suite.Equal([]string{"uuid2"}, sACL1.ACL) - e3 := RemoveFromACL("argo_uuid", "mistype", "sub1", []string{"UserX", "UserZ"}, store) + e3 := RemoveFromACL(suite.ctx, "argo_uuid", "mistype", "sub1", []string{"UserX", "UserZ"}, store) suite.Equal("wrong resource type", e3.Error()) } -func (suite *AuthTestSuite) TestGetPushWorkerToken() { +func (suite *AuthTestSuite) TestGetPushWorker() { store := stores.NewMockStore("", "") // normal case of push enabled true and correct push worker token - u1, err1 := GetPushWorker("push_token", store) + u1, err1 := GetPushWorker(suite.ctx, "push_token", store) suite.Equal(User{"uuid7", []ProjectRoles{}, "push_worker_0", "", "", "", "", "push_token", "foo-email", []string{"push_worker"}, "2009-11-10T23:00:00Z", "2009-11-10T23:00:00Z", ""}, u1) suite.Nil(err1) // incorrect push worker token - u4, err4 := GetPushWorker("missing", store) + u4, err4 := GetPushWorker(suite.ctx, "missing", store) suite.Equal(User{}, u4) suite.Equal("push_500", err4.Error()) } @@ -862,7 +865,7 @@ func (suite *AuthTestSuite) TestRegisterUser() { store := stores.NewMockStore("", "") - ur, err := RegisterUser("ruuid1", "n1", "f1", "l1", "e1", "o1", "d1", "time", "atkn", PendingRegistrationStatus, store) + ur, err := RegisterUser(suite.ctx, "ruuid1", "n1", "f1", "l1", "e1", "o1", "d1", "time", "atkn", PendingRegistrationStatus, store) suite.Nil(err) suite.Equal(UserRegistration{ UUID: "ruuid1", @@ -883,7 +886,7 @@ func (suite *AuthTestSuite) TestFindUserRegistration() { store := stores.NewMockStore("", "") - ur1, e1 := FindUserRegistration("ur-uuid1", "pending", store) + ur1, e1 := FindUserRegistration(suite.ctx, "ur-uuid1", "pending", store) expur1 := UserRegistration{ UUID: "ur-uuid1", Name: "urname", @@ -902,7 +905,7 @@ func (suite *AuthTestSuite) TestFindUserRegistration() { suite.Equal(expur1, ur1) // not found - _, e2 := FindUserRegistration("unknown", "pending", store) + _, e2 := FindUserRegistration(suite.ctx, "unknown", "pending", store) suite.Equal(errors.New("not found"), e2) } @@ -910,8 +913,8 @@ func (suite *AuthTestSuite) TestUpdateUserRegistration() { store := stores.NewMockStore("", "") m := time.Date(2020, 8, 5, 11, 33, 45, 0, time.UTC) - e1 := UpdateUserRegistration("ur-uuid1", AcceptedRegistrationStatus, "dc", "uuid1", m, store) - ur1, _ := FindUserRegistration("ur-uuid1", "accepted", store) + e1 := UpdateUserRegistration(suite.ctx, "ur-uuid1", AcceptedRegistrationStatus, "dc", "uuid1", m, store) + ur1, _ := FindUserRegistration(suite.ctx, "ur-uuid1", "accepted", store) expur1 := UserRegistration{ UUID: "ur-uuid1", Name: "urname", @@ -934,7 +937,7 @@ func (suite *AuthTestSuite) TestFindUserRegistrations() { store := stores.NewMockStore("", "") - r1, e1 := FindUserRegistrations("", "", "", "", "", store) + r1, e1 := FindUserRegistrations(suite.ctx, "", "", "", "", "", store) expur1 := UserRegistrationsList{ UserRegistrations: []UserRegistration{{ UUID: "ur-uuid1", @@ -956,7 +959,7 @@ func (suite *AuthTestSuite) TestFindUserRegistrations() { suite.Nil(e1) suite.Equal(expur1, r1) - r2, e2 := FindUserRegistrations("pending", "uratkn-1", "urname", "uremail", "urorg", store) + r2, e2 := FindUserRegistrations(suite.ctx, "pending", "uratkn-1", "urname", "uremail", "urorg", store) suite.Nil(e2) suite.Equal(expur1, r2) diff --git a/auth/users.go b/auth/users.go index 8965b2d1..951a6737 100644 --- a/auth/users.go +++ b/auth/users.go @@ -1,6 +1,7 @@ package auth import ( + "context" "crypto/rand" "crypto/sha256" "encoding/base64" @@ -122,9 +123,9 @@ func GetUserFromJSON(input []byte) (User, error) { } // RegisterUser registers a new user to the store -func RegisterUser(uuid, name, fname, lname, email, org, desc, registeredAt, atkn, status string, str stores.Store) (UserRegistration, error) { +func RegisterUser(ctx context.Context, uuid, name, fname, lname, email, org, desc, registeredAt, atkn, status string, str stores.Store) (UserRegistration, error) { - err := str.RegisterUser(uuid, name, fname, lname, email, org, desc, registeredAt, atkn, status) + err := str.RegisterUser(ctx, uuid, name, fname, lname, email, org, desc, registeredAt, atkn, status) if err != nil { return UserRegistration{}, err } @@ -144,13 +145,13 @@ func RegisterUser(uuid, name, fname, lname, email, org, desc, registeredAt, atkn } // DeleteUserRegistration removes the respective registration from the store based on the given uuid -func DeleteUserRegistration(regUUID string, str stores.Store) error { - return str.DeleteRegistration(regUUID) +func DeleteUserRegistration(ctx context.Context, regUUID string, str stores.Store) error { + return str.DeleteRegistration(ctx, regUUID) } -func FindUserRegistration(regUUID, status string, str stores.Store) (UserRegistration, error) { +func FindUserRegistration(ctx context.Context, regUUID, status string, str stores.Store) (UserRegistration, error) { - q, err := str.QueryRegistrations(regUUID, status, "", "", "", "") + q, err := str.QueryRegistrations(ctx, regUUID, status, "", "", "", "") if err != nil { return UserRegistration{}, err } @@ -161,7 +162,7 @@ func FindUserRegistration(regUUID, status string, str stores.Store) (UserRegistr usernameC := "" if q[0].ModifiedBy != "" { - usr, err := str.QueryUsers("", q[0].ModifiedBy, "") + usr, err := str.QueryUsers(ctx, "", q[0].ModifiedBy, "") if err == nil && len(usr) > 0 { usernameC = usr[0].Name @@ -187,9 +188,9 @@ func FindUserRegistration(regUUID, status string, str stores.Store) (UserRegistr return ur, nil } -func FindUserRegistrations(status, activationToken, name, email, org string, str stores.Store) (UserRegistrationsList, error) { +func FindUserRegistrations(ctx context.Context, status, activationToken, name, email, org string, str stores.Store) (UserRegistrationsList, error) { - q, err := str.QueryRegistrations("", status, activationToken, name, email, org) + q, err := str.QueryRegistrations(ctx, "", status, activationToken, name, email, org) if err != nil { return UserRegistrationsList{}, err } @@ -202,7 +203,7 @@ func FindUserRegistrations(status, activationToken, name, email, org string, str usernameC := "" if ur.ModifiedBy != "" { - usr, err := str.QueryUsers("", ur.ModifiedBy, "") + usr, err := str.QueryUsers(ctx, "", ur.ModifiedBy, "") if err == nil && len(usr) > 0 { usernameC = usr[0].Name @@ -231,12 +232,12 @@ func FindUserRegistrations(status, activationToken, name, email, org string, str return urList, nil } -func UpdateUserRegistration(regUUID, status, declineComment, modifiedBy string, modifiedAt time.Time, refStr stores.Store) error { +func UpdateUserRegistration(ctx context.Context, regUUID, status, declineComment, modifiedBy string, modifiedAt time.Time, refStr stores.Store) error { // only accept decline comment with the decline status action if status != DeclinedRegistrationStatus { declineComment = "" } - return refStr.UpdateRegistration(regUUID, status, declineComment, modifiedBy, modifiedAt.UTC().Format("2006-01-02T15:04:05Z")) + return refStr.UpdateRegistration(ctx, regUUID, status, declineComment, modifiedBy, modifiedAt.UTC().Format("2006-01-02T15:04:05Z")) } // NewUser accepts parameters and creates a new user @@ -259,11 +260,18 @@ func NewUser(uuid string, projects []ProjectRoles, name string, fname string, ln } // GetPushWorker returns a push worker user by token -func GetPushWorker(pwToken string, store stores.Store) (User, error) { +func GetPushWorker(ctx context.Context, pwToken string, store stores.Store) (User, error) { - pw, err := GetUserByToken(pwToken, store) + pw, err := GetUserByToken(ctx, pwToken, store) if err != nil { - log.Errorf("Could not retrieve push worker user with token %v, %v", pwToken, err.Error()) + log.WithFields( + log.Fields{ + "trace_id": ctx.Value("trace_id"), + "type": "service_log", + "token": pwToken, + "error": err.Error(), + }, + ).Error("Could not retrieve push worker user") return User{}, errors.New("push_500") } @@ -271,10 +279,10 @@ func GetPushWorker(pwToken string, store stores.Store) (User, error) { } // GetUserByToken returns a specific user by his token -func GetUserByToken(token string, store stores.Store) (User, error) { +func GetUserByToken(ctx context.Context, token string, store stores.Store) (User, error) { result := User{} - user, err := store.GetUserFromToken(token) + user, err := store.GetUserFromToken(ctx, token) if err != nil { return result, err @@ -282,7 +290,7 @@ func GetUserByToken(token string, store stores.Store) (User, error) { usernameC := "" if user.CreatedBy != "" { - usr, err := store.QueryUsers("", user.CreatedBy, "") + usr, err := store.QueryUsers(ctx, "", user.CreatedBy, "") if err == nil && len(usr) > 0 { usernameC = usr[0].Name @@ -291,16 +299,16 @@ func GetUserByToken(token string, store stores.Store) (User, error) { pRoles := []ProjectRoles{} for _, pItem := range user.Projects { - prName := projects.GetNameByUUID(pItem.ProjectUUID, store) + prName := projects.GetNameByUUID(ctx, pItem.ProjectUUID, store) // Get User topics and subscriptions - topicList, _ := store.QueryTopicsByACL(pItem.ProjectUUID, user.UUID) + topicList, _ := store.QueryTopicsByACL(ctx, pItem.ProjectUUID, user.UUID) topicNames := []string{} for _, tpItem := range topicList { topicNames = append(topicNames, tpItem.Name) } - subList, _ := store.QuerySubsByACL(pItem.ProjectUUID, user.UUID) + subList, _ := store.QuerySubsByACL(ctx, pItem.ProjectUUID, user.UUID) subNames := []string{} for _, sbItem := range subList { subNames = append(subNames, sbItem.Name) @@ -318,10 +326,10 @@ func GetUserByToken(token string, store stores.Store) (User, error) { } // FindUsers returns a specific user or a list of all available users belonging to a project in the datastore. -func FindUsers(projectUUID string, uuid string, name string, priviledged bool, store stores.Store) (Users, error) { +func FindUsers(ctx context.Context, projectUUID string, uuid string, name string, priviledged bool, store stores.Store) (Users, error) { result := Users{} - users, err := store.QueryUsers(projectUUID, uuid, name) + users, err := store.QueryUsers(ctx, projectUUID, uuid, name) for _, item := range users { @@ -335,7 +343,7 @@ func FindUsers(projectUUID string, uuid string, name string, priviledged bool, s // if call made by priviledged user (superuser), show service roles, token and user creator info if priviledged { if item.CreatedBy != "" { - usr, err := store.QueryUsers("", item.CreatedBy, "") + usr, err := store.QueryUsers(ctx, "", item.CreatedBy, "") if err == nil && len(usr) > 0 { usernameC = usr[0].Name @@ -352,16 +360,16 @@ func FindUsers(projectUUID string, uuid string, name string, priviledged bool, s if !priviledged && pItem.ProjectUUID != projectUUID { continue } - prName := projects.GetNameByUUID(pItem.ProjectUUID, store) + prName := projects.GetNameByUUID(ctx, pItem.ProjectUUID, store) // Get User topics and subscriptions - topicList, _ := store.QueryTopicsByACL(pItem.ProjectUUID, item.UUID) + topicList, _ := store.QueryTopicsByACL(ctx, pItem.ProjectUUID, item.UUID) topicNames := []string{} for _, tpItem := range topicList { topicNames = append(topicNames, tpItem.Name) } - subList, _ := store.QuerySubsByACL(pItem.ProjectUUID, item.UUID) + subList, _ := store.QuerySubsByACL(ctx, pItem.ProjectUUID, item.UUID) subNames := []string{} for _, sbItem := range subList { subNames = append(subNames, sbItem.Name) @@ -391,7 +399,7 @@ func FindUsers(projectUUID string, uuid string, name string, priviledged bool, s } // PaginatedFindUsers returns a page of users -func PaginatedFindUsers(pageToken string, pageSize int64, projectUUID string, privileged, detailedView bool, store stores.Store) (PaginatedUsers, error) { +func PaginatedFindUsers(ctx context.Context, pageToken string, pageSize int64, projectUUID string, privileged, detailedView bool, store stores.Store) (PaginatedUsers, error) { var totalSize int64 var nextPageToken string @@ -401,13 +409,20 @@ func PaginatedFindUsers(pageToken string, pageSize int64, projectUUID string, pr // decode the base64 pageToken if pageTokenBytes, err = base64.StdEncoding.DecodeString(pageToken); err != nil { - log.Errorf("Page token %v produced an error while being decoded to base64: %v", pageToken, err.Error()) + log.WithFields( + log.Fields{ + "trace_id": ctx.Value("trace_id"), + "type": "request_log", + "page_token": pageToken, + "error": err.Error(), + }, + ).Error("error while decoding to base64") return PaginatedUsers{}, err } result := PaginatedUsers{Users: []User{}} - if users, totalSize, nextPageToken, err = store.PaginatedQueryUsers(string(pageTokenBytes), pageSize, projectUUID); err != nil { + if users, totalSize, nextPageToken, err = store.PaginatedQueryUsers(ctx, string(pageTokenBytes), pageSize, projectUUID); err != nil { return result, err } @@ -420,7 +435,7 @@ func PaginatedFindUsers(pageToken string, pageSize int64, projectUUID string, pr // if call made by priviledged user (superuser), show service roles, token and user creator info if privileged { if item.CreatedBy != "" { - usr, err := store.QueryUsers("", item.CreatedBy, "") + usr, err := store.QueryUsers(ctx, "", item.CreatedBy, "") if err == nil && len(usr) > 0 { usernameC = usr[0].Name @@ -440,16 +455,16 @@ func PaginatedFindUsers(pageToken string, pageSize int64, projectUUID string, pr if !privileged && pItem.ProjectUUID != projectUUID { continue } - prName := projects.GetNameByUUID(pItem.ProjectUUID, store) + prName := projects.GetNameByUUID(ctx, pItem.ProjectUUID, store) // Get User topics and subscriptions - topicList, _ := store.QueryTopicsByACL(pItem.ProjectUUID, item.UUID) + topicList, _ := store.QueryTopicsByACL(ctx, pItem.ProjectUUID, item.UUID) topicNames := []string{} for _, tpItem := range topicList { topicNames = append(topicNames, tpItem.Name) } - subList, _ := store.QuerySubsByACL(pItem.ProjectUUID, item.UUID) + subList, _ := store.QuerySubsByACL(ctx, pItem.ProjectUUID, item.UUID) subNames := []string{} for _, sbItem := range subList { subNames = append(subNames, sbItem.Name) @@ -473,15 +488,15 @@ func PaginatedFindUsers(pageToken string, pageSize int64, projectUUID string, pr } // Authenticate based on token -func Authenticate(projectUUID string, token string, store stores.Store) ([]string, string) { - return store.GetUserRoles(projectUUID, token) +func Authenticate(ctx context.Context, projectUUID string, token string, store stores.Store) ([]string, string) { + return store.GetUserRoles(ctx, projectUUID, token) } // ExistsWithName returns true if a user with name exists -func ExistsWithName(name string, store stores.Store) bool { +func ExistsWithName(ctx context.Context, name string, store stores.Store) bool { result := false - users, err := store.QueryUsers("", "", name) + users, err := store.QueryUsers(ctx, "", "", name) if len(users) > 0 && err == nil { result = true } @@ -490,10 +505,10 @@ func ExistsWithName(name string, store stores.Store) bool { } // ExistsWithUUID return true if a user with uuid exists -func ExistsWithUUID(uuid string, store stores.Store) bool { +func ExistsWithUUID(ctx context.Context, uuid string, store stores.Store) bool { result := false - users, err := store.QueryUsers("", uuid, "") + users, err := store.QueryUsers(ctx, "", uuid, "") if len(users) > 0 && err == nil { result = true } @@ -502,9 +517,9 @@ func ExistsWithUUID(uuid string, store stores.Store) bool { } // GetNameByUUID queries user by UUID and returns the user's name. If not found, returns an empty string -func GetNameByUUID(uuid string, store stores.Store) string { +func GetNameByUUID(ctx context.Context, uuid string, store stores.Store) string { result := "" - users, err := store.QueryUsers("", uuid, "") + users, err := store.QueryUsers(ctx, "", uuid, "") if len(users) > 0 && err == nil { result = users[0].Name } @@ -513,11 +528,11 @@ func GetNameByUUID(uuid string, store stores.Store) string { } // GetUserByUUID returns user information by UUID -func GetUserByUUID(uuid string, store stores.Store) (User, error) { +func GetUserByUUID(ctx context.Context, uuid string, store stores.Store) (User, error) { var result User - users, err := store.QueryUsers("", uuid, "") + users, err := store.QueryUsers(ctx, "", uuid, "") if err != nil { return User{}, err @@ -537,7 +552,7 @@ func GetUserByUUID(uuid string, store stores.Store) (User, error) { //convert the Quser to User usernameC := "" if user.CreatedBy != "" { - usr, err := store.QueryUsers("", user.CreatedBy, "") + usr, err := store.QueryUsers(ctx, "", user.CreatedBy, "") if err == nil && len(usr) > 0 { usernameC = usr[0].Name @@ -546,16 +561,16 @@ func GetUserByUUID(uuid string, store stores.Store) (User, error) { pRoles := []ProjectRoles{} for _, pItem := range user.Projects { - prName := projects.GetNameByUUID(pItem.ProjectUUID, store) + prName := projects.GetNameByUUID(ctx, pItem.ProjectUUID, store) // Get User topics and subscriptions - topicList, _ := store.QueryTopicsByACL(pItem.ProjectUUID, user.UUID) + topicList, _ := store.QueryTopicsByACL(ctx, pItem.ProjectUUID, user.UUID) topicNames := []string{} for _, tpItem := range topicList { topicNames = append(topicNames, tpItem.Name) } - subList, _ := store.QuerySubsByACL(pItem.ProjectUUID, user.UUID) + subList, _ := store.QuerySubsByACL(ctx, pItem.ProjectUUID, user.UUID) subNames := []string{} for _, sbItem := range subList { subNames = append(subNames, sbItem.Name) @@ -574,9 +589,9 @@ func GetUserByUUID(uuid string, store stores.Store) (User, error) { } // GetUUIDByName queries user by name and returns the corresponding UUID -func GetUUIDByName(name string, store stores.Store) string { +func GetUUIDByName(ctx context.Context, name string, store stores.Store) string { result := "" - users, err := store.QueryUsers("", "", name) + users, err := store.QueryUsers(ctx, "", "", name) if len(users) > 0 && err == nil { result = users[0].UUID @@ -586,24 +601,24 @@ func GetUUIDByName(name string, store stores.Store) string { } // UpdateUserToken updates an existing user's token -func UpdateUserToken(uuid string, token string, store stores.Store) (User, error) { - if err := store.UpdateUserToken(uuid, token); err != nil { +func UpdateUserToken(ctx context.Context, uuid string, token string, store stores.Store) (User, error) { + if err := store.UpdateUserToken(ctx, uuid, token); err != nil { return User{}, err } // reflect stored object - stored, err := FindUsers("", uuid, "", true, store) + stored, err := FindUsers(ctx, "", uuid, "", true, store) return stored.One(), err } // AppendToUserProjects appends a unique project to the user's project list -func AppendToUserProjects(userUUID string, projectUUID string, store stores.Store, pRoles ...string) error { +func AppendToUserProjects(ctx context.Context, userUUID string, projectUUID string, store stores.Store, pRoles ...string) error { - pName := projects.GetNameByUUID(projectUUID, store) + pName := projects.GetNameByUUID(ctx, projectUUID, store) if pName == "" { return fmt.Errorf("invalid project %v", projectUUID) } - validRoles := store.GetAllRoles() + validRoles := store.GetAllRoles(ctx) for _, role := range pRoles { if !IsRoleValid(role, validRoles) { @@ -611,7 +626,7 @@ func AppendToUserProjects(userUUID string, projectUUID string, store stores.Stor } } - err := store.AppendToUserProjects(userUUID, projectUUID, pRoles...) + err := store.AppendToUserProjects(ctx, userUUID, projectUUID, pRoles...) if err != nil { return err } @@ -621,11 +636,11 @@ func AppendToUserProjects(userUUID string, projectUUID string, store stores.Stor // UpdateUser updates an existing user's information // IF the function caller needs to have a view on the updated user object it can set the reflectObj to true -func UpdateUser(uuid, firstName, lastName, organization, description string, name string, projectList []ProjectRoles, email string, serviceRoles []string, modifiedOn time.Time, reflectObj bool, store stores.Store) (User, error) { +func UpdateUser(ctx context.Context, uuid, firstName, lastName, organization, description string, name string, projectList []ProjectRoles, email string, serviceRoles []string, modifiedOn time.Time, reflectObj bool, store stores.Store) (User, error) { prList := []stores.QProjectRoles{} - validRoles := store.GetAllRoles() + validRoles := store.GetAllRoles(ctx) var duplicates []string // Prep project roles for datastore insert @@ -646,7 +661,7 @@ func UpdateUser(uuid, firstName, lastName, organization, description string, nam duplicates = append(duplicates, item.Project) - prUUID := projects.GetUUIDByName(item.Project, store) + prUUID := projects.GetUUIDByName(ctx, item.Project, store) // If project name doesn't reflect a uuid, then is non existent if prUUID == "" { return User{}, errors.New("invalid project: " + item.Project) @@ -674,13 +689,13 @@ func UpdateUser(uuid, firstName, lastName, organization, description string, nam } } - if err := store.UpdateUser(uuid, firstName, lastName, organization, description, prList, name, email, serviceRoles, modifiedOn); err != nil { + if err := store.UpdateUser(ctx, uuid, firstName, lastName, organization, description, prList, name, email, serviceRoles, modifiedOn); err != nil { return User{}, err } // reflect stored object if reflectObj { - stored, err := FindUsers("", uuid, "", true, store) + stored, err := FindUsers(ctx, "", uuid, "", true, store) return stored.One(), err } @@ -688,13 +703,13 @@ func UpdateUser(uuid, firstName, lastName, organization, description string, nam } // CreateUser creates a new user -func CreateUser(uuid string, name string, fname string, lname string, org string, desc string, projectList []ProjectRoles, token string, email string, serviceRoles []string, createdOn time.Time, createdBy string, store stores.Store) (User, error) { +func CreateUser(ctx context.Context, uuid string, name string, fname string, lname string, org string, desc string, projectList []ProjectRoles, token string, email string, serviceRoles []string, createdOn time.Time, createdBy string, store stores.Store) (User, error) { // check if project with the same name exists - if ExistsWithName(name, store) { + if ExistsWithName(ctx, name, store) { return User{}, errors.New("exists") } - validRoles := store.GetAllRoles() + validRoles := store.GetAllRoles(ctx) var duplicates []string // Prep project roles for datastore insert @@ -717,7 +732,7 @@ func CreateUser(uuid string, name string, fname string, lname string, org string // add project name to duplicate check list duplicates = append(duplicates, item.Project) - prUUID := projects.GetUUIDByName(item.Project, store) + prUUID := projects.GetUUIDByName(ctx, item.Project, store) // If project name doesn't reflect a uuid, then is non existent if prUUID == "" { return User{}, errors.New("invalid project: " + item.Project) @@ -740,12 +755,12 @@ func CreateUser(uuid string, name string, fname string, lname string, org string } } - if err := store.InsertUser(uuid, prList, name, fname, lname, org, desc, token, email, serviceRoles, createdOn, createdOn, createdBy); err != nil { + if err := store.InsertUser(ctx, uuid, prList, name, fname, lname, org, desc, token, email, serviceRoles, createdOn, createdOn, createdBy); err != nil { return User{}, errors.New("backend error") } // reflect stored object - stored, err := FindUsers("", "", name, true, store) + stored, err := FindUsers(ctx, "", "", name, true, store) return stored.One(), err } @@ -827,8 +842,8 @@ func IsAdminViewer(roles []string) bool { } // RemoveUser removes an existing user -func RemoveUser(uuid string, store stores.Store) error { - return store.RemoveUser(uuid) +func RemoveUser(ctx context.Context, uuid string, store stores.Store) error { + return store.RemoveUser(ctx, uuid) } // IsRoleValid checks if a role is a valid against a list of valid roles @@ -842,8 +857,8 @@ func IsRoleValid(role string, validRoles []string) bool { } // AreValidUsers accepts a user array of usernames and checks if users exist in the store -func AreValidUsers(projectUUID string, users []string, store stores.Store) (bool, error) { - found, notFound := store.HasUsers(projectUUID, users) +func AreValidUsers(ctx context.Context, projectUUID string, users []string, store stores.Store) (bool, error) { + found, notFound := store.HasUsers(ctx, projectUUID, users) if found { return true, nil } @@ -863,12 +878,17 @@ func AreValidUsers(projectUUID string, users []string, store stores.Store) (bool } // PerResource (for topics and subscriptions) -func PerResource(project string, resType string, resName string, userUUID string, store stores.Store) bool { +func PerResource(ctx context.Context, project string, resType string, resName string, userUUID string, store stores.Store) bool { if resType == "topics" || resType == "subscriptions" { - err := store.ExistsInACL(project, resType, resName, userUUID) + err := store.ExistsInACL(ctx, project, resType, resName, userUUID) if err != nil { - log.Errorln(err.Error()) + log.WithFields( + log.Fields{ + "trace_id": ctx.Value("trace_id"), + "type": "system_log", + }, + ).Error(err.Error()) return false } @@ -879,7 +899,7 @@ func PerResource(project string, resType string, resName string, userUUID string } // Authorize based on resource and role information -func Authorize(resource string, roles []string, store stores.Store) bool { +func Authorize(ctx context.Context, resource string, roles []string, store stores.Store) bool { // check if _admin_ is in roles for _, role := range roles { if role == "_admin_" { @@ -887,5 +907,5 @@ func Authorize(resource string, roles []string, store stores.Store) bool { } } - return store.HasResourceRoles(resource, roles) + return store.HasResourceRoles(ctx, resource, roles) } diff --git a/brokers/broker.go b/brokers/broker.go index d4bec9bf..f2d4f482 100644 --- a/brokers/broker.go +++ b/brokers/broker.go @@ -13,12 +13,12 @@ type Broker interface { InitConfig() Initialize(peers []string) CloseConnections() - Publish(topic string, payload messages.Message) (string, string, int, int64, error) - GetMinOffset(topic string) int64 - GetMaxOffset(topic string) int64 + Publish(ctx context.Context, topic string, payload messages.Message) (string, string, int, int64, error) + GetMinOffset(ctx context.Context, topic string) int64 + GetMaxOffset(ctx context.Context, topic string) int64 Consume(ctx context.Context, topic string, offset int64, imm bool, max int64) ([]string, error) - DeleteTopic(topic string) error - TimeToOffset(topic string, time time.Time) (int64, error) + DeleteTopic(ctx context.Context, topic string) error + TimeToOffset(ctx context.Context, topic string, time time.Time) (int64, error) } var ErrOffsetOff = errors.New("Offset is off") diff --git a/brokers/kafka.go b/brokers/kafka.go index 77fd9536..dd75c6a1 100644 --- a/brokers/kafka.go +++ b/brokers/kafka.go @@ -176,9 +176,9 @@ func (b *KafkaBroker) init(peers []string) error { } // Publish function publish a message to the broker -func (b *KafkaBroker) Publish(topic string, msg messages.Message) (string, string, int, int64, error) { +func (b *KafkaBroker) Publish(ctx context.Context, topic string, msg messages.Message) (string, string, int, int64, error) { - off := b.GetMaxOffset(topic) + off := b.GetMaxOffset(ctx, topic) msg.ID = strconv.FormatInt(off, 10) // Stamp time to UTC Z to nanoseconds zNano := "2006-01-02T15:04:05.999999999Z" @@ -202,6 +202,7 @@ func (b *KafkaBroker) Publish(topic string, msg messages.Message) (string, strin "backend_service": "kafka", "topic": topic, "error": err.Error(), + "trace_id": ctx.Value("trace_id"), }, ).Errorf("Could not publish message to topic") @@ -212,8 +213,8 @@ func (b *KafkaBroker) Publish(topic string, msg messages.Message) (string, strin } -// GetOffset returns a current topic's offset -func (b *KafkaBroker) GetMaxOffset(topic string) int64 { +// GetMaxOffset returns a current topic's offset +func (b *KafkaBroker) GetMaxOffset(ctx context.Context, topic string) int64 { // Fetch offset loff, err := b.Client.GetOffset(topic, 0, sarama.OffsetNewest) if err != nil { @@ -223,14 +224,15 @@ func (b *KafkaBroker) GetMaxOffset(topic string) int64 { "backend_service": "kafka", "backend_hosts": b.Servers, "error": err.Error(), + "trace_id": ctx.Value("trace_id"), }, ).Errorf("Could not retrieve max offset") } return loff } -// GetOffset returns a current topic's offset -func (b *KafkaBroker) GetMinOffset(topic string) int64 { +// GetMinOffset returns a current topic's offset +func (b *KafkaBroker) GetMinOffset(ctx context.Context, topic string) int64 { // Fetch offset loff, err := b.Client.GetOffset(topic, 0, sarama.OffsetOldest) if err != nil { @@ -240,6 +242,7 @@ func (b *KafkaBroker) GetMinOffset(topic string) int64 { "backend_service": "kafka", "backend_hosts": b.Servers, "error": err.Error(), + "trace_id": ctx.Value("trace_id"), }, ).Errorf("Could not retrieve min offset") } @@ -248,12 +251,12 @@ func (b *KafkaBroker) GetMinOffset(topic string) int64 { // TimeToOffset returns the offset of the first message with a timestamp equal or // greater than the time given. -func (b *KafkaBroker) TimeToOffset(topic string, t time.Time) (int64, error) { +func (b *KafkaBroker) TimeToOffset(ctx context.Context, topic string, t time.Time) (int64, error) { return b.Client.GetOffset(topic, 0, t.UnixNano()/int64(time.Millisecond)) } // DeleteTopic deletes the topic from the Kafka cluster -func (b *KafkaBroker) DeleteTopic(topic string) error { +func (b *KafkaBroker) DeleteTopic(ctx context.Context, topic string) error { var err error @@ -292,6 +295,7 @@ func (b *KafkaBroker) Consume(ctx context.Context, topic string, offset int64, i log.WithFields( log.Fields{ + "trace_id": ctx.Value("trace_id"), "type": "backend_log", "backend_service": "kafka", "topic": topic, @@ -310,6 +314,7 @@ func (b *KafkaBroker) Consume(ctx context.Context, topic string, offset int64, i if offset < oldOff { log.WithFields( log.Fields{ + "trace_id": ctx.Value("trace_id"), "type": "backend_log", "backend_service": "kafka", "topic": topic, @@ -325,6 +330,7 @@ func (b *KafkaBroker) Consume(ctx context.Context, topic string, offset int64, i if err != nil { log.WithFields( log.Fields{ + "trace_id": ctx.Value("trace_id"), "type": "backend_log", "backend_service": "kafka", "topic": topic, @@ -338,6 +344,7 @@ func (b *KafkaBroker) Consume(ctx context.Context, topic string, offset int64, i if err := partitionConsumer.Close(); err != nil { log.WithFields( log.Fields{ + "trace_id": ctx.Value("trace_id"), "type": "backend_log", "backend_service": "kafka", "topic": topic, @@ -374,6 +381,7 @@ ConsumerLoop: log.WithFields( log.Fields{ + "trace_id": ctx.Value("trace_id"), "type": "backend_log", "backend_service": "kafka", "topic": topic, diff --git a/brokers/mock.go b/brokers/mock.go index 989d585f..99eff922 100644 --- a/brokers/mock.go +++ b/brokers/mock.go @@ -93,10 +93,10 @@ func (b *MockBroker) Initialize(peers []string) { } // Publish function publish a message to the broker -func (b *MockBroker) Publish(topic string, msg messages.Message) (string, string, int, int64, error) { +func (b *MockBroker) Publish(ctx context.Context, topic string, msg messages.Message) (string, string, int, int64, error) { payload, _ := msg.ExportJSON() b.MsgList = append(b.MsgList, payload) - off := b.GetMaxOffset(topic) - 1 + off := b.GetMaxOffset(ctx, topic) - 1 msgID := strconv.FormatInt(off, 10) // split the name that SHOULD come in the form of project_uuid.topic_name s := strings.Split(topic, ".") @@ -104,12 +104,12 @@ func (b *MockBroker) Publish(topic string, msg messages.Message) (string, string } // GetOffset returns a current topic's offset -func (b *MockBroker) GetMaxOffset(topic string) int64 { +func (b *MockBroker) GetMaxOffset(ctx context.Context, topic string) int64 { return int64(len(b.MsgList) + 1) } // GetOffset returns a current topic's offset -func (b *MockBroker) GetMinOffset(topic string) int64 { +func (b *MockBroker) GetMinOffset(ctx context.Context, topic string) int64 { return int64(len(b.MsgList)) } @@ -119,7 +119,7 @@ func (b *MockBroker) Consume(ctx context.Context, topic string, offset int64, im } // Delete topic from the broker -func (b *MockBroker) DeleteTopic(topic string) error { +func (b *MockBroker) DeleteTopic(ctx context.Context, topic string) error { _, ok := b.Topics[topic] if !ok { @@ -130,7 +130,7 @@ func (b *MockBroker) DeleteTopic(topic string) error { return nil } -func (b *MockBroker) TimeToOffset(topic string, time time.Time) (int64, error) { +func (b *MockBroker) TimeToOffset(ctx context.Context, topic string, time time.Time) (int64, error) { topicTimeIndices, ok := b.TopicTimeIndices[topic] diff --git a/handlers/errors.go b/handlers/errors.go index 95723dd7..1360e766 100644 --- a/handlers/errors.go +++ b/handlers/errors.go @@ -25,7 +25,7 @@ type APIError struct { Reason string `json:"reason"` } -// api err to be used when dealing with an invalid request body +// APIErrorInvalidRequestBody to be used when dealing with an invalid request body var APIErrorInvalidRequestBody = func() APIErrorRoot { apiErrBody := APIErrorBody{ @@ -39,7 +39,7 @@ var APIErrorInvalidRequestBody = func() APIErrorRoot { } } -// api err to be used when a name provided through the url parameters is not valid +// APIErrorInvalidName to be used when a name provided through the url parameters is not valid var APIErrorInvalidName = func(key string) APIErrorRoot { apiErrBody := APIErrorBody{ @@ -53,7 +53,7 @@ var APIErrorInvalidName = func(key string) APIErrorRoot { } } -// api err to be used when data provided is invalid +// APIErrorInvalidData to be used when data provided is invalid var APIErrorInvalidData = func(msg string) APIErrorRoot { apiErrBody := APIErrorBody{ @@ -67,7 +67,7 @@ var APIErrorInvalidData = func(msg string) APIErrorRoot { } } -// api err to be used when argument's provided are invalid according to the resource +// APIErrorInvalidArgument to be used when argument's provided are invalid according to the resource var APIErrorInvalidArgument = func(resource string) APIErrorRoot { apiErrBody := APIErrorBody{ @@ -81,7 +81,7 @@ var APIErrorInvalidArgument = func(resource string) APIErrorRoot { } } -// api err to be used when a user is unauthorized +// APIErrorUnauthorized to be used when a user is unauthorized var APIErrorUnauthorized = func() APIErrorRoot { apiErrBody := APIErrorBody{ @@ -95,7 +95,7 @@ var APIErrorUnauthorized = func() APIErrorRoot { } } -// api err to be used when access to a resource is forbidden for the request user +// APIErrorForbidden to be used when access to a resource is forbidden for the request user var APIErrorForbidden = func() APIErrorRoot { apiErrBody := APIErrorBody{ @@ -109,13 +109,13 @@ var APIErrorForbidden = func() APIErrorRoot { } } -// api err to be used when access to a resource is forbidden for the request user +// APIErrorForbiddenWithMsg to be used when access to a resource is forbidden for the request user var APIErrorForbiddenWithMsg = func(msg string) APIErrorRoot { apiErrBody := APIErrorBody{Code: http.StatusForbidden, Message: fmt.Sprintf("Access to this resource is forbidden. %v", msg), Status: "FORBIDDEN"} return APIErrorRoot{Body: apiErrBody} } -// api err for dealing with absent resources +// APIErrorNotFound for dealing with absent resources var APIErrorNotFound = func(resource string) APIErrorRoot { apiErrBody := APIErrorBody{ @@ -129,7 +129,7 @@ var APIErrorNotFound = func(resource string) APIErrorRoot { } } -// api err for dealing with timeouts +// APIErrorTimeout for dealing with timeouts var APIErrorTimeout = func(msg string) APIErrorRoot { apiErrBody := APIErrorBody{ @@ -143,7 +143,7 @@ var APIErrorTimeout = func(msg string) APIErrorRoot { } } -// api err for dealing with already existing resources +// APIErrorConflict for dealing with already existing resources var APIErrorConflict = func(resource string) APIErrorRoot { apiErrBody := APIErrorBody{ @@ -157,7 +157,7 @@ var APIErrorConflict = func(resource string) APIErrorRoot { } } -// api error to be used when push enabled false +// APIErrorPushConflict to be used when push enabled false var APIErrorPushConflict = func() APIErrorRoot { apiErrBody := APIErrorBody{ @@ -171,7 +171,7 @@ var APIErrorPushConflict = func() APIErrorRoot { } } -// api error to be used to format generic conflict errors +// APIErrorGenericConflict to be used to format generic conflict errors var APIErrorGenericConflict = func(msg string) APIErrorRoot { apiErrBody := APIErrorBody{ @@ -185,7 +185,7 @@ var APIErrorGenericConflict = func(msg string) APIErrorRoot { } } -// api error to be used when push enabled false +// APIErrorPullNoTopic to be used when push enabled false var APIErrorPullNoTopic = func() APIErrorRoot { apiErrBody := APIErrorBody{ @@ -199,7 +199,7 @@ var APIErrorPullNoTopic = func() APIErrorRoot { } } -// api err for dealing with too large messages +// APIErrTooLargeMessage for dealing with too large messages var APIErrTooLargeMessage = func(resource string) APIErrorRoot { apiErrBody := APIErrorBody{ @@ -213,7 +213,7 @@ var APIErrTooLargeMessage = func(resource string) APIErrorRoot { } } -// api err for dealing with generic internal errors +// APIErrGenericInternal for dealing with generic internal errors var APIErrGenericInternal = func(msg string) APIErrorRoot { apiErrBody := APIErrorBody{ @@ -227,7 +227,7 @@ var APIErrGenericInternal = func(msg string) APIErrorRoot { } } -// api err for dealing with generic internal errors +// APIErrPushVerification for dealing with generic internal errors var APIErrPushVerification = func(msg string) APIErrorRoot { apiErrBody := APIErrorBody{ @@ -241,7 +241,7 @@ var APIErrPushVerification = func(msg string) APIErrorRoot { } } -// api err for dealing with internal errors when marshaling json to struct +// APIErrExportJSON for dealing with internal errors when marshaling json to struct var APIErrExportJSON = func() APIErrorRoot { apiErrBody := APIErrorBody{ @@ -255,7 +255,7 @@ var APIErrExportJSON = func() APIErrorRoot { } } -// api err for dealing with internal errors when querying the datastore +// APIErrQueryDatastore for dealing with internal errors when querying the datastore var APIErrQueryDatastore = func() APIErrorRoot { apiErrBody := APIErrorBody{ @@ -269,7 +269,7 @@ var APIErrQueryDatastore = func() APIErrorRoot { } } -// api err for dealing with internal errors related to acknowledgement +// APIErrHandlingAcknowledgement for dealing with internal errors related to acknowledgement var APIErrHandlingAcknowledgement = func() APIErrorRoot { apiErrBody := APIErrorBody{ @@ -283,7 +283,7 @@ var APIErrHandlingAcknowledgement = func() APIErrorRoot { } } -// api err for dealing with generic backend errors +// APIErrGenericBackend for dealing with generic backend errors var APIErrGenericBackend = func() APIErrorRoot { apiErrBody := APIErrorBody{ @@ -297,7 +297,7 @@ var APIErrGenericBackend = func() APIErrorRoot { } } -// api error to be used when push enabled true but push worker was not able to be retrieved +// APIErrInternalPush to be used when push enabled true but push worker was not able to be retrieved var APIErrInternalPush = func() APIErrorRoot { apiErrBody := APIErrorBody{ diff --git a/handlers/handlers.go b/handlers/handlers.go index df39cd5e..ef33e101 100644 --- a/handlers/handlers.go +++ b/handlers/handlers.go @@ -16,6 +16,7 @@ import ( gorillaContext "github.com/gorilla/context" "github.com/gorilla/mux" log "github.com/sirupsen/logrus" + "github.com/twinj/uuid" "net/http" "sort" "time" @@ -24,7 +25,8 @@ import ( // WrapValidate handles validation func WrapValidate(hfn http.HandlerFunc) http.HandlerFunc { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) urlVars := mux.Vars(r) // sort keys @@ -38,7 +40,7 @@ func WrapValidate(hfn http.HandlerFunc) http.HandlerFunc { for _, key := range keys { if validation.ValidName(urlVars[key]) == false { err := APIErrorInvalidName(key) - respondErr(w, err) + respondErr(rCTX, w, err) return } } @@ -61,7 +63,11 @@ func WrapMockAuthConfig(hfn http.HandlerFunc, cfg *config.APICfg, brk brokers.Br nStr := str.Clone() defer nStr.Close() - projectUUID := projects.GetUUIDByName(urlVars["project"], nStr) + traceId := uuid.NewV4().String() + gorillaContext.Set(r, "trace_id", traceId) + + projectUUID := projects.GetUUIDByName(context.WithValue(context.Background(), "trace_id", traceId), + urlVars["project"], nStr) gorillaContext.Set(r, "auth_project_uuid", projectUUID) gorillaContext.Set(r, "brk", brk) gorillaContext.Set(r, "str", nStr) @@ -86,6 +92,8 @@ func WrapConfig(hfn http.HandlerFunc, cfg *config.APICfg, brk brokers.Broker, st nStr := str.Clone() defer nStr.Close() + traceId := uuid.NewV4().String() + gorillaContext.Set(r, "trace_id", traceId) gorillaContext.Set(r, "brk", brk) gorillaContext.Set(r, "str", nStr) gorillaContext.Set(r, "mgr", mgr) @@ -107,6 +115,17 @@ func WrapLog(hfn http.Handler, name string) http.HandlerFunc { start := time.Now() + log.WithFields( + log.Fields{ + "type": "request_log", + "method": r.Method, + "path": r.URL.Path, + "action": name, + "requester": gorillaContext.Get(r, "auth_user_uuid"), + "trace_id": gorillaContext.Get(r, "trace_id"), + }, + ).Info("New Request accepted . . .") + hfn.ServeHTTP(w, r) log.WithFields( @@ -117,6 +136,7 @@ func WrapLog(hfn http.Handler, name string) http.HandlerFunc { "action": name, "requester": gorillaContext.Get(r, "auth_user_uuid"), "processing_time": time.Since(start).String(), + "trace_id": gorillaContext.Get(r, "trace_id"), }, ).Info("") }) @@ -125,6 +145,8 @@ func WrapLog(hfn http.Handler, name string) http.HandlerFunc { // WrapAuthenticate handle wrapper to apply authentication func WrapAuthenticate(hfn http.Handler, extractToken RequestTokenExtractStrategy) http.HandlerFunc { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) urlVars := mux.Vars(r) @@ -133,7 +155,7 @@ func WrapAuthenticate(hfn http.Handler, extractToken RequestTokenExtractStrategy // if the url parameter 'key' is empty or absent, end the request with an unauthorized response if apiKey == "" { err := APIErrorUnauthorized() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -141,14 +163,14 @@ func WrapAuthenticate(hfn http.Handler, extractToken RequestTokenExtractStrategy serviceToken := gorillaContext.Get(r, "auth_service_token").(string) projectName := urlVars["project"] - projectUUID := projects.GetUUIDByName(urlVars["project"], refStr) + projectUUID := projects.GetUUIDByName(rCTX, urlVars["project"], refStr) // In all cases instead of project create if "projects:create" != mux.CurrentRoute(r).GetName() { // Check if given a project name the project wasn't found if projectName != "" && projectUUID == "" { apiErr := APIErrorNotFound("project") - respondErr(w, apiErr) + respondErr(rCTX, w, apiErr) return } } @@ -163,10 +185,10 @@ func WrapAuthenticate(hfn http.Handler, extractToken RequestTokenExtractStrategy return } - roles, user := auth.Authenticate(projectUUID, apiKey, refStr) + roles, user := auth.Authenticate(rCTX, projectUUID, apiKey, refStr) if len(roles) > 0 { - userUUID := auth.GetUUIDByName(user, refStr) + userUUID := auth.GetUUIDByName(rCTX, user, refStr) gorillaContext.Set(r, "auth_roles", roles) gorillaContext.Set(r, "auth_user", user) gorillaContext.Set(r, "auth_user_uuid", userUUID) @@ -174,7 +196,7 @@ func WrapAuthenticate(hfn http.Handler, extractToken RequestTokenExtractStrategy hfn.ServeHTTP(w, r) } else { err := APIErrorUnauthorized() - respondErr(w, err) + respondErr(rCTX, w, err) } }) @@ -183,6 +205,8 @@ func WrapAuthenticate(hfn http.Handler, extractToken RequestTokenExtractStrategy // WrapAuthorize handle wrapper to apply authorization func WrapAuthorize(hfn http.Handler, routeName string, extractToken RequestTokenExtractStrategy) http.HandlerFunc { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) refStr := gorillaContext.Get(r, "str").(stores.Store) refRoles := gorillaContext.Get(r, "auth_roles").([]string) @@ -195,17 +219,19 @@ func WrapAuthorize(hfn http.Handler, routeName string, extractToken RequestToken return } - if auth.Authorize(routeName, refRoles, refStr) { + if auth.Authorize(rCTX, routeName, refRoles, refStr) { hfn.ServeHTTP(w, r) } else { err := APIErrorForbidden() - respondErr(w, err) + respondErr(rCTX, w, err) } }) } // HealthCheck returns an ok message to make sure the service is up and running func HealthCheck(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) var err error var bytes []byte @@ -235,16 +261,16 @@ func HealthCheck(w http.ResponseWriter, r *http.Request) { tokenExtractStrategy := GetRequestTokenExtractStrategy(authOption) token := tokenExtractStrategy(r) - user, _ := auth.GetUserByToken(token, refStr) + user, _ := auth.GetUserByToken(rCTX, token, refStr) // if the user has a name, the token is valid if user.Name == "" { - respondErr(w, APIErrorForbidden()) + respondErr(rCTX, w, APIErrorForbidden()) return } if !auth.IsAdminViewer(user.ServiceRoles) && !auth.IsServiceAdmin(user.ServiceRoles) { - respondErr(w, APIErrorUnauthorized()) + respondErr(rCTX, w, APIErrorUnauthorized()) return } @@ -255,7 +281,7 @@ func HealthCheck(w http.ResponseWriter, r *http.Request) { } if pushEnabled { - _, err := auth.GetPushWorker(pwToken, refStr) + _, err := auth.GetPushWorker(rCTX, pwToken, refStr) if err != nil { healthMsg.Status = "warning" } @@ -273,7 +299,7 @@ func HealthCheck(w http.ResponseWriter, r *http.Request) { if bytes, err = json.MarshalIndent(healthMsg, "", " "); err != nil { err := APIErrGenericInternal(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -282,6 +308,8 @@ func HealthCheck(w http.ResponseWriter, r *http.Request) { // ListVersion displays version information about the service func ListVersion(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) // Add content type header to the response contentType := "application/json" @@ -307,7 +335,7 @@ func ListVersion(w http.ResponseWriter, r *http.Request) { token := tokenExtractStrategy(r) if token != "" { - user, _ := auth.GetUserByToken(token, refStr) + user, _ := auth.GetUserByToken(rCTX, token, refStr) // set uuid for logging gorillaContext.Set(r, "auth_user_uuid", user.UUID) @@ -323,7 +351,7 @@ func ListVersion(w http.ResponseWriter, r *http.Request) { output, err := json.MarshalIndent(v, "", " ") if err != nil { err := APIErrGenericInternal(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -337,9 +365,10 @@ func respondOK(w http.ResponseWriter, output []byte) { } // respondErr is used to finalize response writer with proper error codes and error output -func respondErr(w http.ResponseWriter, apiErr APIErrorRoot) { +func respondErr(ctx context.Context, w http.ResponseWriter, apiErr APIErrorRoot) { log.WithFields( log.Fields{ + "trace_id": ctx.Value("trace_id"), "type": "service_log", "status_code": apiErr.Body.Code, }, @@ -350,7 +379,8 @@ func respondErr(w http.ResponseWriter, apiErr APIErrorRoot) { w.Write(output) } -// A function type that refers to all the functions that can extract an api access token from the request +// RequestTokenExtractStrategy is a function type that refers to all the functions +// that can extract an api access token from the request type RequestTokenExtractStrategy func(r *http.Request) string // UrlKeyExtract extracts the api access token from the url parameter key diff --git a/handlers/metrics.go b/handlers/metrics.go index 4e6c57fe..aec6e06b 100644 --- a/handlers/metrics.go +++ b/handlers/metrics.go @@ -1,6 +1,7 @@ package handlers import ( + "context" "encoding/json" "fmt" "github.com/ARGOeu/argo-messaging/config" @@ -19,6 +20,8 @@ import ( // OpMetrics (GET) all operational metrics func OpMetrics(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) // Init output output := []byte("") @@ -32,11 +35,11 @@ func OpMetrics(w http.ResponseWriter, r *http.Request) { refStr := gorillaContext.Get(r, "str").(stores.Store) // Get Results Object - res, err := metrics.GetUsageCpuMem(refStr) + res, err := metrics.GetUsageCpuMem(rCTX, refStr) if err != nil && err.Error() != "not found" { err := APIErrQueryDatastore() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -45,7 +48,7 @@ func OpMetrics(w http.ResponseWriter, r *http.Request) { if err != nil { err := APIErrExportJSON() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -56,6 +59,8 @@ func OpMetrics(w http.ResponseWriter, r *http.Request) { // VaMetrics (GET) retrieves metrics regrading projects, users, subscriptions, topics func VaMetrics(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) // Add content type header to the response contentType := "application/json" @@ -74,7 +79,7 @@ func VaMetrics(w http.ResponseWriter, r *http.Request) { startDate, err = time.Parse("2006-01-02", r.URL.Query().Get("start_date")) if err != nil { err := APIErrorInvalidData("Start date is not in valid format") - respondErr(w, err) + respondErr(rCTX, w, err) return } } else { @@ -86,7 +91,7 @@ func VaMetrics(w http.ResponseWriter, r *http.Request) { endDate, err = time.Parse("2006-01-02", r.URL.Query().Get("end_date")) if err != nil { err := APIErrorInvalidData("End date is not in valid format") - respondErr(w, err) + respondErr(rCTX, w, err) return } } else { @@ -95,7 +100,7 @@ func VaMetrics(w http.ResponseWriter, r *http.Request) { if startDate.After(endDate) { err := APIErrorInvalidData("Start date cannot be after the end date") - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -105,17 +110,17 @@ func VaMetrics(w http.ResponseWriter, r *http.Request) { projectsList = strings.Split(projectsUrlValue, ",") } - vr, err := metrics.GetVAReport(projectsList, startDate, endDate, refStr) + vr, err := metrics.GetVAReport(rCTX, projectsList, startDate, endDate, refStr) if err != nil { err := APIErrorNotFound(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } output, err := json.MarshalIndent(vr, "", " ") if err != nil { err := APIErrExportJSON() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -127,6 +132,8 @@ func VaMetrics(w http.ResponseWriter, r *http.Request) { // alongside service operational metrics // This handler is supposed to be used for project admins in order to get usage information for their projects func UserUsageReport(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) // Add content type header to the response contentType := "application/json" @@ -143,20 +150,20 @@ func UserUsageReport(w http.ResponseWriter, r *http.Request) { if token == "" { err := APIErrorUnauthorized() - respondErr(w, err) + respondErr(rCTX, w, err) return } - user, err := auth.GetUserByToken(token, refStr) + user, err := auth.GetUserByToken(rCTX, token, refStr) if err != nil { if err.Error() == "not found" { err := APIErrorUnauthorized() - respondErr(w, err) + respondErr(rCTX, w, err) return } err := APIErrQueryDatastore() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -168,7 +175,7 @@ func UserUsageReport(w http.ResponseWriter, r *http.Request) { startDate, err = time.Parse("2006-01-02", r.URL.Query().Get("start_date")) if err != nil { err := APIErrorInvalidData("Start date is not in valid format") - respondErr(w, err) + respondErr(rCTX, w, err) return } } else { @@ -180,7 +187,7 @@ func UserUsageReport(w http.ResponseWriter, r *http.Request) { endDate, err = time.Parse("2006-01-02", r.URL.Query().Get("end_date")) if err != nil { err := APIErrorInvalidData("End date is not in valid format") - respondErr(w, err) + respondErr(rCTX, w, err) return } } else { @@ -189,7 +196,7 @@ func UserUsageReport(w http.ResponseWriter, r *http.Request) { if startDate.After(endDate) { err := APIErrorInvalidData("Start date cannot be after the end date") - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -242,10 +249,10 @@ func UserUsageReport(w http.ResponseWriter, r *http.Request) { }, } if len(queryProjects) > 0 { - vr, err = metrics.GetUserUsageReport(queryProjects, startDate, endDate, refStr) + vr, err = metrics.GetUserUsageReport(rCTX, queryProjects, startDate, endDate, refStr) if err != nil { err := APIErrorNotFound(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } } @@ -253,7 +260,7 @@ func UserUsageReport(w http.ResponseWriter, r *http.Request) { output, err := json.MarshalIndent(vr, "", " ") if err != nil { err := APIErrExportJSON() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -263,6 +270,8 @@ func UserUsageReport(w http.ResponseWriter, r *http.Request) { // ProjectMetrics (GET) metrics for one project (number of topics) func ProjectMetrics(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) // Init output output := []byte("") @@ -292,17 +301,17 @@ func ProjectMetrics(w http.ResponseWriter, r *http.Request) { numTopics := int64(0) numSubs := int64(0) - numTopics2, err2 := metrics.GetProjectTopics(projectUUID, refStr) + numTopics2, err2 := metrics.GetProjectTopics(rCTX, projectUUID, refStr) if err2 != nil { err := APIErrExportJSON() - respondErr(w, err) + respondErr(rCTX, w, err) return } numTopics = numTopics2 - numSubs2, err2 := metrics.GetProjectSubs(projectUUID, refStr) + numSubs2, err2 := metrics.GetProjectSubs(rCTX, projectUUID, refStr) if err2 != nil { err := APIErrExportJSON() - respondErr(w, err) + respondErr(rCTX, w, err) return } numSubs = numSubs2 @@ -310,9 +319,9 @@ func ProjectMetrics(w http.ResponseWriter, r *http.Request) { var timePoints []metrics.Timepoint var err error - if timePoints, err = metrics.GetDailyProjectMsgCount(projectUUID, refStr); err != nil { + if timePoints, err = metrics.GetDailyProjectMsgCount(rCTX, projectUUID, refStr); err != nil { err := APIErrGenericBackend() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -322,10 +331,10 @@ func ProjectMetrics(w http.ResponseWriter, r *http.Request) { res.Metrics = append(res.Metrics, m2) // ProjectUUID User topics aggregation - m3, err := metrics.AggrProjectUserTopics(projectUUID, refStr) + m3, err := metrics.AggrProjectUserTopics(rCTX, projectUUID, refStr) if err != nil { err := APIErrExportJSON() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -334,10 +343,10 @@ func ProjectMetrics(w http.ResponseWriter, r *http.Request) { } // ProjectUUID User subscriptions aggregation - m4, err := metrics.AggrProjectUserSubs(projectUUID, refStr) + m4, err := metrics.AggrProjectUserSubs(rCTX, projectUUID, refStr) if err != nil { err := APIErrExportJSON() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -352,7 +361,7 @@ func ProjectMetrics(w http.ResponseWriter, r *http.Request) { resJSON, err := res.ExportJSON() if err != nil { err := APIErrExportJSON() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -363,6 +372,8 @@ func ProjectMetrics(w http.ResponseWriter, r *http.Request) { // TopicMetrics (GET) metrics for one topic func TopicMetrics(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) // Init output output := []byte("") @@ -391,24 +402,24 @@ func TopicMetrics(w http.ResponseWriter, r *http.Request) { if refAuthResource && auth.IsPublisher(refRoles) { - if auth.PerResource(projectUUID, "topics", urlTopic, refUserUUID, refStr) == false { + if auth.PerResource(rCTX, projectUUID, "topics", urlTopic, refUserUUID, refStr) == false { err := APIErrorForbidden() - respondErr(w, err) + respondErr(rCTX, w, err) return } } // Number of bytes and number of messages - resultsMsg, err := topics.FindMetric(projectUUID, urlTopic, refStr) + resultsMsg, err := topics.FindMetric(rCTX, projectUUID, urlTopic, refStr) if err != nil { if err.Error() == "not found" { err := APIErrorNotFound("Topic") - respondErr(w, err) + respondErr(rCTX, w, err) return } err := APIErrGenericInternal(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -416,22 +427,22 @@ func TopicMetrics(w http.ResponseWriter, r *http.Request) { numBytes := resultsMsg.TotalBytes numSubs := int64(0) - numSubs, err = metrics.GetProjectSubsByTopic(projectUUID, urlTopic, refStr) + numSubs, err = metrics.GetProjectSubsByTopic(rCTX, projectUUID, urlTopic, refStr) if err != nil { if err.Error() == "not found" { err := APIErrorNotFound("Topic") - respondErr(w, err) + respondErr(rCTX, w, err) return } err := APIErrGenericBackend() - respondErr(w, err) + respondErr(rCTX, w, err) return } var timePoints []metrics.Timepoint - if timePoints, err = metrics.GetDailyTopicMsgCount(projectUUID, urlTopic, refStr); err != nil { + if timePoints, err = metrics.GetDailyTopicMsgCount(rCTX, projectUUID, urlTopic, refStr); err != nil { err := APIErrGenericBackend() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -449,7 +460,7 @@ func TopicMetrics(w http.ResponseWriter, r *http.Request) { resJSON, err := res.ExportJSON() if err != nil { err := APIErrExportJSON() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -460,6 +471,8 @@ func TopicMetrics(w http.ResponseWriter, r *http.Request) { // SubMetrics (GET) metrics for one subscription func SubMetrics(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) // Init output output := []byte("") @@ -488,23 +501,23 @@ func SubMetrics(w http.ResponseWriter, r *http.Request) { if refAuthResource && auth.IsConsumer(refRoles) { - if auth.PerResource(projectUUID, "subscriptions", urlSub, refUserUUID, refStr) == false { + if auth.PerResource(rCTX, projectUUID, "subscriptions", urlSub, refUserUUID, refStr) == false { err := APIErrorForbidden() - respondErr(w, err) + respondErr(rCTX, w, err) return } } - resultMsg, err := subscriptions.FindMetric(projectUUID, urlSub, refStr) + resultMsg, err := subscriptions.FindMetric(rCTX, projectUUID, urlSub, refStr) if err != nil { if err.Error() == "not found" { err := APIErrorNotFound("Subscription") - respondErr(w, err) + respondErr(rCTX, w, err) return } err := APIErrGenericBackend() - respondErr(w, err) + respondErr(rCTX, w, err) } numMsg := resultMsg.MsgNum @@ -521,7 +534,7 @@ func SubMetrics(w http.ResponseWriter, r *http.Request) { resJSON, err := res.ExportJSON() if err != nil { err := APIErrExportJSON() - respondErr(w, err) + respondErr(rCTX, w, err) return } diff --git a/handlers/projects.go b/handlers/projects.go index 3aa34734..aa5bec76 100644 --- a/handlers/projects.go +++ b/handlers/projects.go @@ -1,6 +1,7 @@ package handlers import ( + "context" "encoding/json" "fmt" "github.com/ARGOeu/argo-messaging/auth" @@ -19,6 +20,8 @@ import ( // ProjectDelete (DEL) deletes an existing project (also removes it's topics and subscriptions) func ProjectDelete(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) // Init output output := []byte("") @@ -35,15 +38,15 @@ func ProjectDelete(w http.ResponseWriter, r *http.Request) { // Get project UUID First to use as reference projectUUID := gorillaContext.Get(r, "auth_project_uuid").(string) // RemoveProject removes also attached subs and topics from the datastore - err := projects.RemoveProject(projectUUID, refStr) + err := projects.RemoveProject(rCTX, projectUUID, refStr) if err != nil { if err.Error() == "not found" { err := APIErrorNotFound("ProjectUUID") - respondErr(w, err) + respondErr(rCTX, w, err) return } err := APIErrGenericInternal(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -53,6 +56,8 @@ func ProjectDelete(w http.ResponseWriter, r *http.Request) { // ProjectUpdate (PUT) updates the name or the description of an existing project func ProjectUpdate(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) // Init output output := []byte("") @@ -69,7 +74,7 @@ func ProjectUpdate(w http.ResponseWriter, r *http.Request) { body, err := ioutil.ReadAll(r.Body) if err != nil { err := APIErrorInvalidRequestBody() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -77,31 +82,30 @@ func ProjectUpdate(w http.ResponseWriter, r *http.Request) { postBody, err := projects.GetFromJSON(body) if err != nil { err := APIErrorInvalidRequestBody() - respondErr(w, err) - log.Error(string(body[:])) + respondErr(rCTX, w, err) return } modified := time.Now().UTC() // Get Result Object - res, err := projects.UpdateProject(projectUUID, postBody.Name, postBody.Description, modified, refStr) + res, err := projects.UpdateProject(rCTX, projectUUID, postBody.Name, postBody.Description, modified, refStr) if err != nil { if err.Error() == "not found" { err := APIErrorNotFound("ProjectUUID") - respondErr(w, err) + respondErr(rCTX, w, err) return } if strings.HasPrefix(err.Error(), "invalid") { err := APIErrorInvalidData(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } err := APIErrGenericInternal(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -109,7 +113,7 @@ func ProjectUpdate(w http.ResponseWriter, r *http.Request) { resJSON, err := res.ExportJSON() if err != nil { err := APIErrExportJSON() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -120,6 +124,8 @@ func ProjectUpdate(w http.ResponseWriter, r *http.Request) { // ProjectCreate (POST) creates a new project func ProjectCreate(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) // Init output output := []byte("") @@ -141,7 +147,7 @@ func ProjectCreate(w http.ResponseWriter, r *http.Request) { body, err := ioutil.ReadAll(r.Body) if err != nil { err := APIErrorInvalidRequestBody() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -149,7 +155,7 @@ func ProjectCreate(w http.ResponseWriter, r *http.Request) { postBody, err := projects.GetFromJSON(body) if err != nil { err := APIErrorInvalidArgument("Project") - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -157,16 +163,16 @@ func ProjectCreate(w http.ResponseWriter, r *http.Request) { created := time.Now().UTC() // Get Result Object - res, err := projects.CreateProject(uuid, urlProject, created, refUserUUID, postBody.Description, refStr) + res, err := projects.CreateProject(rCTX, uuid, urlProject, created, refUserUUID, postBody.Description, refStr) if err != nil { if err.Error() == "exists" { err := APIErrorConflict("Project") - respondErr(w, err) + respondErr(rCTX, w, err) return } err := APIErrGenericInternal(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -174,7 +180,7 @@ func ProjectCreate(w http.ResponseWriter, r *http.Request) { resJSON, err := res.ExportJSON() if err != nil { err := APIErrExportJSON() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -185,6 +191,8 @@ func ProjectCreate(w http.ResponseWriter, r *http.Request) { // ProjectListAll (GET) all projects func ProjectListAll(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) // Init output output := []byte("") @@ -199,11 +207,11 @@ func ProjectListAll(w http.ResponseWriter, r *http.Request) { // Get Results Object - res, err := projects.Find("", "", refStr) + res, err := projects.Find(rCTX, "", "", refStr) if err != nil && err.Error() != "not found" { err := APIErrQueryDatastore() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -212,7 +220,7 @@ func ProjectListAll(w http.ResponseWriter, r *http.Request) { if err != nil { err := APIErrExportJSON() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -223,6 +231,8 @@ func ProjectListAll(w http.ResponseWriter, r *http.Request) { // ProjectListOne (GET) one project func ProjectListOne(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) // Init output output := []byte("") @@ -240,17 +250,17 @@ func ProjectListOne(w http.ResponseWriter, r *http.Request) { refStr := gorillaContext.Get(r, "str").(stores.Store) // Get Results Object - results, err := projects.Find("", urlProject, refStr) + results, err := projects.Find(rCTX, "", urlProject, refStr) if err != nil { if err.Error() == "not found" { err := APIErrorNotFound("ProjectUUID") - respondErr(w, err) + respondErr(rCTX, w, err) return } err := APIErrQueryDatastore() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -260,7 +270,7 @@ func ProjectListOne(w http.ResponseWriter, r *http.Request) { if err != nil { err := APIErrExportJSON() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -271,6 +281,8 @@ func ProjectListOne(w http.ResponseWriter, r *http.Request) { // ProjectUserListOne (GET) one user member of a specific project func ProjectUserListOne(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) // Init output output := []byte("") @@ -293,17 +305,17 @@ func ProjectUserListOne(w http.ResponseWriter, r *http.Request) { priviledged := auth.IsServiceAdmin(refRoles) // Get Results Object - results, err := auth.FindUsers(projectUUID, "", urlUser, priviledged, refStr) + results, err := auth.FindUsers(rCTX, projectUUID, "", urlUser, priviledged, refStr) if err != nil { if err.Error() == "not found" { err := APIErrorNotFound("User") - respondErr(w, err) + respondErr(rCTX, w, err) return } err := APIErrQueryDatastore() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -314,7 +326,7 @@ func ProjectUserListOne(w http.ResponseWriter, r *http.Request) { if err != nil { err := APIErrExportJSON() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -325,6 +337,8 @@ func ProjectUserListOne(w http.ResponseWriter, r *http.Request) { // ProjectUserCreate (POST) creates a user under the respective project by the project's admin func ProjectUserCreate(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) // Init output output := []byte("") @@ -347,7 +361,7 @@ func ProjectUserCreate(w http.ResponseWriter, r *http.Request) { body, err := ioutil.ReadAll(r.Body) if err != nil { err := APIErrorInvalidRequestBody() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -355,8 +369,7 @@ func ProjectUserCreate(w http.ResponseWriter, r *http.Request) { postBody, err := auth.GetUserFromJSON(body) if err != nil { err := APIErrorInvalidArgument("User") - respondErr(w, err) - log.Error(string(body[:])) + respondErr(rCTX, w, err) return } @@ -364,10 +377,10 @@ func ProjectUserCreate(w http.ResponseWriter, r *http.Request) { postBody.ServiceRoles = []string{} // allow the user to be created to only have reference to the project under which is being created - prName := projects.GetNameByUUID(refProjUUID, refStr) + prName := projects.GetNameByUUID(rCTX, refProjUUID, refStr) if prName == "" { err := APIErrGenericInternal("Internal Error") - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -395,29 +408,29 @@ func ProjectUserCreate(w http.ResponseWriter, r *http.Request) { created := time.Now().UTC() // Get Result Object - res, err := auth.CreateUser(uuid, urlUser, "", "", "", "", postBody.Projects, token, postBody.Email, postBody.ServiceRoles, created, refUserUUID, refStr) + res, err := auth.CreateUser(rCTX, uuid, urlUser, "", "", "", "", postBody.Projects, token, postBody.Email, postBody.ServiceRoles, created, refUserUUID, refStr) if err != nil { if err.Error() == "exists" { err := APIErrorConflict("User") - respondErr(w, err) + respondErr(rCTX, w, err) return } if strings.HasPrefix(err.Error(), "invalid") { err := APIErrorInvalidData(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } if strings.HasPrefix(err.Error(), "duplicate") { err := APIErrorInvalidData(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } err := APIErrGenericInternal(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -425,7 +438,7 @@ func ProjectUserCreate(w http.ResponseWriter, r *http.Request) { resJSON, err := res.ExportJSON() if err != nil { err := APIErrExportJSON() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -436,6 +449,8 @@ func ProjectUserCreate(w http.ResponseWriter, r *http.Request) { // ProjectUserUpdate (PUT) updates a user under the respective project by the project's admin func ProjectUserUpdate(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) // Init output output := []byte("") @@ -455,10 +470,10 @@ func ProjectUserUpdate(w http.ResponseWriter, r *http.Request) { refRoles := gorillaContext.Get(r, "auth_roles").([]string) // allow the user to be updated to only have reference to the project under which is being updated - prName := projects.GetNameByUUID(refProjUUID, refStr) + prName := projects.GetNameByUUID(rCTX, refProjUUID, refStr) if prName == "" { err := APIErrGenericInternal("Internal Error") - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -466,7 +481,7 @@ func ProjectUserUpdate(w http.ResponseWriter, r *http.Request) { body, err := ioutil.ReadAll(r.Body) if err != nil { err := APIErrorInvalidRequestBody() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -474,20 +489,20 @@ func ProjectUserUpdate(w http.ResponseWriter, r *http.Request) { postBody, err := auth.GetUserFromJSON(body) if err != nil { err := APIErrorInvalidArgument("User") - respondErr(w, err) + respondErr(rCTX, w, err) return } - u, err := auth.FindUsers("", "", urlUser, true, refStr) + u, err := auth.FindUsers(rCTX, "", "", urlUser, true, refStr) if err != nil { if err.Error() == "not found" { err := APIErrorNotFound("User") - respondErr(w, err) + respondErr(rCTX, w, err) return } err := APIErrQueryDatastore() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -518,7 +533,7 @@ func ProjectUserUpdate(w http.ResponseWriter, r *http.Request) { if !found { err := APIErrorForbiddenWithMsg("User is not a member of the project") - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -537,46 +552,46 @@ func ProjectUserUpdate(w http.ResponseWriter, r *http.Request) { userOrg := u.One().Organization userDesc := u.One().Description - _, err = auth.UpdateUser(userUUID, userFN, userLN, userOrg, userDesc, userName, userProjects, userEmail, userSRoles, modified, false, refStr) + _, err = auth.UpdateUser(rCTX, userUUID, userFN, userLN, userOrg, userDesc, userName, userProjects, userEmail, userSRoles, modified, false, refStr) if err != nil { // In case of invalid project or role in post body if err.Error() == "not found" { err := APIErrorNotFound("User") - respondErr(w, err) + respondErr(rCTX, w, err) return } if strings.HasPrefix(err.Error(), "invalid") { err := APIErrorInvalidData(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } if strings.HasPrefix(err.Error(), "duplicate") { err := APIErrorInvalidData(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } err := APIErrGenericInternal(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } - stored, err := auth.FindUsers(refProjUUID, userUUID, urlUser, privileged, refStr) + stored, err := auth.FindUsers(rCTX, refProjUUID, userUUID, urlUser, privileged, refStr) if err != nil { if err.Error() == "not found" { err := APIErrorNotFound("User") - respondErr(w, err) + respondErr(rCTX, w, err) return } err := APIErrGenericInternal(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -584,7 +599,7 @@ func ProjectUserUpdate(w http.ResponseWriter, r *http.Request) { resJSON, err := json.MarshalIndent(stored.One(), "", " ") if err != nil { err := APIErrExportJSON() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -595,6 +610,8 @@ func ProjectUserUpdate(w http.ResponseWriter, r *http.Request) { // ProjectUserRemove (POST) removes a user from the respective project func ProjectUserRemove(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) // Add content type header to the response contentType := "application/json" @@ -609,18 +626,18 @@ func ProjectUserRemove(w http.ResponseWriter, r *http.Request) { refStr := gorillaContext.Get(r, "str").(stores.Store) refProjUUID := gorillaContext.Get(r, "auth_project_uuid").(string) - projName := projects.GetNameByUUID(refProjUUID, refStr) + projName := projects.GetNameByUUID(rCTX, refProjUUID, refStr) - u, err := auth.FindUsers("", "", urlUser, true, refStr) + u, err := auth.FindUsers(rCTX, "", "", urlUser, true, refStr) if err != nil { if err.Error() == "not found" { err := APIErrorNotFound("User") - respondErr(w, err) + respondErr(rCTX, w, err) return } err := APIErrQueryDatastore() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -639,7 +656,7 @@ func ProjectUserRemove(w http.ResponseWriter, r *http.Request) { if !found { err := APIErrorForbiddenWithMsg("User is not a member of the project") - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -654,19 +671,19 @@ func ProjectUserRemove(w http.ResponseWriter, r *http.Request) { userOrg := u.One().Organization userDesc := u.One().Description - _, err = auth.UpdateUser(userUUID, userFN, userLN, userOrg, userDesc, userName, userProjects, userEmail, userSRoles, modified, false, refStr) + _, err = auth.UpdateUser(rCTX, userUUID, userFN, userLN, userOrg, userDesc, userName, userProjects, userEmail, userSRoles, modified, false, refStr) if err != nil { // In case of invalid project or role in post body if err.Error() == "not found" { err := APIErrorNotFound("User") - respondErr(w, err) + respondErr(rCTX, w, err) return } err := APIErrGenericInternal(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -676,6 +693,8 @@ func ProjectUserRemove(w http.ResponseWriter, r *http.Request) { // ProjectUserAdd (POST) adds a user to the respective project func ProjectUserAdd(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) // Add content type header to the response contentType := "application/json" @@ -691,18 +710,18 @@ func ProjectUserAdd(w http.ResponseWriter, r *http.Request) { refProjUUID := gorillaContext.Get(r, "auth_project_uuid").(string) refRoles := gorillaContext.Get(r, "auth_roles").([]string) - projName := projects.GetNameByUUID(refProjUUID, refStr) + projName := projects.GetNameByUUID(rCTX, refProjUUID, refStr) - u, err := auth.FindUsers("", "", urlUser, true, refStr) + u, err := auth.FindUsers(rCTX, "", "", urlUser, true, refStr) if err != nil { if err.Error() == "not found" { err := APIErrorNotFound("User") - respondErr(w, err) + respondErr(rCTX, w, err) return } err := APIErrQueryDatastore() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -710,7 +729,7 @@ func ProjectUserAdd(w http.ResponseWriter, r *http.Request) { body, err := ioutil.ReadAll(r.Body) if err != nil { err := APIErrorInvalidRequestBody() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -719,7 +738,7 @@ func ProjectUserAdd(w http.ResponseWriter, r *http.Request) { err = json.Unmarshal(body, &data) if err != nil { err := APIErrorInvalidRequestBody() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -734,7 +753,7 @@ func ProjectUserAdd(w http.ResponseWriter, r *http.Request) { if found { err := APIErrorGenericConflict("User is already a member of the project") - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -755,42 +774,42 @@ func ProjectUserAdd(w http.ResponseWriter, r *http.Request) { Roles: data.Roles, }) - _, err = auth.UpdateUser(userUUID, userFN, userLN, userOrg, userDesc, userName, userProjects, userEmail, userSRoles, modified, false, refStr) + _, err = auth.UpdateUser(rCTX, userUUID, userFN, userLN, userOrg, userDesc, userName, userProjects, userEmail, userSRoles, modified, false, refStr) if err != nil { // In case of invalid project or role in post body if err.Error() == "not found" { err := APIErrorNotFound("User") - respondErr(w, err) + respondErr(rCTX, w, err) return } if strings.HasPrefix(err.Error(), "invalid") { err := APIErrorInvalidData(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } err := APIErrGenericInternal(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } // Write response privileged := auth.IsServiceAdmin(refRoles) fmt.Println(privileged) - results, err := auth.FindUsers(refProjUUID, "", urlUser, privileged, refStr) + results, err := auth.FindUsers(rCTX, refProjUUID, "", urlUser, privileged, refStr) if err != nil { if err.Error() == "not found" { err := APIErrorNotFound("User") - respondErr(w, err) + respondErr(rCTX, w, err) return } err := APIErrQueryDatastore() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -801,7 +820,7 @@ func ProjectUserAdd(w http.ResponseWriter, r *http.Request) { if err != nil { err := APIErrExportJSON() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -811,6 +830,8 @@ func ProjectUserAdd(w http.ResponseWriter, r *http.Request) { // ProjectListUsers (GET) all users belonging to a project func ProjectListUsers(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) var err error var pageSize int @@ -842,9 +863,16 @@ func ProjectListUsers(w http.ResponseWriter, r *http.Request) { if strPageSize != "" { if pageSize, err = strconv.Atoi(strPageSize); err != nil { - log.Errorf("Pagesize %v produced an error while being converted to int: %v", strPageSize, err.Error()) + log.WithFields( + log.Fields{ + "trace_id": rCTX.Value("trace_id"), + "type": "request_log", + "page_size": pageSize, + "error": err.Error(), + }, + ).Error("error while converting page size to int") err := APIErrorInvalidData("Invalid page size") - respondErr(w, err) + respondErr(rCTX, w, err) return } } @@ -854,11 +882,11 @@ func ProjectListUsers(w http.ResponseWriter, r *http.Request) { // Get Results Object - call is always privileged because this handler is only accessible by service admins paginatedUsers, err = - auth.PaginatedFindUsers(pageToken, int64(pageSize), projectUUID, privileged, usersDetailedView, refStr) + auth.PaginatedFindUsers(rCTX, pageToken, int64(pageSize), projectUUID, privileged, usersDetailedView, refStr) if err != nil { err := APIErrorInvalidData("Invalid page token") - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -867,7 +895,7 @@ func ProjectListUsers(w http.ResponseWriter, r *http.Request) { if err != nil { err := APIErrExportJSON() - respondErr(w, err) + respondErr(rCTX, w, err) return } diff --git a/handlers/projects_test.go b/handlers/projects_test.go index 4e13b4a8..e51a0927 100644 --- a/handlers/projects_test.go +++ b/handlers/projects_test.go @@ -2,6 +2,7 @@ package handlers import ( "bytes" + "context" "fmt" "github.com/ARGOeu/argo-messaging/auth" "github.com/ARGOeu/argo-messaging/brokers" @@ -717,7 +718,7 @@ func (suite *ProjectsHandlersTestSuite) TestProjectUserCreate() { router.HandleFunc("/v1/projects/{project}/members/{user}", WrapMockAuthConfig(ProjectUserCreate, cfgKafka, &brk, str, &mgr, pc)) router.ServeHTTP(w, req) if t.expectedStatusCode == 200 { - u, _ := auth.FindUsers("argo_uuid", "", t.user, true, str) + u, _ := auth.FindUsers(context.Background(), "argo_uuid", "", t.user, true, str) t.expectedResponse = strings.Replace(t.expectedResponse, "{{UUID}}", u.List[0].UUID, 1) t.expectedResponse = strings.Replace(t.expectedResponse, "{{TOKEN}}", u.List[0].Token, 1) t.expectedResponse = strings.Replace(t.expectedResponse, "{{CON}}", u.List[0].CreatedOn, 1) @@ -936,7 +937,7 @@ func (suite *ProjectsHandlersTestSuite) TestProjectUserUpdate() { router.HandleFunc("/v1/projects/{project}/members/{user}", WrapMockAuthConfig(ProjectUserUpdate, cfgKafka, &brk, str, &mgr, pc, t.authRole)) router.ServeHTTP(w, req) if t.expectedStatusCode == 200 { - u, _ := auth.FindUsers("argo_uuid", "", t.user, true, str) + u, _ := auth.FindUsers(context.Background(), "argo_uuid", "", t.user, true, str) t.expectedResponse = strings.Replace(t.expectedResponse, "{{UUID}}", u.List[0].UUID, 1) t.expectedResponse = strings.Replace(t.expectedResponse, "{{TOKEN}}", u.List[0].Token, 1) t.expectedResponse = strings.Replace(t.expectedResponse, "{{CON}}", u.List[0].CreatedOn, 1) @@ -1170,7 +1171,7 @@ func (suite *ProjectsHandlersTestSuite) TestProjectUserAdd() { router.HandleFunc("/v1/projects/{project}/members/{user}:add", WrapMockAuthConfig(ProjectUserAdd, cfgKafka, &brk, str, &mgr, pc, t.authRole)) router.ServeHTTP(w, req) if t.expectedStatusCode == 200 { - u, _ := auth.FindUsers("argo_uuid", "", t.user, true, str) + u, _ := auth.FindUsers(context.Background(), "argo_uuid", "", t.user, true, str) t.expectedResponse = strings.Replace(t.expectedResponse, "{{UUID}}", u.List[0].UUID, 1) t.expectedResponse = strings.Replace(t.expectedResponse, "{{TOKEN}}", u.List[0].Token, 1) t.expectedResponse = strings.Replace(t.expectedResponse, "{{CON}}", u.List[0].CreatedOn, 1) diff --git a/handlers/registrations.go b/handlers/registrations.go index 93505459..f6bdfaf2 100644 --- a/handlers/registrations.go +++ b/handlers/registrations.go @@ -1,6 +1,7 @@ package handlers import ( + "context" "encoding/json" "fmt" "github.com/ARGOeu/argo-messaging/auth" @@ -14,8 +15,10 @@ import ( "time" ) -// RegisterUser(POST) registers a new user +// RegisterUser (POST) registers a new user func RegisterUser(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) // Init output output := []byte("") @@ -34,7 +37,7 @@ func RegisterUser(w http.ResponseWriter, r *http.Request) { body, err := ioutil.ReadAll(r.Body) if err != nil { err := APIErrorInvalidRequestBody() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -43,14 +46,14 @@ func RegisterUser(w http.ResponseWriter, r *http.Request) { err = json.Unmarshal(body, &requestBody) if err != nil { err := APIErrorInvalidArgument("User") - respondErr(w, err) + respondErr(rCTX, w, err) return } // check if a user with that name already exists - if auth.ExistsWithName(requestBody.Name, refStr) { + if auth.ExistsWithName(rCTX, requestBody.Name, refStr) { err := APIErrorConflict("User") - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -59,23 +62,23 @@ func RegisterUser(w http.ResponseWriter, r *http.Request) { tkn, err := auth.GenToken() if err != nil { err := APIErrGenericInternal("") - respondErr(w, err) + respondErr(rCTX, w, err) return } - ur, err := auth.RegisterUser(uuid, requestBody.Name, requestBody.FirstName, requestBody.LastName, requestBody.Email, + ur, err := auth.RegisterUser(rCTX, uuid, requestBody.Name, requestBody.FirstName, requestBody.LastName, requestBody.Email, requestBody.Organization, requestBody.Description, registered, tkn, auth.PendingRegistrationStatus, refStr) if err != nil { err := APIErrGenericInternal(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } output, err = json.MarshalIndent(ur, "", " ") if err != nil { err := APIErrGenericInternal(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -84,6 +87,8 @@ func RegisterUser(w http.ResponseWriter, r *http.Request) { // AcceptUserRegister (POST) accepts a user registration and creates the respective user func AcceptRegisterUser(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) contentType := "application/json" charset := "utf-8" @@ -97,17 +102,17 @@ func AcceptRegisterUser(w http.ResponseWriter, r *http.Request) { refStr := gorillaContext.Get(r, "str").(stores.Store) refUserUUID := gorillaContext.Get(r, "auth_user_uuid").(string) - ru, err := auth.FindUserRegistration(regUUID, auth.PendingRegistrationStatus, refStr) + ru, err := auth.FindUserRegistration(rCTX, regUUID, auth.PendingRegistrationStatus, refStr) if err != nil { if err.Error() == "not found" { err := APIErrorNotFound("User registration") - respondErr(w, err) + respondErr(rCTX, w, err) return } err := APIErrGenericInternal(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -115,32 +120,38 @@ func AcceptRegisterUser(w http.ResponseWriter, r *http.Request) { token, err := auth.GenToken() // generate a new user token created := time.Now().UTC() // Get Result Object - res, err := auth.CreateUser(userUUID, ru.Name, ru.FirstName, ru.LastName, ru.Organization, ru.Description, + res, err := auth.CreateUser(rCTX, userUUID, ru.Name, ru.FirstName, ru.LastName, ru.Organization, ru.Description, []auth.ProjectRoles{}, token, ru.Email, []string{}, created, refUserUUID, refStr) if err != nil { if err.Error() == "exists" { err := APIErrorConflict("User") - respondErr(w, err) + respondErr(rCTX, w, err) return } err := APIErrGenericInternal(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } // update the registration - err = auth.UpdateUserRegistration(regUUID, auth.AcceptedRegistrationStatus, "", refUserUUID, created, refStr) + err = auth.UpdateUserRegistration(rCTX, regUUID, auth.AcceptedRegistrationStatus, "", refUserUUID, created, refStr) if err != nil { - log.Errorf("Could not update registration, %v", err.Error()) + log.WithFields( + log.Fields{ + "trace_id": rCTX.Value("trace_id"), + "type": "service_log", + "error": err.Error(), + }, + ).Error("Could not update registration") } // Output result to JSON resJSON, err := res.ExportJSON() if err != nil { err := APIErrExportJSON() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -149,6 +160,8 @@ func AcceptRegisterUser(w http.ResponseWriter, r *http.Request) { } func DeclineRegisterUser(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) contentType := "application/json" charset := "utf-8" @@ -162,17 +175,17 @@ func DeclineRegisterUser(w http.ResponseWriter, r *http.Request) { // Grab context references refStr := gorillaContext.Get(r, "str").(stores.Store) - _, err := auth.FindUserRegistration(regUUID, auth.PendingRegistrationStatus, refStr) + _, err := auth.FindUserRegistration(rCTX, regUUID, auth.PendingRegistrationStatus, refStr) if err != nil { if err.Error() == "not found" { err := APIErrorNotFound("User registration") - respondErr(w, err) + respondErr(rCTX, w, err) return } err := APIErrGenericInternal(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -183,15 +196,15 @@ func DeclineRegisterUser(w http.ResponseWriter, r *http.Request) { err = json.NewDecoder(r.Body).Decode(&reqBody) if err != nil { err := APIErrorInvalidRequestBody() - respondErr(w, err) + respondErr(rCTX, w, err) return } } - err = auth.UpdateUserRegistration(regUUID, auth.DeclinedRegistrationStatus, reqBody["comment"], refUserUUID, time.Now().UTC(), refStr) + err = auth.UpdateUserRegistration(rCTX, regUUID, auth.DeclinedRegistrationStatus, reqBody["comment"], refUserUUID, time.Now().UTC(), refStr) if err != nil { err := APIErrGenericInternal(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -199,8 +212,10 @@ func DeclineRegisterUser(w http.ResponseWriter, r *http.Request) { } -// ListOneRegistration(GET) retrieves information for a specific registration based on the provided activation token +// ListOneRegistration (GET) retrieves information for a specific registration based on the provided activation token func ListOneRegistration(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) contentType := "application/json" charset := "utf-8" @@ -213,32 +228,34 @@ func ListOneRegistration(w http.ResponseWriter, r *http.Request) { // Grab context references refStr := gorillaContext.Get(r, "str").(stores.Store) - ur, err := auth.FindUserRegistration(regUUID, "", refStr) + ur, err := auth.FindUserRegistration(rCTX, regUUID, "", refStr) if err != nil { if err.Error() == "not found" { err := APIErrorNotFound("User registration") - respondErr(w, err) + respondErr(rCTX, w, err) return } err := APIErrGenericInternal(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } urb, err := json.MarshalIndent(ur, "", " ") if err != nil { err := APIErrGenericInternal(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } respondOK(w, urb) } -// ListAllRegistrations(GET) retrieves information about all the registrations in the service +// ListAllRegistrations (GET) retrieves information about all the registrations in the service func ListAllRegistrations(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) contentType := "application/json" charset := "utf-8" @@ -253,18 +270,18 @@ func ListAllRegistrations(w http.ResponseWriter, r *http.Request) { org := r.URL.Query().Get("organization") activationToken := r.URL.Query().Get("activation_token") - ur, err := auth.FindUserRegistrations(status, activationToken, name, email, org, refStr) + ur, err := auth.FindUserRegistrations(rCTX, status, activationToken, name, email, org, refStr) if err != nil { err := APIErrGenericInternal(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } urb, err := json.MarshalIndent(ur, "", " ") if err != nil { err := APIErrGenericInternal(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -273,6 +290,8 @@ func ListAllRegistrations(w http.ResponseWriter, r *http.Request) { // DeleteRegistration (DELETE) removes a registration from the service's store func DeleteRegistration(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) contentType := "application/json" charset := "utf-8" @@ -285,24 +304,24 @@ func DeleteRegistration(w http.ResponseWriter, r *http.Request) { // Grab context references refStr := gorillaContext.Get(r, "str").(stores.Store) - _, err := auth.FindUserRegistration(regUUID, "", refStr) + _, err := auth.FindUserRegistration(rCTX, regUUID, "", refStr) if err != nil { if err.Error() == "not found" { err := APIErrorNotFound("User registration") - respondErr(w, err) + respondErr(rCTX, w, err) return } err := APIErrGenericInternal(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } - err = auth.DeleteUserRegistration(regUUID, refStr) + err = auth.DeleteUserRegistration(rCTX, regUUID, refStr) if err != nil { err := APIErrGenericInternal(err.Error()) - respondErr(w, err) + respondErr(rCTX , w, err) return } respondOK(w, nil) diff --git a/handlers/registrations_test.go b/handlers/registrations_test.go index 8843afd4..69b5b231 100644 --- a/handlers/registrations_test.go +++ b/handlers/registrations_test.go @@ -1,6 +1,7 @@ package handlers import ( + "context" "fmt" "github.com/ARGOeu/argo-messaging/auth" "github.com/ARGOeu/argo-messaging/brokers" @@ -191,7 +192,7 @@ func (suite *RegistrationsHandlersTestSuite) TestAcceptRegisterUser() { router.HandleFunc("/v1/registrations/{uuid}:accept", WrapMockAuthConfig(AcceptRegisterUser, cfgKafka, &brk, str, &mgr, pc)) router.ServeHTTP(w, req) if t.expectedStatusCode == 200 { - u, _ := auth.FindUsers("", "", t.uname, true, str) + u, _ := auth.FindUsers(context.Background(), "", "", t.uname, true, str) t.expectedResponse = strings.Replace(t.expectedResponse, "{{UUID}}", u.List[0].UUID, 1) t.expectedResponse = strings.Replace(t.expectedResponse, "{{TOKEN}}", u.List[0].Token, 1) t.expectedResponse = strings.Replace(t.expectedResponse, "{{CON}}", u.List[0].CreatedOn, 1) diff --git a/handlers/schemas.go b/handlers/schemas.go index 72ce5c4f..39ad91f3 100644 --- a/handlers/schemas.go +++ b/handlers/schemas.go @@ -2,6 +2,7 @@ package handlers import ( "bytes" + "context" "encoding/base64" "encoding/json" "fmt" @@ -14,8 +15,10 @@ import ( "net/http" ) -// SchemaCreate(POST) handles the creation of a new schema +// SchemaCreate (POST) handles the creation of a new schema func SchemaCreate(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) // Add content type header to the response contentType := "application/json" @@ -39,28 +42,28 @@ func SchemaCreate(w http.ResponseWriter, r *http.Request) { err := json.NewDecoder(r.Body).Decode(&schema) if err != nil { err := APIErrorInvalidArgument("Schema") - respondErr(w, err) + respondErr(rCTX, w, err) return } - schema, err = schemas.Create(projectUUID, schemaUUID, schemaName, schema.Type, schema.RawSchema, refStr) + schema, err = schemas.Create(rCTX, projectUUID, schemaUUID, schemaName, schema.Type, schema.RawSchema, refStr) if err != nil { if err.Error() == "exists" { err := APIErrorConflict("Schema") - respondErr(w, err) + respondErr(rCTX, w, err) return } if err.Error() == "unsupported" { err := APIErrorInvalidData(schemas.UnsupportedSchemaError) - respondErr(w, err) + respondErr(rCTX, w, err) return } err := APIErrorInvalidData(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -68,8 +71,10 @@ func SchemaCreate(w http.ResponseWriter, r *http.Request) { respondOK(w, output) } -// SchemaListOne(GET) retrieves information about the requested schema +// SchemaListOne (GET) retrieves information about the requested schema func SchemaListOne(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) // Add content type header to the response contentType := "application/json" @@ -85,16 +90,16 @@ func SchemaListOne(w http.ResponseWriter, r *http.Request) { // Get project UUID First to use as reference projectUUID := gorillaContext.Get(r, "auth_project_uuid").(string) - schemasList, err := schemas.Find(projectUUID, "", schemaName, refStr) + schemasList, err := schemas.Find(rCTX, projectUUID, "", schemaName, refStr) if err != nil { err := APIErrGenericInternal(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } if schemasList.Empty() { err := APIErrorNotFound("Schema") - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -102,8 +107,10 @@ func SchemaListOne(w http.ResponseWriter, r *http.Request) { respondOK(w, output) } -// SchemaLisAll(GET) retrieves all the schemas under the given project +// SchemaLisAll (GET) retrieves all the schemas under the given project func SchemaListAll(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) // Add content type header to the response contentType := "application/json" @@ -115,10 +122,10 @@ func SchemaListAll(w http.ResponseWriter, r *http.Request) { // Get project UUID First to use as reference projectUUID := gorillaContext.Get(r, "auth_project_uuid").(string) - schemasList, err := schemas.Find(projectUUID, "", "", refStr) + schemasList, err := schemas.Find(rCTX, projectUUID, "", "", refStr) if err != nil { err := APIErrGenericInternal(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -126,8 +133,10 @@ func SchemaListAll(w http.ResponseWriter, r *http.Request) { respondOK(w, output) } -// SchemaUpdate(PUT) updates the given schema +// SchemaUpdate (PUT) updates the given schema func SchemaUpdate(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) // Add content type header to the response contentType := "application/json" @@ -143,16 +152,16 @@ func SchemaUpdate(w http.ResponseWriter, r *http.Request) { // Get project UUID First to use as reference projectUUID := gorillaContext.Get(r, "auth_project_uuid").(string) - schemasList, err := schemas.Find(projectUUID, "", schemaName, refStr) + schemasList, err := schemas.Find(rCTX, projectUUID, "", schemaName, refStr) if err != nil { err := APIErrGenericInternal(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } if schemasList.Empty() { err := APIErrorNotFound("Schema") - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -161,7 +170,7 @@ func SchemaUpdate(w http.ResponseWriter, r *http.Request) { err = json.NewDecoder(r.Body).Decode(&updatedSchema) if err != nil { err := APIErrorInvalidArgument("Schema") - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -169,30 +178,30 @@ func SchemaUpdate(w http.ResponseWriter, r *http.Request) { _, schemaName, err := schemas.ExtractSchema(updatedSchema.FullName) if err != nil { err := APIErrorInvalidData(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } updatedSchema.Name = schemaName } - schema, err := schemas.Update(schemasList.Schemas[0], updatedSchema.Name, updatedSchema.Type, updatedSchema.RawSchema, refStr) + schema, err := schemas.Update(rCTX, schemasList.Schemas[0], updatedSchema.Name, updatedSchema.Type, updatedSchema.RawSchema, refStr) if err != nil { if err.Error() == "exists" { err := APIErrorConflict("Schema") - respondErr(w, err) + respondErr(rCTX, w, err) return } if err.Error() == "unsupported" { err := APIErrorInvalidData(schemas.UnsupportedSchemaError) - respondErr(w, err) + respondErr(rCTX, w, err) return } err := APIErrorInvalidData(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -201,6 +210,8 @@ func SchemaUpdate(w http.ResponseWriter, r *http.Request) { } func SchemaDelete(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) // Add content type header to the response contentType := "application/json" @@ -217,31 +228,33 @@ func SchemaDelete(w http.ResponseWriter, r *http.Request) { // Get project UUID First to use as reference projectUUID := gorillaContext.Get(r, "auth_project_uuid").(string) - schemasList, err := schemas.Find(projectUUID, "", schemaName, refStr) + schemasList, err := schemas.Find(rCTX, projectUUID, "", schemaName, refStr) if err != nil { err := APIErrGenericInternal(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } if schemasList.Empty() { err := APIErrorNotFound("Schema") - respondErr(w, err) + respondErr(rCTX, w, err) return } - err = schemas.Delete(schemasList.Schemas[0].UUID, refStr) + err = schemas.Delete(rCTX, schemasList.Schemas[0].UUID, refStr) if err != nil { err := APIErrGenericInternal(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } respondOK(w, nil) } -// SchemaValidateMessage(POST) validates the given message against the schema +// SchemaValidateMessage (POST) validates the given message against the schema func SchemaValidateMessage(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) // Add content type header to the response contentType := "application/json" @@ -257,16 +270,16 @@ func SchemaValidateMessage(w http.ResponseWriter, r *http.Request) { // Get project UUID First to use as reference projectUUID := gorillaContext.Get(r, "auth_project_uuid").(string) - schemasList, err := schemas.Find(projectUUID, "", schemaName, refStr) + schemasList, err := schemas.Find(rCTX, projectUUID, "", schemaName, refStr) if err != nil { err := APIErrGenericInternal(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } if schemasList.Empty() { err := APIErrorNotFound("Schema") - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -274,7 +287,7 @@ func SchemaValidateMessage(w http.ResponseWriter, r *http.Request) { _, err = buf.ReadFrom(r.Body) if err != nil { err := APIErrorInvalidData(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -294,7 +307,7 @@ func SchemaValidateMessage(w http.ResponseWriter, r *http.Request) { err := json.Unmarshal(buf.Bytes(), &body) if err != nil { err := APIErrorInvalidRequestBody() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -310,7 +323,7 @@ func SchemaValidateMessage(w http.ResponseWriter, r *http.Request) { } else { err := APIErrorInvalidArgument("Schema Payload") - respondErr(w, err) + respondErr(rCTX, w, err) return } } @@ -319,11 +332,11 @@ func SchemaValidateMessage(w http.ResponseWriter, r *http.Request) { if err != nil { if err.Error() == "500" { err := APIErrGenericInternal(schemas.GenericError) - respondErr(w, err) + respondErr(rCTX, w, err) return } else { err := APIErrorInvalidData(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } } diff --git a/handlers/subscriptions.go b/handlers/subscriptions.go index f7dfe4eb..a8f3e1f5 100644 --- a/handlers/subscriptions.go +++ b/handlers/subscriptions.go @@ -24,6 +24,8 @@ import ( // SubAck (POST) acknowledge the consumption of specific messages func SubAck(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) // Init output output := []byte("") @@ -43,7 +45,7 @@ func SubAck(w http.ResponseWriter, r *http.Request) { body, err := ioutil.ReadAll(r.Body) if err != nil { err := APIErrorInvalidRequestBody() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -51,7 +53,7 @@ func SubAck(w http.ResponseWriter, r *http.Request) { postBody, err := subscriptions.GetAckFromJSON(body) if err != nil { err := APIErrorInvalidData("Invalid ack parameter") - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -61,22 +63,22 @@ func SubAck(w http.ResponseWriter, r *http.Request) { // Check if sub exists - cur_sub, err := subscriptions.Find(projectUUID, "", subName, "", 0, refStr) + cur_sub, err := subscriptions.Find(rCTX, projectUUID, "", subName, "", 0, refStr) if err != nil { err := APIErrHandlingAcknowledgement() - respondErr(w, err) + respondErr(rCTX, w, err) return } if len(cur_sub.Subscriptions) == 0 { err := APIErrorNotFound("Subscription") - respondErr(w, err) + respondErr(rCTX, w, err) return } // Get list of AckIDs if postBody.IDs == nil { err := APIErrorInvalidData("Invalid ack id") - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -84,7 +86,7 @@ func SubAck(w http.ResponseWriter, r *http.Request) { for _, ackID := range postBody.IDs { if validation.ValidAckID(projectName, subName, ackID) == false { err := APIErrorInvalidData("Invalid ack id") - respondErr(w, err) + respondErr(rCTX, w, err) return } } @@ -93,7 +95,7 @@ func SubAck(w http.ResponseWriter, r *http.Request) { maxAckID, err := subscriptions.GetMaxAckID(postBody.IDs) if err != nil { err := APIErrHandlingAcknowledgement() - respondErr(w, err) + respondErr(rCTX, w, err) return } // Extract offset from max ackID @@ -101,7 +103,7 @@ func SubAck(w http.ResponseWriter, r *http.Request) { if err != nil { err := APIErrorInvalidData("Invalid ack id") - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -109,17 +111,17 @@ func SubAck(w http.ResponseWriter, r *http.Request) { t := time.Now().UTC() ts := t.Format(zSec) - err = refStr.UpdateSubOffsetAck(projectUUID, urlVars["subscription"], int64(off+1), ts) + err = refStr.UpdateSubOffsetAck(rCTX, projectUUID, urlVars["subscription"], int64(off+1), ts) if err != nil { if err.Error() == "ack timeout" { err := APIErrorTimeout(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } err := APIErrGenericInternal(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -133,6 +135,8 @@ func SubAck(w http.ResponseWriter, r *http.Request) { // SubListOne (GET) one subscription func SubListOne(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) // Init output output := []byte("") @@ -149,18 +153,18 @@ func SubListOne(w http.ResponseWriter, r *http.Request) { refStr := gorillaContext.Get(r, "str").(stores.Store) projectUUID := gorillaContext.Get(r, "auth_project_uuid").(string) - results, err := subscriptions.Find(projectUUID, "", urlVars["subscription"], "", 0, refStr) + results, err := subscriptions.Find(rCTX, projectUUID, "", urlVars["subscription"], "", 0, refStr) if err != nil { err := APIErrGenericBackend() - respondErr(w, err) + respondErr(rCTX, w, err) return } // If not found if results.Empty() { err := APIErrorNotFound("Subscription") - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -178,7 +182,7 @@ func SubListOne(w http.ResponseWriter, r *http.Request) { if err != nil { err := APIErrExportJSON() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -189,6 +193,8 @@ func SubListOne(w http.ResponseWriter, r *http.Request) { // SubSetOffset (PUT) sets subscriptions current offset func SubSetOffset(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) // Init output output := []byte("") @@ -207,7 +213,7 @@ func SubSetOffset(w http.ResponseWriter, r *http.Request) { body, err := ioutil.ReadAll(r.Body) if err != nil { err := APIErrorInvalidRequestBody() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -215,7 +221,7 @@ func SubSetOffset(w http.ResponseWriter, r *http.Request) { postBody, err := subscriptions.GetSetOffsetJSON(body) if err != nil { err := APIErrorInvalidArgument("Offset") - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -226,38 +232,40 @@ func SubSetOffset(w http.ResponseWriter, r *http.Request) { projectUUID := gorillaContext.Get(r, "auth_project_uuid").(string) // Find Subscription - results, err := subscriptions.Find(projectUUID, "", urlVars["subscription"], "", 0, refStr) + results, err := subscriptions.Find(rCTX, projectUUID, "", urlVars["subscription"], "", 0, refStr) if err != nil { err := APIErrGenericBackend() - respondErr(w, err) + respondErr(rCTX, w, err) return } // If not found if results.Empty() { err := APIErrorNotFound("Subscription") - respondErr(w, err) + respondErr(rCTX, w, err) return } brk_topic := projectUUID + "." + results.Subscriptions[0].Topic - min_offset := refBrk.GetMinOffset(brk_topic) - max_offset := refBrk.GetMaxOffset(brk_topic) + min_offset := refBrk.GetMinOffset(rCTX, brk_topic) + max_offset := refBrk.GetMaxOffset(rCTX, brk_topic) //Check if given offset is between min max if postBody.Offset < min_offset || postBody.Offset > max_offset { err := APIErrorInvalidData("Offset out of bounds") - respondErr(w, err) + respondErr(rCTX, w, err) } // Get subscription offsets - refStr.UpdateSubOffset(projectUUID, urlSub, postBody.Offset) + refStr.UpdateSubOffset(rCTX, projectUUID, urlSub, postBody.Offset) respondOK(w, output) } // SubGetOffsets (GET) gets offset indices from a subscription func SubGetOffsets(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) // Init output output := []byte("") @@ -276,31 +284,31 @@ func SubGetOffsets(w http.ResponseWriter, r *http.Request) { projectUUID := gorillaContext.Get(r, "auth_project_uuid").(string) - results, err := subscriptions.Find(projectUUID, "", urlVars["subscription"], "", 0, refStr) + results, err := subscriptions.Find(rCTX, projectUUID, "", urlVars["subscription"], "", 0, refStr) if err != nil { err := APIErrGenericBackend() - respondErr(w, err) + respondErr(rCTX, w, err) return } // If not found if results.Empty() { err := APIErrorNotFound("Subscription") - respondErr(w, err) + respondErr(rCTX, w, err) return } // Output result to JSON brkTopic := projectUUID + "." + results.Subscriptions[0].Topic curOffset := results.Subscriptions[0].Offset - minOffset := refBrk.GetMinOffset(brkTopic) - maxOffset := refBrk.GetMaxOffset(brkTopic) + minOffset := refBrk.GetMinOffset(rCTX, brkTopic) + maxOffset := refBrk.GetMaxOffset(rCTX, brkTopic) // if the current subscription offset is behind the min available offset for the topic // update it if curOffset < minOffset { - refStr.UpdateSubOffset(projectUUID, urlVars["subscription"], minOffset) + refStr.UpdateSubOffset(rCTX, projectUUID, urlVars["subscription"], minOffset) curOffset = minOffset } @@ -314,7 +322,7 @@ func SubGetOffsets(w http.ResponseWriter, r *http.Request) { resJSON, err := offResult.ExportJSON() if err != nil { err := APIErrExportJSON() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -325,6 +333,8 @@ func SubGetOffsets(w http.ResponseWriter, r *http.Request) { // SubTimeToOffset (GET) gets offset indices closest to a timestamp func SubTimeToOffset(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) // Init output output := []byte("") @@ -343,42 +353,41 @@ func SubTimeToOffset(w http.ResponseWriter, r *http.Request) { projectUUID := gorillaContext.Get(r, "auth_project_uuid").(string) - results, err := subscriptions.Find(projectUUID, "", urlVars["subscription"], "", 0, refStr) + results, err := subscriptions.Find(rCTX, projectUUID, "", urlVars["subscription"], "", 0, refStr) if err != nil { err := APIErrGenericBackend() - respondErr(w, err) + respondErr(rCTX, w, err) return } // If not found if results.Empty() { err := APIErrorNotFound("Subscription") - respondErr(w, err) + respondErr(rCTX, w, err) return } t, err := time.Parse("2006-01-02T15:04:05.000Z", r.URL.Query().Get("time")) if err != nil { err := APIErrorInvalidData("Time is not in valid Zulu format.") - respondErr(w, err) + respondErr(rCTX, w, err) return } // Output result to JSON brkTopic := projectUUID + "." + results.Subscriptions[0].Topic - off, err := refBrk.TimeToOffset(brkTopic, t.Local()) + off, err := refBrk.TimeToOffset(rCTX, brkTopic, t.Local()) if err != nil { - log.Errorf(err.Error()) err := APIErrGenericBackend() - respondErr(w, err) + respondErr(rCTX, w, err) return } if off < 0 { err := APIErrorGenericConflict("Timestamp is out of bounds for the subscription's topic/partition") - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -386,7 +395,7 @@ func SubTimeToOffset(w http.ResponseWriter, r *http.Request) { output, err = json.Marshal(topicOffset) if err != nil { err := APIErrExportJSON() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -395,6 +404,8 @@ func SubTimeToOffset(w http.ResponseWriter, r *http.Request) { // SubDelete (DEL) deletes an existing subscription func SubDelete(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) // Init output output := []byte("") @@ -412,29 +423,29 @@ func SubDelete(w http.ResponseWriter, r *http.Request) { projectUUID := gorillaContext.Get(r, "auth_project_uuid").(string) // Get Result Object - results, err := subscriptions.Find(projectUUID, "", urlVars["subscription"], "", 0, refStr) + results, err := subscriptions.Find(rCTX, projectUUID, "", urlVars["subscription"], "", 0, refStr) if err != nil { err := APIErrGenericBackend() - respondErr(w, err) + respondErr(rCTX, w, err) return } // If not found if results.Empty() { err := APIErrorNotFound("Subscription") - respondErr(w, err) + respondErr(rCTX, w, err) return } - err = subscriptions.RemoveSub(projectUUID, urlVars["subscription"], refStr) + err = subscriptions.RemoveSub(rCTX, projectUUID, urlVars["subscription"], refStr) if err != nil { if err.Error() == "not found" { err := APIErrorNotFound("Subscription") - respondErr(w, err) + respondErr(rCTX, w, err) return } err := APIErrGenericInternal(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -453,6 +464,8 @@ func SubDelete(w http.ResponseWriter, r *http.Request) { // SubModACL (POST) modifies the ACL func SubModACL(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) // Init output output := []byte("") @@ -471,7 +484,7 @@ func SubModACL(w http.ResponseWriter, r *http.Request) { body, err := ioutil.ReadAll(r.Body) if err != nil { err := APIErrorInvalidRequestBody() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -479,8 +492,7 @@ func SubModACL(w http.ResponseWriter, r *http.Request) { postBody, err := auth.GetACLFromJSON(body) if err != nil { err := APIErrorInvalidArgument("Subscription ACL") - respondErr(w, err) - log.Error(string(body[:])) + respondErr(rCTX, w, err) return } @@ -490,24 +502,24 @@ func SubModACL(w http.ResponseWriter, r *http.Request) { projectUUID := gorillaContext.Get(r, "auth_project_uuid").(string) // check if user list contain valid users for the given project - _, err = auth.AreValidUsers(projectUUID, postBody.AuthUsers, refStr) + _, err = auth.AreValidUsers(rCTX, projectUUID, postBody.AuthUsers, refStr) if err != nil { err := APIErrorRoot{Body: APIErrorBody{Code: http.StatusNotFound, Message: err.Error(), Status: "NOT_FOUND"}} - respondErr(w, err) + respondErr(rCTX, w, err) return } - err = auth.ModACL(projectUUID, "subscriptions", urlSub, postBody.AuthUsers, refStr) + err = auth.ModACL(rCTX, projectUUID, "subscriptions", urlSub, postBody.AuthUsers, refStr) if err != nil { if err.Error() == "not found" { err := APIErrorNotFound("Subscription") - respondErr(w, err) + respondErr(rCTX, w, err) return } err := APIErrGenericInternal(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -516,6 +528,8 @@ func SubModACL(w http.ResponseWriter, r *http.Request) { // SubModPush (POST) modifies the push configuration func SubModPush(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) // Init output output := []byte("") @@ -539,7 +553,7 @@ func SubModPush(w http.ResponseWriter, r *http.Request) { body, err := ioutil.ReadAll(r.Body) if err != nil { err := APIErrorInvalidRequestBody() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -547,22 +561,22 @@ func SubModPush(w http.ResponseWriter, r *http.Request) { postBody, err := subscriptions.GetFromJSON(body) if err != nil { err := APIErrorInvalidArgument("Subscription") - respondErr(w, err) + respondErr(rCTX, w, err) return } // Get Result Object - res, err := subscriptions.Find(projectUUID, "", subName, "", 0, refStr) + res, err := subscriptions.Find(rCTX, projectUUID, "", subName, "", 0, refStr) if err != nil { err := APIErrGenericBackend() - respondErr(w, err) + respondErr(rCTX, w, err) return } if res.Empty() { err := APIErrorNotFound("Subscription") - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -591,14 +605,14 @@ func SubModPush(w http.ResponseWriter, r *http.Request) { // check the state of the push functionality if !pushEnabled { err := APIErrorPushConflict() - respondErr(w, err) + respondErr(rCTX, w, err) return } - pushWorker, err = auth.GetPushWorker(pwToken, refStr) + pushWorker, err = auth.GetPushWorker(rCTX, pwToken, refStr) if err != nil { err := APIErrInternalPush() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -616,7 +630,7 @@ func SubModPush(w http.ResponseWriter, r *http.Request) { if !subscriptions.IsRetryPolicySupported(rPolicy) { err := APIErrorInvalidData(subscriptions.UnSupportedRetryPolicyError) - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -628,7 +642,7 @@ func SubModPush(w http.ResponseWriter, r *http.Request) { // Check if push endpoint is not a valid https:// endpoint if !(validation.IsValidHTTPS(pushEnd)) { err := APIErrorInvalidData("Push endpoint should be addressed by a valid https url") - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -639,9 +653,16 @@ func SubModPush(w http.ResponseWriter, r *http.Request) { if postBody.PushCfg.Pend != existingSub.PushCfg.Pend { vhash, err = auth.GenToken() if err != nil { - log.Errorf("Could not generate verification hash for subscription %v, %v", urlVars["subscription"], err.Error()) + log.WithFields( + log.Fields{ + "trace_id": rCTX.Value("trace_id"), + "type": "service_log", + "subscription": urlVars["subscription"], + "error": err.Error(), + }, + ).Error("Could not generate verification hash for subscription") err := APIErrGenericInternal("Could not generate verification hash") - respondErr(w, err) + respondErr(rCTX, w, err) return } // else keep the already existing data @@ -655,7 +676,7 @@ func SubModPush(w http.ResponseWriter, r *http.Request) { if authzType != "" { if !subscriptions.IsAuthorizationHeaderTypeSupported(authzType) { err := APIErrorInvalidData(subscriptions.UnSupportedAuthorizationHeader) - respondErr(w, err) + respondErr(rCTX, w, err) return } } @@ -672,9 +693,16 @@ func SubModPush(w http.ResponseWriter, r *http.Request) { if authzType == subscriptions.AutoGenerationAuthorizationHeader { authzHeaderValue, err = auth.GenToken() if err != nil { - log.Errorf("Could not generate authorization header for subscription %v, %v", urlVars["subscription"], err.Error()) + log.WithFields( + log.Fields{ + "trace_id": rCTX.Value("trace_id"), + "type": "service_log", + "subscription": urlVars["subscription"], + "error": err.Error(), + }, + ).Error("Could not generate auth header for subscription") err := APIErrGenericInternal("Could not generate authorization header") - respondErr(w, err) + respondErr(rCTX, w, err) return } } @@ -720,13 +748,13 @@ func SubModPush(w http.ResponseWriter, r *http.Request) { if postBody.PushCfg.MattermostUrl == "" { err := APIErrorInvalidData("Field mattermostUrl cannot be empty") - respondErr(w, err) + respondErr(rCTX, w, err) return } } else { err := APIErrorInvalidData(subscriptions.UnsupportedPushConfig) - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -751,22 +779,22 @@ func SubModPush(w http.ResponseWriter, r *http.Request) { MattermostChannel: mattermostChannel, Base64Decode: base64Decode, } - err = subscriptions.ModSubPush(projectUUID, subName, cfg, refStr) + err = subscriptions.ModSubPush(rCTX, projectUUID, subName, cfg, refStr) if err != nil { if err.Error() == "not found" { err := APIErrorNotFound("Subscription") - respondErr(w, err) + respondErr(rCTX, w, err) return } err := APIErrGenericInternal(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } // if this is an deactivate request, try to retrieve the push worker in order to remove him from the sub's acl if existingSub.PushCfg != (subscriptions.PushConfig{}) && postBody.PushCfg == (subscriptions.PushConfig{}) { - pushWorker, _ = auth.GetPushWorker(pwToken, refStr) + pushWorker, _ = auth.GetPushWorker(rCTX, pwToken, refStr) } // if the sub, was push enabled before the update and the endpoint was verified @@ -779,10 +807,10 @@ func SubModPush(w http.ResponseWriter, r *http.Request) { apsc.DeactivateSubscription(context.TODO(), existingSub.FullName).Result(false) // remove the push worker user from the sub's acl - err = auth.RemoveFromACL(projectUUID, "subscriptions", existingSub.Name, []string{pushWorker.Name}, refStr) + err = auth.RemoveFromACL(rCTX, projectUUID, "subscriptions", existingSub.Name, []string{pushWorker.Name}, refStr) if err != nil { err := APIErrGenericInternal(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } } @@ -821,18 +849,18 @@ func SubModPush(w http.ResponseWriter, r *http.Request) { apsc.ActivateSubscription(context.TODO(), s).Result(false) // modify the sub's acl with the push worker's uuid - err = auth.AppendToACL(projectUUID, "subscriptions", existingSub.Name, []string{pushWorker.Name}, refStr) + err = auth.AppendToACL(rCTX, projectUUID, "subscriptions", existingSub.Name, []string{pushWorker.Name}, refStr) if err != nil { err := APIErrGenericInternal(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } // link the sub's project with the push worker - err = auth.AppendToUserProjects(pushWorker.UUID, projectUUID, refStr) + err = auth.AppendToUserProjects(rCTX, pushWorker.UUID, projectUUID, refStr) if err != nil { err := APIErrGenericInternal(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } } @@ -844,6 +872,8 @@ func SubModPush(w http.ResponseWriter, r *http.Request) { // SubVerifyPushEndpoint (POST) verifies the ownership of a push endpoint registered in a push enabled subscription func SubVerifyPushEndpoint(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) // Add content type header to the response contentType := "application/json" @@ -869,29 +899,29 @@ func SubVerifyPushEndpoint(w http.ResponseWriter, r *http.Request) { // check the state of the push functionality if !pushEnabled { err := APIErrorPushConflict() - respondErr(w, err) + respondErr(rCTX, w, err) return } - pushW, err := auth.GetPushWorker(pwToken, refStr) + pushW, err := auth.GetPushWorker(rCTX, pwToken, refStr) if err != nil { err := APIErrInternalPush() - respondErr(w, err) + respondErr(rCTX, w, err) return } // Get Result Object - res, err := subscriptions.Find(projectUUID, "", subName, "", 0, refStr) + res, err := subscriptions.Find(rCTX, projectUUID, "", subName, "", 0, refStr) if err != nil { err := APIErrGenericBackend() - respondErr(w, err) + respondErr(rCTX, w, err) return } if res.Empty() { err := APIErrorNotFound("Subscription") - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -900,32 +930,32 @@ func SubVerifyPushEndpoint(w http.ResponseWriter, r *http.Request) { // check that the subscription is push enabled if sub.PushCfg.Type != subscriptions.HttpEndpointPushConfig { err := APIErrorGenericConflict("Subscription is not in http push mode") - respondErr(w, err) + respondErr(rCTX, w, err) return } // check that the endpoint isn't already verified if sub.PushCfg.Verified { err := APIErrorGenericConflict("Push endpoint is already verified") - respondErr(w, err) + respondErr(rCTX, w, err) return } // verify the push endpoint c := new(http.Client) - err = subscriptions.VerifyPushEndpoint(sub, c, refStr) + err = subscriptions.VerifyPushEndpoint(rCTX, sub, c, refStr) if err != nil { err := APIErrPushVerification(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } // activate the subscription on the push backend apsc := gorillaContext.Get(r, "apsc").(push.Client) - err = activatePushSubscription(sub, pushW, apsc, refStr) + err = activatePushSubscription(rCTX, sub, pushW, apsc, refStr) if err != nil { err := APIErrGenericInternal(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -934,6 +964,8 @@ func SubVerifyPushEndpoint(w http.ResponseWriter, r *http.Request) { // SubModAck (POST) modifies the Ack deadline of the subscription func SubModAck(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) // Init output output := []byte("") @@ -952,7 +984,7 @@ func SubModAck(w http.ResponseWriter, r *http.Request) { body, err := ioutil.ReadAll(r.Body) if err != nil { err := APIErrorInvalidRequestBody() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -960,8 +992,7 @@ func SubModAck(w http.ResponseWriter, r *http.Request) { postBody, err := subscriptions.GetAckDeadlineFromJSON(body) if err != nil { err := APIErrorInvalidArgument("ackDeadlineSeconds(needs value between 0 and 600)") - respondErr(w, err) - log.Error(string(body[:])) + respondErr(rCTX, w, err) return } @@ -970,20 +1001,20 @@ func SubModAck(w http.ResponseWriter, r *http.Request) { // Get project UUID First to use as reference projectUUID := gorillaContext.Get(r, "auth_project_uuid").(string) - err = subscriptions.ModAck(projectUUID, urlSub, postBody.AckDeadline, refStr) + err = subscriptions.ModAck(rCTX, projectUUID, urlSub, postBody.AckDeadline, refStr) if err != nil { if err.Error() == "wrong value" { - respondErr(w, APIErrorInvalidArgument("ackDeadlineSeconds(needs value between 0 and 600)")) + respondErr(rCTX, w, APIErrorInvalidArgument("ackDeadlineSeconds(needs value between 0 and 600)")) return } if err.Error() == "not found" { err := APIErrorNotFound("Subscription") - respondErr(w, err) + respondErr(rCTX, w, err) return } err := APIErrGenericInternal(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -992,6 +1023,8 @@ func SubModAck(w http.ResponseWriter, r *http.Request) { // SubCreate (PUT) creates a new subscription func SubCreate(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) // Init output output := []byte("") @@ -1013,7 +1046,7 @@ func SubCreate(w http.ResponseWriter, r *http.Request) { body, err := ioutil.ReadAll(r.Body) if err != nil { err := APIErrorInvalidRequestBody() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -1021,8 +1054,7 @@ func SubCreate(w http.ResponseWriter, r *http.Request) { postBody, err := subscriptions.GetFromJSON(body) if err != nil { err := APIErrorInvalidArgument("Subscription") - respondErr(w, err) - log.Error(string(body[:])) + respondErr(rCTX, w, err) return } @@ -1030,20 +1062,20 @@ func SubCreate(w http.ResponseWriter, r *http.Request) { if err != nil { err := APIErrorInvalidName("Topic") - respondErr(w, err) + respondErr(rCTX, w, err) return } - if topics.HasTopic(projectUUID, tName, refStr) == false { + if topics.HasTopic(rCTX, projectUUID, tName, refStr) == false { err := APIErrorNotFound("Topic") - respondErr(w, err) + respondErr(rCTX, w, err) return } // Get current topic offset - tProjectUUID := projects.GetUUIDByName(tProject, refStr) + tProjectUUID := projects.GetUUIDByName(rCTX, tProject, refStr) fullTopic := tProjectUUID + "." + tName - curOff := refBrk.GetMaxOffset(fullTopic) + curOff := refBrk.GetMaxOffset(rCTX, fullTopic) pushConfig := subscriptions.PushConfig{} @@ -1057,14 +1089,14 @@ func SubCreate(w http.ResponseWriter, r *http.Request) { if !pushEnabled { err := APIErrorPushConflict() - respondErr(w, err) + respondErr(rCTX, w, err) return } - _, err = auth.GetPushWorker(pwToken, refStr) + _, err = auth.GetPushWorker(rCTX, pwToken, refStr) if err != nil { err := APIErrInternalPush() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -1076,7 +1108,7 @@ func SubCreate(w http.ResponseWriter, r *http.Request) { // Check if push endpoint is not a valid https:// endpoint if !(validation.IsValidHTTPS(pushConfig.Pend)) { err := APIErrorInvalidData("Push endpoint should be addressed by a valid https url") - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -1093,7 +1125,7 @@ func SubCreate(w http.ResponseWriter, r *http.Request) { if !subscriptions.IsAuthorizationHeaderTypeSupported(pushConfig.AuthorizationHeader.Type) { err := APIErrorInvalidData(subscriptions.UnSupportedAuthorizationHeader) - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -1101,9 +1133,16 @@ func SubCreate(w http.ResponseWriter, r *http.Request) { case subscriptions.AutoGenerationAuthorizationHeader: pushConfig.AuthorizationHeader.Value, err = auth.GenToken() if err != nil { - log.Errorf("Could not generate authorization header for subscription %v, %v", urlVars["subscription"], err.Error()) + log.WithFields( + log.Fields{ + "trace_id": rCTX.Value("trace_id"), + "type": "service_log", + "subscription": urlVars["subscription"], + "error": err.Error(), + }, + ).Error("Could not generate auth header for subscription") err := APIErrGenericInternal("Could not generate authorization header") - respondErr(w, err) + respondErr(rCTX, w, err) return } case subscriptions.DisabledAuthorizationHeader: @@ -1112,16 +1151,23 @@ func SubCreate(w http.ResponseWriter, r *http.Request) { pushConfig.VerificationHash, err = auth.GenToken() if err != nil { - log.Errorf("Could not generate verification hash for subscription %v, %v", urlVars["subscription"], err.Error()) + log.WithFields( + log.Fields{ + "trace_id": rCTX.Value("trace_id"), + "type": "service_log", + "subscription": urlVars["subscription"], + "error": err.Error(), + }, + ).Error("Could not generate verification hash for subscription") err := APIErrGenericInternal("Could not generate verification hash") - respondErr(w, err) + respondErr(rCTX, w, err) return } pushConfig.Verified = false } else if pushConfig.Type == subscriptions.MattermostPushConfig { if postBody.PushCfg.MattermostUrl == "" { err := APIErrorInvalidData("Field mattermostUrl cannot be empty") - respondErr(w, err) + respondErr(rCTX, w, err) return } pushConfig.MattermostUrl = postBody.PushCfg.MattermostUrl @@ -1130,7 +1176,7 @@ func SubCreate(w http.ResponseWriter, r *http.Request) { pushConfig.Verified = true } else { err := APIErrorInvalidData(subscriptions.UnsupportedPushConfig) - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -1149,7 +1195,7 @@ func SubCreate(w http.ResponseWriter, r *http.Request) { if !subscriptions.IsRetryPolicySupported(pushConfig.RetPol.PolicyType) { err := APIErrorInvalidData(subscriptions.UnSupportedRetryPolicyError) - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -1158,17 +1204,17 @@ func SubCreate(w http.ResponseWriter, r *http.Request) { created := time.Now().UTC() // Get Result Object - res, err := subscriptions.Create(projectUUID, urlVars["subscription"], tName, curOff, + res, err := subscriptions.Create(rCTX, projectUUID, urlVars["subscription"], tName, curOff, postBody.Ack, pushConfig, created, refStr) if err != nil { if err.Error() == "exists" { err := APIErrorConflict("Subscription") - respondErr(w, err) + respondErr(rCTX, w, err) return } err := APIErrGenericInternal(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -1178,22 +1224,22 @@ func SubCreate(w http.ResponseWriter, r *http.Request) { pwToken := gorillaContext.Get(r, "push_worker_token").(string) - pushWorker, err := auth.GetPushWorker(pwToken, refStr) + pushWorker, err := auth.GetPushWorker(rCTX, pwToken, refStr) if err != nil { err := APIErrInternalPush() - respondErr(w, err) + respondErr(rCTX, w, err) return } apsc := gorillaContext.Get(r, "apsc").(push.Client) - activatePushSubscription(res, pushWorker, apsc, refStr) + activatePushSubscription(rCTX, res, pushWorker, apsc, refStr) } // Output result to JSON resJSON, err := res.ExportJSON() if err != nil { err := APIErrExportJSON() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -1205,6 +1251,8 @@ func SubCreate(w http.ResponseWriter, r *http.Request) { // SubACL (GET) one sub's authorized users func SubACL(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) // Init output output := []byte("") @@ -1223,12 +1271,12 @@ func SubACL(w http.ResponseWriter, r *http.Request) { // Get project UUID First to use as reference projectUUID := gorillaContext.Get(r, "auth_project_uuid").(string) - res, err := auth.GetACL(projectUUID, "subscriptions", urlSub, refStr) + res, err := auth.GetACL(rCTX, projectUUID, "subscriptions", urlSub, refStr) // If not found if err != nil { err := APIErrorNotFound("Subscription") - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -1236,7 +1284,7 @@ func SubACL(w http.ResponseWriter, r *http.Request) { resJSON, err := res.ExportJSON() if err != nil { err := APIErrExportJSON() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -1245,8 +1293,10 @@ func SubACL(w http.ResponseWriter, r *http.Request) { respondOK(w, output) } -//SubListAll (GET) all subscriptions +// SubListAll (GET) all subscriptions func SubListAll(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) var err error var strPageSize string @@ -1279,17 +1329,24 @@ func SubListAll(w http.ResponseWriter, r *http.Request) { if strPageSize != "" { if pageSize, err = strconv.Atoi(strPageSize); err != nil { - log.Errorf("Pagesize %v produced an error while being converted to int: %v", strPageSize, err.Error()) + log.WithFields( + log.Fields{ + "trace_id": rCTX.Value("trace_id"), + "type": "request_log", + "page_size": pageSize, + "error": err.Error(), + }, + ).Error("error while converting page size to int") err := APIErrorInvalidData("Invalid page size") - respondErr(w, err) + respondErr(rCTX, w, err) return } } - res, err = subscriptions.Find(projectUUID, userUUID, "", pageToken, int64(pageSize), refStr) + res, err = subscriptions.Find(rCTX, projectUUID, userUUID, "", pageToken, int64(pageSize), refStr) if err != nil { err := APIErrorInvalidData("Invalid page token") - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -1297,7 +1354,7 @@ func SubListAll(w http.ResponseWriter, r *http.Request) { resJSON, err := res.ExportJSON() if err != nil { err := APIErrExportJSON() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -1308,6 +1365,8 @@ func SubListAll(w http.ResponseWriter, r *http.Request) { // SubPull (POST) consumes messages from the underlying topic func SubPull(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) // Init output output := []byte("") @@ -1333,16 +1392,16 @@ func SubPull(w http.ResponseWriter, r *http.Request) { projectUUID := gorillaContext.Get(r, "auth_project_uuid").(string) // Get the subscription - results, err := subscriptions.Find(projectUUID, "", urlSub, "", 0, refStr) + results, err := subscriptions.Find(rCTX, projectUUID, "", urlSub, "", 0, refStr) if err != nil { err := APIErrGenericBackend() - respondErr(w, err) + respondErr(rCTX, w, err) return } if results.Empty() { err := APIErrorNotFound("Subscription") - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -1354,14 +1413,14 @@ func SubPull(w http.ResponseWriter, r *http.Request) { // if the subscription is push enabled but push enabled is false, don't allow push worker user to consume if targetSub.PushCfg != (subscriptions.PushConfig{}) && !pushEnabled && auth.IsPushWorker(refRoles) { err := APIErrorPushConflict() - respondErr(w, err) + respondErr(rCTX, w, err) return } // if the subscription is push enabled, allow only push worker and service_admin users to pull from it if targetSub.PushCfg != (subscriptions.PushConfig{}) && !auth.IsPushWorker(refRoles) && !auth.IsServiceAdmin(refRoles) { err := APIErrorForbidden() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -1369,17 +1428,17 @@ func SubPull(w http.ResponseWriter, r *http.Request) { // - if enabled in config // - if user has only consumer role if refAuthResource && auth.IsConsumer(refRoles) { - if auth.PerResource(projectUUID, "subscriptions", targetSub.Name, refUserUUID, refStr) == false { + if auth.PerResource(rCTX, projectUUID, "subscriptions", targetSub.Name, refUserUUID, refStr) == false { err := APIErrorForbidden() - respondErr(w, err) + respondErr(rCTX, w, err) return } } // check if the subscription's topic exists - if !topics.HasTopic(projectUUID, targetSub.Topic, refStr) { + if !topics.HasTopic(rCTX, projectUUID, targetSub.Topic, refStr) { err := APIErrorPullNoTopic() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -1387,7 +1446,7 @@ func SubPull(w http.ResponseWriter, r *http.Request) { body, err := ioutil.ReadAll(r.Body) if err != nil { err := APIErrorInvalidRequestBody() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -1395,7 +1454,7 @@ func SubPull(w http.ResponseWriter, r *http.Request) { pullInfo, err := subscriptions.GetPullOptionsJSON(body) if err != nil { err := APIErrorInvalidArgument("Pull Parameters") - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -1417,23 +1476,43 @@ func SubPull(w http.ResponseWriter, r *http.Request) { if err != nil { // If tracked offset is off if err == brokers.ErrOffsetOff { - log.Debug("Will increment now...") + log.WithFields( + log.Fields{ + "trace_id": rCTX.Value("trace_id"), + "type": "sservice_log", + "subscription": targetSub.FullName, + }, + ).Debug("Will increment now . . .") // Increment tracked offset to current min offset - targetSub.Offset = refBrk.GetMinOffset(fullTopic) - refStr.UpdateSubOffset(projectUUID, targetSub.Name, targetSub.Offset) + targetSub.Offset = refBrk.GetMinOffset(rCTX, fullTopic) + refStr.UpdateSubOffset(rCTX, projectUUID, targetSub.Name, targetSub.Offset) // Try again to consume msgs, err = refBrk.Consume(r.Context(), fullTopic, targetSub.Offset, retImm, int64(max)) // If still error respond and return if err != nil { - log.Errorf("Couldn't consume messages for subscription %v, %v", targetSub.FullName, err.Error()) + log.WithFields( + log.Fields{ + "trace_id": rCTX.Value("trace_id"), + "type": "service_log", + "error": err.Error(), + "subscription": targetSub.FullName, + }, + ).Error("Couldn't consume messages for subscription") err := APIErrGenericBackend() - respondErr(w, err) + respondErr(rCTX, w, err) return } } else { - log.Errorf("Couldn't consume messages for subscription %v, %v", targetSub.FullName, err.Error()) + log.WithFields( + log.Fields{ + "trace_id": rCTX.Value("trace_id"), + "type": "service_log", + "error": err.Error(), + "subscription": targetSub.FullName, + }, + ).Error("Couldn't consume messages for subscription") err := APIErrGenericBackend() - respondErr(w, err) + respondErr(rCTX, w, err) return } } @@ -1452,7 +1531,7 @@ func SubPull(w http.ResponseWriter, r *http.Request) { curMsg, err := messages.LoadMsgJSON([]byte(msg)) if err != nil { err := APIErrGenericInternal("Message retrieved from broker network has invalid JSON Structure") - respondErr(w, err) + respondErr(rCTX, w, err) return } // calc the message id = message's kafka offset (read offst + msg position) @@ -1469,9 +1548,9 @@ func SubPull(w http.ResponseWriter, r *http.Request) { consumeTime := time.Now().UTC() // increment subscription number of message metric - refStr.IncrementSubMsgNum(projectUUID, urlSub, msgCount) - refStr.IncrementSubBytes(projectUUID, urlSub, recList.TotalSize()) - refStr.UpdateSubLatestConsume(projectUUID, targetSub.Name, consumeTime) + refStr.IncrementSubMsgNum(rCTX, projectUUID, urlSub, msgCount) + refStr.IncrementSubBytes(rCTX, projectUUID, urlSub, recList.TotalSize()) + refStr.UpdateSubLatestConsume(rCTX, projectUUID, targetSub.Name, consumeTime) // count the rate of consumed messages per sec between the last two consume events var dt float64 = 1 @@ -1481,13 +1560,13 @@ func SubPull(w http.ResponseWriter, r *http.Request) { dt = consumeTime.Sub(targetSub.LatestConsume).Seconds() } - refStr.UpdateSubConsumeRate(projectUUID, targetSub.Name, float64(msgCount)/dt) + refStr.UpdateSubConsumeRate(rCTX, projectUUID, targetSub.Name, float64(msgCount)/dt) resJSON, err := recList.ExportJSON() if err != nil { err := APIErrExportJSON() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -1495,26 +1574,26 @@ func SubPull(w http.ResponseWriter, r *http.Request) { zSec := "2006-01-02T15:04:05Z" t := time.Now().UTC() ts := t.Format(zSec) - refStr.UpdateSubPull(targetSub.ProjectUUID, targetSub.Name, int64(len(recList.RecMsgs))+targetSub.Offset, ts) + refStr.UpdateSubPull(rCTX, targetSub.ProjectUUID, targetSub.Name, int64(len(recList.RecMsgs))+targetSub.Offset, ts) output = []byte(resJSON) respondOK(w, output) } -func activatePushSubscription(sub subscriptions.Subscription, pushW auth.User, +func activatePushSubscription(rCTX context.Context, sub subscriptions.Subscription, pushW auth.User, apsc push.Client, refStr stores.Store) error { // activate the subscription on the push server apsc.ActivateSubscription(context.TODO(), sub).Result(false) // modify the sub's acl with the push worker's uuid - err := auth.AppendToACL(sub.ProjectUUID, "subscriptions", sub.Name, []string{pushW.Name}, refStr) + err := auth.AppendToACL(rCTX, sub.ProjectUUID, "subscriptions", sub.Name, []string{pushW.Name}, refStr) if err != nil { return err } // link the sub's project with the push worker - err = auth.AppendToUserProjects(pushW.UUID, sub.ProjectUUID, refStr) + err = auth.AppendToUserProjects(rCTX, pushW.UUID, sub.ProjectUUID, refStr) if err != nil { return err } diff --git a/handlers/subscriptions_test.go b/handlers/subscriptions_test.go index 1fd21014..01f799e5 100644 --- a/handlers/subscriptions_test.go +++ b/handlers/subscriptions_test.go @@ -2,6 +2,7 @@ package handlers import ( "bytes" + "context" "fmt" "github.com/ARGOeu/argo-messaging/brokers" "github.com/ARGOeu/argo-messaging/config" @@ -23,9 +24,11 @@ import ( type SubscriptionsHandlersTestSuite struct { cfgStr string suite.Suite + ctx context.Context } func (suite *SubscriptionsHandlersTestSuite) SetupTest() { + suite.ctx = context.Background() suite.cfgStr = `{ "bind_ip":"", "port":8080, @@ -191,7 +194,7 @@ func (suite *SubscriptionsHandlersTestSuite) TestSubModPushConfigToActive() { w := httptest.NewRecorder() router.HandleFunc("/v1/projects/{project}/subscriptions/{subscription}:modifyPushConfig", WrapMockAuthConfig(SubModPush, cfgKafka, &brk, str, &mgr, pc)) router.ServeHTTP(w, req) - sub, _ := str.QueryOneSub("argo_uuid", "sub1") + sub, _ := str.QueryOneSub(suite.ctx, "argo_uuid", "sub1") suite.Equal(200, w.Code) suite.Equal("", w.Body.String()) suite.Equal("https://www.example.com", sub.PushEndpoint) @@ -228,7 +231,7 @@ func (suite *SubscriptionsHandlersTestSuite) TestSubModPushConfigToInactive() { w := httptest.NewRecorder() router.HandleFunc("/v1/projects/{project}/subscriptions/{subscription}:modifyPushConfig", WrapMockAuthConfig(SubModPush, cfgKafka, &brk, str, &mgr, pc)) router.ServeHTTP(w, req) - sub, _ := str.QueryOneSub("argo_uuid", "sub4") + sub, _ := str.QueryOneSub(suite.ctx, "argo_uuid", "sub4") suite.Equal(200, w.Code) suite.Equal("", w.Body.String()) suite.Equal("", sub.PushEndpoint) @@ -237,7 +240,7 @@ func (suite *SubscriptionsHandlersTestSuite) TestSubModPushConfigToInactive() { suite.Equal("", sub.VerificationHash) suite.False(sub.Verified) // check to see that the push worker user has been removed from the subscription's acl - a1, _ := str.QueryACL("argo_uuid", "subscriptions", "sub4") + a1, _ := str.QueryACL(suite.ctx, "argo_uuid", "subscriptions", "sub4") suite.Equal([]string{"uuid2", "uuid4"}, a1.ACL) } @@ -267,7 +270,7 @@ func (suite *SubscriptionsHandlersTestSuite) TestSubModPushConfigToInactivePushD w := httptest.NewRecorder() router.HandleFunc("/v1/projects/{project}/subscriptions/{subscription}:modifyPushConfig", WrapMockAuthConfig(SubModPush, cfgKafka, &brk, str, &mgr, pc)) router.ServeHTTP(w, req) - sub, _ := str.QueryOneSub("argo_uuid", "sub4") + sub, _ := str.QueryOneSub(suite.ctx, "argo_uuid", "sub4") suite.Equal(200, w.Code) suite.Equal("", w.Body.String()) suite.Equal("", sub.PushEndpoint) @@ -302,7 +305,7 @@ func (suite *SubscriptionsHandlersTestSuite) TestSubModPushConfigToInactiveMissi w := httptest.NewRecorder() router.HandleFunc("/v1/projects/{project}/subscriptions/{subscription}:modifyPushConfig", WrapMockAuthConfig(SubModPush, cfgKafka, &brk, str, &mgr, pc)) router.ServeHTTP(w, req) - sub, _ := str.QueryOneSub("argo_uuid", "sub4") + sub, _ := str.QueryOneSub(suite.ctx, "argo_uuid", "sub4") suite.Equal(200, w.Code) suite.Equal("", w.Body.String()) suite.Equal("", sub.PushEndpoint) @@ -348,10 +351,10 @@ func (suite *SubscriptionsHandlersTestSuite) TestSubModPushConfigUpdate() { mgr := oldPush.Manager{} pc := new(push.MockClient) w := httptest.NewRecorder() - subBeforeUpdate, _ := str.QueryOneSub("argo_uuid", "sub4") + subBeforeUpdate, _ := str.QueryOneSub(suite.ctx, "argo_uuid", "sub4") router.HandleFunc("/v1/projects/{project}/subscriptions/{subscription}:modifyPushConfig", WrapMockAuthConfig(SubModPush, cfgKafka, &brk, str, &mgr, pc)) router.ServeHTTP(w, req) - sub, _ := str.QueryOneSub("argo_uuid", "sub4") + sub, _ := str.QueryOneSub(suite.ctx, "argo_uuid", "sub4") suite.Equal(200, w.Code) suite.Equal("", w.Body.String()) suite.Equal("https://www.example2.com", sub.PushEndpoint) @@ -405,7 +408,7 @@ func (suite *SubscriptionsHandlersTestSuite) TestSubModPushConfigUpdateMattermos router.HandleFunc("/v1/projects/{project}/subscriptions/{subscription}:modifyPushConfig", WrapMockAuthConfig(SubModPush, cfgKafka, &brk, str, &mgr, pc)) router.ServeHTTP(w, req) - sub, _ := str.QueryOneSub("argo_uuid", "sub4") + sub, _ := str.QueryOneSub(suite.ctx, "argo_uuid", "sub4") suite.Equal(200, w.Code) suite.Equal("", w.Body.String()) suite.Equal("", sub.PushEndpoint) @@ -593,7 +596,7 @@ func (suite *SubscriptionsHandlersTestSuite) TestSubModPushConfigUpdateAuthzDisa w := httptest.NewRecorder() router.HandleFunc("/v1/projects/{project}/subscriptions/{subscription}:modifyPushConfig", WrapMockAuthConfig(SubModPush, cfgKafka, &brk, str, &mgr, pc)) router.ServeHTTP(w, req) - sub, _ := str.QueryOneSub("argo_uuid", "sub4") + sub, _ := str.QueryOneSub(suite.ctx, "argo_uuid", "sub4") suite.Equal(200, w.Code) suite.Equal("", w.Body.String()) suite.Equal("https://www.example2.com", sub.PushEndpoint) @@ -648,7 +651,7 @@ func (suite *SubscriptionsHandlersTestSuite) TestVerifyPushEndpoint() { suite.Equal(200, w.Code) suite.Equal("", w.Body.String()) // check to see that the push worker user has been added to the subscription's acl - a1, _ := str.QueryACL("argo_uuid", "subscriptions", "push-sub-v1") + a1, _ := str.QueryACL(suite.ctx, "argo_uuid", "subscriptions", "push-sub-v1") suite.Equal([]string{"uuid7"}, a1.ACL) } @@ -701,7 +704,7 @@ func (suite *SubscriptionsHandlersTestSuite) TestVerifyPushEndpointHashMisMatch( suite.Equal(401, w.Code) suite.Equal(expResp, w.Body.String()) // check to see that the push worker user has NOT been added to the subscription's acl - a1, _ := str.QueryACL("argo_uuid", "subscriptions", "push-sub-v1") + a1, _ := str.QueryACL(suite.ctx, "argo_uuid", "subscriptions", "push-sub-v1") suite.Equal(0, len(a1.ACL)) } @@ -754,7 +757,7 @@ func (suite *SubscriptionsHandlersTestSuite) TestVerifyPushEndpointUnknownRespon suite.Equal(401, w.Code) suite.Equal(expResp, w.Body.String()) // check to see that the push worker user has NOT been added to the subscription's acl - a1, _ := str.QueryACL("argo_uuid", "subscriptions", "push-sub-v1") + a1, _ := str.QueryACL(suite.ctx, "argo_uuid", "subscriptions", "push-sub-v1") suite.Equal(0, len(a1.ACL)) } @@ -801,7 +804,7 @@ func (suite *SubscriptionsHandlersTestSuite) TestVerifyPushEndpointPushServerErr suite.Equal(200, w.Code) suite.Equal("", w.Body.String()) // check to see that the push worker user has been added to the subscription's acl - a1, _ := str.QueryACL("argo_uuid", "subscriptions", "errorSub") + a1, _ := str.QueryACL(suite.ctx, "argo_uuid", "subscriptions", "errorSub") suite.Equal([]string{"uuid7"}, a1.ACL) } @@ -938,7 +941,7 @@ func (suite *SubscriptionsHandlersTestSuite) TestSubCreatePushConfig() { w := httptest.NewRecorder() router.HandleFunc("/v1/projects/{project}/subscriptions/{subscription}", WrapMockAuthConfig(SubCreate, cfgKafka, &brk, str, &mgr, pc)) router.ServeHTTP(w, req) - sub, _ := str.QueryOneSub("argo_uuid", "subNew") + sub, _ := str.QueryOneSub(suite.ctx, "argo_uuid", "subNew") expResp = strings.Replace(expResp, "{{VHASH}}", sub.VerificationHash, 1) expResp = strings.Replace(expResp, "{{AUTHZV}}", sub.AuthorizationHeader, 1) expResp = strings.Replace(expResp, "{{CON}}", sub.CreatedOn.Format("2006-01-02T15:04:05Z"), 1) @@ -1000,7 +1003,7 @@ func (suite *SubscriptionsHandlersTestSuite) TestSubCreatePushConfigMattermost() w := httptest.NewRecorder() router.HandleFunc("/v1/projects/{project}/subscriptions/{subscription}", WrapMockAuthConfig(SubCreate, cfgKafka, &brk, str, &mgr, pc)) router.ServeHTTP(w, req) - sub, _ := str.QueryOneSub("argo_uuid", "subNew") + sub, _ := str.QueryOneSub(suite.ctx, "argo_uuid", "subNew") expResp = strings.Replace(expResp, "{{CON}}", sub.CreatedOn.Format("2006-01-02T15:04:05Z"), 1) suite.Equal(200, w.Code) suite.Equal(expResp, w.Body.String()) @@ -1146,7 +1149,7 @@ func (suite *SubscriptionsHandlersTestSuite) TestSubCreatePushConfigSlowStart() w := httptest.NewRecorder() router.HandleFunc("/v1/projects/{project}/subscriptions/{subscription}", WrapMockAuthConfig(SubCreate, cfgKafka, &brk, str, &mgr, pc)) router.ServeHTTP(w, req) - sub, _ := str.QueryOneSub("argo_uuid", "subNew") + sub, _ := str.QueryOneSub(suite.ctx, "argo_uuid", "subNew") expResp = strings.Replace(expResp, "{{VHASH}}", sub.VerificationHash, 1) expResp = strings.Replace(expResp, "{{CON}}", sub.CreatedOn.Format("2006-01-02T15:04:05Z"), 1) suite.Equal(0, sub.RetPeriod) @@ -1190,7 +1193,7 @@ func (suite *SubscriptionsHandlersTestSuite) TestSubCreatePushConfigMissingPushW router.ServeHTTP(w, req) // subscription should not have been inserted to the store if it has push configuration // but we can't retrieve the push worker - _, errSub := str.QueryOneSub("argo_uuid", "subNew") + _, errSub := str.QueryOneSub(suite.ctx, "argo_uuid", "subNew") suite.Equal(500, w.Code) suite.Equal(expResp, w.Body.String()) suite.Equal("empty", errSub.Error()) @@ -1232,7 +1235,7 @@ func (suite *SubscriptionsHandlersTestSuite) TestSubCreatePushConfigPushDisabled router.ServeHTTP(w, req) // subscription should not have been inserted to the store if it has push configuration // but push enables is false - _, errSub := str.QueryOneSub("argo_uuid", "subNew") + _, errSub := str.QueryOneSub(suite.ctx, "argo_uuid", "subNew") suite.Equal(409, w.Code) suite.Equal(expResp, w.Body.String()) suite.Equal("empty", errSub.Error()) @@ -1355,7 +1358,7 @@ func (suite *SubscriptionsHandlersTestSuite) TestSubCreate() { w := httptest.NewRecorder() router.HandleFunc("/v1/projects/{project}/subscriptions/{subscription}", WrapMockAuthConfig(SubCreate, cfgKafka, &brk, str, &mgr, nil)) router.ServeHTTP(w, req) - sub, _ := str.QueryOneSub("argo_uuid", "subNew") + sub, _ := str.QueryOneSub(suite.ctx, "argo_uuid", "subNew") fmt.Println(sub) expResp = strings.Replace(expResp, "{{CON}}", sub.CreatedOn.Format("2006-01-02T15:04:05Z"), 1) suite.Equal(200, w.Code) @@ -2358,7 +2361,7 @@ func (suite *SubscriptionsHandlersTestSuite) TestSubPullOne() { router.ServeHTTP(w, req) suite.Equal(200, w.Code) suite.Equal(expJSON, w.Body.String()) - spc, _, _, _ := str.QuerySubs("argo_uuid", "", "sub1", "", 0) + spc, _, _, _ := str.QuerySubs(suite.ctx, "argo_uuid", "", "sub1", "", 0) suite.True(tn.Before(spc[0].LatestConsume)) suite.NotEqual(spc[0].ConsumeRate, 10) @@ -2562,7 +2565,7 @@ func (suite *SubscriptionsHandlersTestSuite) TestSubModAck() { suite.Equal(200, w.Code) suite.Equal(expJSON1, w.Body.String()) - subRes, err := str.QueryOneSub("argo_uuid", "sub1") + subRes, err := str.QueryOneSub(suite.ctx, "argo_uuid", "sub1") suite.Equal(33, subRes.Ack) req2, err := http.NewRequest("POST", url, bytes.NewBuffer([]byte(postJSON2))) @@ -2865,7 +2868,7 @@ func (suite *SubscriptionsHandlersTestSuite) TestValidationInSubs() { req, err := http.NewRequest("GET", url, bytes.NewBuffer([]byte(""))) router := mux.NewRouter().StrictSlash(true) mgr := oldPush.Manager{} - router.HandleFunc("/v1/projects/{project}/subscriptions/{subscription}", WrapValidate(WrapMockAuthConfig(SubListOne, cfgKafka, &brk, str, &mgr, nil))) + router.HandleFunc("/v1/projects/{project}/subscriptions/{subscription}", WrapMockAuthConfig(WrapValidate(SubListOne), cfgKafka, &brk, str, &mgr, nil)) if err != nil { log.Fatal(err) diff --git a/handlers/topics.go b/handlers/topics.go index 81e56ac6..ce50a9cb 100644 --- a/handlers/topics.go +++ b/handlers/topics.go @@ -1,6 +1,7 @@ package handlers import ( + "context" "encoding/json" "fmt" "github.com/ARGOeu/argo-messaging/auth" @@ -21,6 +22,8 @@ import ( // TopicDelete (DEL) deletes an existing topic func TopicDelete(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) // Init output output := []byte("") @@ -40,22 +43,29 @@ func TopicDelete(w http.ResponseWriter, r *http.Request) { // Get Result Object - err := topics.RemoveTopic(projectUUID, urlVars["topic"], refStr) + err := topics.RemoveTopic(rCTX, projectUUID, urlVars["topic"], refStr) if err != nil { if err.Error() == "not found" { err := APIErrorNotFound("Topic") - respondErr(w, err) + respondErr(rCTX, w, err) return } err := APIErrGenericInternal(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } fullTopic := projectUUID + "." + urlVars["topic"] - err = refBrk.DeleteTopic(fullTopic) + err = refBrk.DeleteTopic(rCTX, fullTopic) if err != nil { - log.Errorf("Couldn't delete topic %v from broker, %v", fullTopic, err.Error()) + log.WithFields( + log.Fields{ + "trace_id": rCTX.Value("trace_id"), + "type": "service_log", + "topic": fullTopic, + "error": err.Error(), + }, + ).Error("Couldn't delete topic from broker") } // Write empty response if anything ok @@ -64,6 +74,8 @@ func TopicDelete(w http.ResponseWriter, r *http.Request) { // TopicModACL (PUT) modifies the ACL func TopicModACL(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) // Init output output := []byte("") @@ -82,7 +94,7 @@ func TopicModACL(w http.ResponseWriter, r *http.Request) { body, err := ioutil.ReadAll(r.Body) if err != nil { err := APIErrorInvalidRequestBody() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -90,8 +102,7 @@ func TopicModACL(w http.ResponseWriter, r *http.Request) { postBody, err := auth.GetACLFromJSON(body) if err != nil { err := APIErrorInvalidArgument("Topic ACL") - respondErr(w, err) - log.Error(string(body[:])) + respondErr(rCTX, w, err) return } @@ -101,24 +112,24 @@ func TopicModACL(w http.ResponseWriter, r *http.Request) { projectUUID := gorillaContext.Get(r, "auth_project_uuid").(string) // check if user list contain valid users for the given project - _, err = auth.AreValidUsers(projectUUID, postBody.AuthUsers, refStr) + _, err = auth.AreValidUsers(rCTX, projectUUID, postBody.AuthUsers, refStr) if err != nil { err := APIErrorRoot{Body: APIErrorBody{Code: http.StatusNotFound, Message: err.Error(), Status: "NOT_FOUND"}} - respondErr(w, err) + respondErr(rCTX, w, err) return } - err = auth.ModACL(projectUUID, "topics", urlTopic, postBody.AuthUsers, refStr) + err = auth.ModACL(rCTX, projectUUID, "topics", urlTopic, postBody.AuthUsers, refStr) if err != nil { if err.Error() == "not found" { err := APIErrorNotFound("Topic") - respondErr(w, err) + respondErr(rCTX, w, err) return } err := APIErrGenericInternal(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -128,6 +139,8 @@ func TopicModACL(w http.ResponseWriter, r *http.Request) { // TopicCreate (PUT) creates a new topic func TopicCreate(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) // Init output output := []byte("") @@ -154,7 +167,7 @@ func TopicCreate(w http.ResponseWriter, r *http.Request) { if err != nil { err := APIErrorInvalidRequestBody() - respondErr(w, err) + respondErr(rCTX, w, err) return } defer r.Body.Close() @@ -163,7 +176,7 @@ func TopicCreate(w http.ResponseWriter, r *http.Request) { err = json.Unmarshal(b, &postBody) if err != nil { err := APIErrorInvalidRequestBody() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -174,19 +187,19 @@ func TopicCreate(w http.ResponseWriter, r *http.Request) { _, schemaName, err := schemas.ExtractSchema(schemaRef) if err != nil { err := APIErrorInvalidData(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } - sl, err := schemas.Find(projectUUID, "", schemaName, refStr) + sl, err := schemas.Find(rCTX, projectUUID, "", schemaName, refStr) if err != nil { err := APIErrGenericInternal(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } if sl.Empty() { err := APIErrorNotFound("Schema") - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -198,22 +211,22 @@ func TopicCreate(w http.ResponseWriter, r *http.Request) { created := time.Now().UTC() // Get Result Object - res, err := topics.CreateTopic(projectUUID, urlVars["topic"], schemaUUID, created, refStr) + res, err := topics.CreateTopic(rCTX, projectUUID, urlVars["topic"], schemaUUID, created, refStr) if err != nil { if err.Error() == "exists" { err := APIErrorConflict("Topic") - respondErr(w, err) + respondErr(rCTX, w, err) return } err := APIErrGenericInternal(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) } // Output result to JSON resJSON, err := res.ExportJSON() if err != nil { err := APIErrExportJSON() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -224,6 +237,8 @@ func TopicCreate(w http.ResponseWriter, r *http.Request) { // TopicAttachSchema (POST) attaches an already created schema to the given topic func TopicAttachSchema(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) // Init output output := []byte("") @@ -243,18 +258,18 @@ func TopicAttachSchema(w http.ResponseWriter, r *http.Request) { postBody := map[string]string{} schemaUUID := "" - results, err := topics.Find(projectUUID, "", urlVars["topic"], "", 0, refStr) + results, err := topics.Find(rCTX, projectUUID, "", urlVars["topic"], "", 0, refStr) if err != nil { err := APIErrGenericBackend() - respondErr(w, err) + respondErr(rCTX, w, err) return } // If not found if results.Empty() { err := APIErrorNotFound("Topic") - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -265,7 +280,7 @@ func TopicAttachSchema(w http.ResponseWriter, r *http.Request) { if err != nil { err := APIErrorInvalidRequestBody() - respondErr(w, err) + respondErr(rCTX, w, err) return } defer r.Body.Close() @@ -274,7 +289,7 @@ func TopicAttachSchema(w http.ResponseWriter, r *http.Request) { err = json.Unmarshal(b, &postBody) if err != nil { err := APIErrorInvalidRequestBody() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -285,19 +300,19 @@ func TopicAttachSchema(w http.ResponseWriter, r *http.Request) { _, schemaName, err := schemas.ExtractSchema(schemaRef) if err != nil { err := APIErrorInvalidData(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } - sl, err := schemas.Find(projectUUID, "", schemaName, refStr) + sl, err := schemas.Find(rCTX, projectUUID, "", schemaName, refStr) if err != nil { err := APIErrGenericInternal(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } if sl.Empty() { err := APIErrorNotFound("Schema") - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -306,10 +321,10 @@ func TopicAttachSchema(w http.ResponseWriter, r *http.Request) { } } - err = topics.AttachSchemaToTopic(projectUUID, urlVars["topic"], schemaUUID, refStr) + err = topics.AttachSchemaToTopic(rCTX, projectUUID, urlVars["topic"], schemaUUID, refStr) if err != nil { err := APIErrGenericInternal(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) } // Write response @@ -319,6 +334,8 @@ func TopicAttachSchema(w http.ResponseWriter, r *http.Request) { // TopicDetachSchema (POST) removes the schema from the given topic func TopicDetachSchema(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) // Init output output := []byte("") @@ -335,25 +352,25 @@ func TopicDetachSchema(w http.ResponseWriter, r *http.Request) { refStr := gorillaContext.Get(r, "str").(stores.Store) projectUUID := gorillaContext.Get(r, "auth_project_uuid").(string) - results, err := topics.Find(projectUUID, "", urlVars["topic"], "", 0, refStr) + results, err := topics.Find(rCTX, projectUUID, "", urlVars["topic"], "", 0, refStr) if err != nil { err := APIErrGenericBackend() - respondErr(w, err) + respondErr(rCTX, w, err) return } // If not found if results.Empty() { err := APIErrorNotFound("Topic") - respondErr(w, err) + respondErr(rCTX, w, err) return } - err = topics.DetachSchemaFromTopic(projectUUID, urlVars["topic"], refStr) + err = topics.DetachSchemaFromTopic(rCTX, projectUUID, urlVars["topic"], refStr) if err != nil { err := APIErrGenericInternal(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) } // Write response @@ -363,6 +380,8 @@ func TopicDetachSchema(w http.ResponseWriter, r *http.Request) { // TopicListOne (GET) one topic func TopicListOne(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) // Init output output := []byte("") @@ -379,18 +398,18 @@ func TopicListOne(w http.ResponseWriter, r *http.Request) { refStr := gorillaContext.Get(r, "str").(stores.Store) projectUUID := gorillaContext.Get(r, "auth_project_uuid").(string) - results, err := topics.Find(projectUUID, "", urlVars["topic"], "", 0, refStr) + results, err := topics.Find(rCTX, projectUUID, "", urlVars["topic"], "", 0, refStr) if err != nil { err := APIErrGenericBackend() - respondErr(w, err) + respondErr(rCTX, w, err) return } // If not found if results.Empty() { err := APIErrorNotFound("Topic") - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -400,7 +419,7 @@ func TopicListOne(w http.ResponseWriter, r *http.Request) { resJSON, err := res.ExportJSON() if err != nil { err := APIErrExportJSON() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -411,6 +430,8 @@ func TopicListOne(w http.ResponseWriter, r *http.Request) { // ListSubsByTopic (GET) lists all subscriptions associated with the given topic func ListSubsByTopic(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) // Add content type header to the response contentType := "application/json" @@ -424,25 +445,25 @@ func ListSubsByTopic(w http.ResponseWriter, r *http.Request) { refStr := gorillaContext.Get(r, "str").(stores.Store) projectUUID := gorillaContext.Get(r, "auth_project_uuid").(string) - results, err := topics.Find(projectUUID, "", urlVars["topic"], "", 0, refStr) + results, err := topics.Find(rCTX, projectUUID, "", urlVars["topic"], "", 0, refStr) if err != nil { err := APIErrGenericBackend() - respondErr(w, err) + respondErr(rCTX, w, err) return } // If not found if results.Empty() { err := APIErrorNotFound("Topic") - respondErr(w, err) + respondErr(rCTX, w, err) return } - subs, err := subscriptions.FindByTopic(projectUUID, results.Topics[0].Name, refStr) + subs, err := subscriptions.FindByTopic(rCTX, projectUUID, results.Topics[0].Name, refStr) if err != nil { err := APIErrGenericBackend() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -450,7 +471,7 @@ func ListSubsByTopic(w http.ResponseWriter, r *http.Request) { resJSON, err := json.Marshal(subs) if err != nil { err := APIErrExportJSON() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -459,6 +480,8 @@ func ListSubsByTopic(w http.ResponseWriter, r *http.Request) { // TopicACL (GET) one topic's authorized users func TopicACL(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) // Init output output := []byte("") @@ -477,12 +500,12 @@ func TopicACL(w http.ResponseWriter, r *http.Request) { // Get project UUID First to use as reference projectUUID := gorillaContext.Get(r, "auth_project_uuid").(string) - res, err := auth.GetACL(projectUUID, "topics", urlTopic, refStr) + res, err := auth.GetACL(rCTX, projectUUID, "topics", urlTopic, refStr) // If not found if err != nil { err := APIErrorNotFound("Topic") - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -490,7 +513,7 @@ func TopicACL(w http.ResponseWriter, r *http.Request) { resJSON, err := res.ExportJSON() if err != nil { err := APIErrExportJSON() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -501,6 +524,8 @@ func TopicACL(w http.ResponseWriter, r *http.Request) { // TopicListAll (GET) all topics func TopicListAll(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) var err error var strPageSize string @@ -533,23 +558,30 @@ func TopicListAll(w http.ResponseWriter, r *http.Request) { if strPageSize != "" { if pageSize, err = strconv.Atoi(strPageSize); err != nil { - log.Errorf("Pagesize %v produced an error while being converted to int: %v", strPageSize, err.Error()) + log.WithFields( + log.Fields{ + "trace_id": rCTX.Value("trace_id"), + "type": "request_log", + "page_size": pageSize, + "error": err.Error(), + }, + ).Error("error while converting page size to int") err := APIErrorInvalidData("Invalid page size") - respondErr(w, err) + respondErr(rCTX, w, err) return } } - if res, err = topics.Find(projectUUID, userUUID, "", pageToken, int64(pageSize), refStr); err != nil { + if res, err = topics.Find(rCTX, projectUUID, userUUID, "", pageToken, int64(pageSize), refStr); err != nil { err := APIErrorInvalidData("Invalid page token") - respondErr(w, err) + respondErr(rCTX, w, err) return } // Output result to JSON resJSON, err := res.ExportJSON() if err != nil { err := APIErrExportJSON() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -560,6 +592,8 @@ func TopicListAll(w http.ResponseWriter, r *http.Request) { // TopicPublish (POST) publish messages to a topic func TopicPublish(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) // Init output output := []byte("") @@ -582,18 +616,18 @@ func TopicPublish(w http.ResponseWriter, r *http.Request) { // Get project UUID First to use as reference projectUUID := gorillaContext.Get(r, "auth_project_uuid").(string) - results, err := topics.Find(projectUUID, "", urlVars["topic"], "", 0, refStr) + results, err := topics.Find(rCTX, projectUUID, "", urlVars["topic"], "", 0, refStr) if err != nil { err := APIErrGenericBackend() - respondErr(w, err) + respondErr(rCTX, w, err) return } // If not found if results.Empty() { err := APIErrorNotFound("Topic") - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -605,9 +639,9 @@ func TopicPublish(w http.ResponseWriter, r *http.Request) { if refAuthResource && auth.IsPublisher(refRoles) { - if auth.PerResource(projectUUID, "topics", urlTopic, refUserUUID, refStr) == false { + if auth.PerResource(rCTX, projectUUID, "topics", urlTopic, refUserUUID, refStr) == false { err := APIErrorForbidden() - respondErr(w, err) + respondErr(rCTX, w, err) return } } @@ -616,7 +650,7 @@ func TopicPublish(w http.ResponseWriter, r *http.Request) { body, err := ioutil.ReadAll(r.Body) if err != nil { err := APIErrorInvalidRequestBody() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -624,7 +658,7 @@ func TopicPublish(w http.ResponseWriter, r *http.Request) { msgList, err := messages.LoadMsgListJSON(body) if err != nil { err := APIErrorInvalidArgument("Message") - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -636,6 +670,7 @@ func TopicPublish(w http.ResponseWriter, r *http.Request) { if err != nil { log.WithFields( log.Fields{ + "trace_id": rCTX.Value("trace_id"), "type": "service_log", "schema_name": res.Schema, "topic_name": res.Name, @@ -643,15 +678,16 @@ func TopicPublish(w http.ResponseWriter, r *http.Request) { }, ).Error("Could not extract schema name") err := APIErrGenericInternal(schemas.GenericError) - respondErr(w, err) + respondErr(rCTX, w, err) return } - sl, err := schemas.Find(projectUUID, "", schemaName, refStr) + sl, err := schemas.Find(rCTX, projectUUID, "", schemaName, refStr) if err != nil { log.WithFields( log.Fields{ + "trace_id": rCTX.Value("trace_id"), "type": "service_log", "schema_name": schemaName, "topic_name": res.Name, @@ -659,7 +695,7 @@ func TopicPublish(w http.ResponseWriter, r *http.Request) { }, ).Error("Could not retrieve schema from the store") err := APIErrGenericInternal(schemas.GenericError) - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -668,24 +704,25 @@ func TopicPublish(w http.ResponseWriter, r *http.Request) { if err != nil { if err.Error() == "500" { err := APIErrGenericInternal(schemas.GenericError) - respondErr(w, err) + respondErr(rCTX, w, err) return } else { err := APIErrorInvalidData(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } } } else { log.WithFields( log.Fields{ + "trace_id": rCTX.Value("trace_id"), "type": "service_log", "schema_name": res.Schema, "topic_name": res.Name, }, ).Error("List of schemas was empty") err := APIErrGenericInternal(schemas.GenericError) - respondErr(w, err) + respondErr(rCTX, w, err) return } } @@ -698,25 +735,25 @@ func TopicPublish(w http.ResponseWriter, r *http.Request) { // Get offset and set it as msg fullTopic := projectUUID + "." + urlTopic - msgID, rTop, _, _, err := refBrk.Publish(fullTopic, msg) + msgID, rTop, _, _, err := refBrk.Publish(rCTX, fullTopic, msg) if err != nil { if err.Error() == "kafka server: Message was too large, server rejected it to avoid allocation error." { err := APIErrTooLargeMessage("Message size too large") - respondErr(w, err) + respondErr(rCTX, w, err) return } err := APIErrGenericBackend() - respondErr(w, err) + respondErr(rCTX, w, err) return } msg.ID = msgID - // Assertions for Succesfull Publish + // Assertions for Successful Publish if rTop != fullTopic { err := APIErrGenericInternal("Broker reports wrong topic") - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -731,17 +768,17 @@ func TopicPublish(w http.ResponseWriter, r *http.Request) { msgCount := int64(len(msgList.Msgs)) // increment topic number of message metric - refStr.IncrementTopicMsgNum(projectUUID, urlTopic, msgCount) + refStr.IncrementTopicMsgNum(rCTX, projectUUID, urlTopic, msgCount) // increment daily count of topic messages year, month, day := publishTime.Date() - refStr.IncrementDailyTopicMsgCount(projectUUID, urlTopic, msgCount, time.Date(year, month, day, 0, 0, 0, 0, time.UTC)) + refStr.IncrementDailyTopicMsgCount(rCTX, projectUUID, urlTopic, msgCount, time.Date(year, month, day, 0, 0, 0, 0, time.UTC)) // increment topic total bytes published - refStr.IncrementTopicBytes(projectUUID, urlTopic, msgList.TotalSize()) + refStr.IncrementTopicBytes(rCTX, projectUUID, urlTopic, msgList.TotalSize()) // update latest publish date for the given topic - refStr.UpdateTopicLatestPublish(projectUUID, urlTopic, publishTime) + refStr.UpdateTopicLatestPublish(rCTX, projectUUID, urlTopic, publishTime) // count the rate of published messages per sec between the last two publish events var dt float64 = 1 @@ -750,13 +787,13 @@ func TopicPublish(w http.ResponseWriter, r *http.Request) { if !res.LatestPublish.IsZero() { dt = publishTime.Sub(res.LatestPublish).Seconds() } - refStr.UpdateTopicPublishRate(projectUUID, urlTopic, float64(msgCount)/dt) + refStr.UpdateTopicPublishRate(rCTX, projectUUID, urlTopic, float64(msgCount)/dt) // Export the msgIDs resJSON, err := msgIDs.ExportJSON() if err != nil { err := APIErrExportJSON() - respondErr(w, err) + respondErr(rCTX , w, err) return } diff --git a/handlers/topics_test.go b/handlers/topics_test.go index 12012f95..6ee4f385 100644 --- a/handlers/topics_test.go +++ b/handlers/topics_test.go @@ -2,6 +2,7 @@ package handlers import ( "bytes" + "context" "fmt" "github.com/ARGOeu/argo-messaging/brokers" "github.com/ARGOeu/argo-messaging/config" @@ -22,9 +23,11 @@ import ( type TopicsHandlersTestSuite struct { suite.Suite cfgStr string + ctx context.Context } func (suite *TopicsHandlersTestSuite) SetupTest() { + suite.ctx = context.Background() suite.cfgStr = `{ "bind_ip":"", "port":8080, @@ -90,7 +93,7 @@ func (suite *TopicsHandlersTestSuite) TestTopicCreate() { mgr := oldPush.Manager{} router.HandleFunc("/v1/projects/{project}/topics/{topic}", WrapMockAuthConfig(TopicCreate, cfgKafka, &brk, str, &mgr, nil)) router.ServeHTTP(w, req) - tp, _, _, _ := str.QueryTopics("argo_uuid", "", "topicNew", "", 1) + tp, _, _, _ := str.QueryTopics(suite.ctx, "argo_uuid", "", "topicNew", "", 1) expResp = strings.Replace(expResp, "{{CON}}", tp[0].CreatedOn.Format("2006-01-02T15:04:05Z"), 1) suite.Equal(200, w.Code) suite.Equal(expResp, w.Body.String()) @@ -118,7 +121,7 @@ func (suite *TopicsHandlersTestSuite) TestTopicAttachSchema() { mgr := oldPush.Manager{} router.HandleFunc("/v1/projects/{project}/topics/{topic}:attachSchema", WrapMockAuthConfig(TopicAttachSchema, cfgKafka, &brk, str, &mgr, nil)) router.ServeHTTP(w, req) - tp, _, _, _ := str.QueryTopics("argo_uuid", "", "topic1", "", 1) + tp, _, _, _ := str.QueryTopics(suite.ctx, "argo_uuid", "", "topic1", "", 1) suite.Equal(200, w.Code) suite.Equal("", w.Body.String()) suite.Equal("schema_uuid_1", tp[0].SchemaUUID) @@ -146,7 +149,7 @@ func (suite *TopicsHandlersTestSuite) TestTopicDetachSchema() { mgr := oldPush.Manager{} router.HandleFunc("/v1/projects/{project}/topics/{topic}:detachSchema", WrapMockAuthConfig(TopicDetachSchema, cfgKafka, &brk, str, &mgr, nil)) router.ServeHTTP(w, req) - tp, _, _, _ := str.QueryTopics("argo_uuid", "", "topic5", "", 1) + tp, _, _, _ := str.QueryTopics(suite.ctx, "argo_uuid", "", "topic5", "", 1) suite.Equal(200, w.Code) suite.Equal("", w.Body.String()) suite.Equal("", tp[0].SchemaUUID) @@ -881,7 +884,7 @@ func (suite *TopicsHandlersTestSuite) TestPublish() { router.ServeHTTP(w, req) suite.Equal(200, w.Code) suite.Equal(expJSON, w.Body.String()) - tpc, _, _, _ := str.QueryTopics("argo_uuid", "", "topic1", "", 0) + tpc, _, _, _ := str.QueryTopics(suite.ctx, "argo_uuid", "", "topic1", "", 0) suite.True(tn.Before(tpc[0].LatestPublish)) suite.NotEqual(tpc[0].PublishRate, 10) @@ -1090,7 +1093,7 @@ func (suite *TopicsHandlersTestSuite) TestValidationInTopics() { req, err := http.NewRequest("GET", url, bytes.NewBuffer([]byte(""))) router := mux.NewRouter().StrictSlash(true) mgr := oldPush.Manager{} - router.HandleFunc("/v1/projects/{project}/topics/{topic}", WrapValidate(WrapMockAuthConfig(TopicListOne, cfgKafka, &brk, str, &mgr, nil))) + router.HandleFunc("/v1/projects/{project}/topics/{topic}", WrapMockAuthConfig(WrapValidate(TopicListOne), cfgKafka, &brk, str, &mgr, nil)) if err != nil { log.Fatal(err) diff --git a/handlers/users.go b/handlers/users.go index 0e7fc6d3..bf1279aa 100644 --- a/handlers/users.go +++ b/handlers/users.go @@ -1,6 +1,7 @@ package handlers import ( + "context" "fmt" "github.com/ARGOeu/argo-messaging/auth" "github.com/ARGOeu/argo-messaging/config" @@ -19,6 +20,8 @@ import ( // UserProfile returns a user's profile based on the provided url parameter(key) func UserProfile(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) // Add content type header to the response contentType := "application/json" @@ -34,20 +37,20 @@ func UserProfile(w http.ResponseWriter, r *http.Request) { if token == "" { err := APIErrorUnauthorized() - respondErr(w, err) + respondErr(rCTX, w, err) return } - result, err := auth.GetUserByToken(token, refStr) + result, err := auth.GetUserByToken(rCTX, token, refStr) if err != nil { if err.Error() == "not found" { err := APIErrorUnauthorized() - respondErr(w, err) + respondErr(rCTX, w, err) return } err := APIErrQueryDatastore() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -56,7 +59,7 @@ func UserProfile(w http.ResponseWriter, r *http.Request) { if err != nil { err := APIErrExportJSON() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -67,6 +70,8 @@ func UserProfile(w http.ResponseWriter, r *http.Request) { // RefreshToken (POST) refreshes user's token func RefreshToken(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) // Init output output := []byte("") @@ -84,19 +89,19 @@ func RefreshToken(w http.ResponseWriter, r *http.Request) { refStr := gorillaContext.Get(r, "str").(stores.Store) // Get Result Object - userUUID := auth.GetUUIDByName(urlUser, refStr) + userUUID := auth.GetUUIDByName(rCTX, urlUser, refStr) token, err := auth.GenToken() // generate a new user token - res, err := auth.UpdateUserToken(userUUID, token, refStr) + res, err := auth.UpdateUserToken(rCTX, userUUID, token, refStr) if err != nil { if err.Error() == "not found" { err := APIErrorNotFound("User") - respondErr(w, err) + respondErr(rCTX, w, err) return } err := APIErrGenericInternal(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -104,7 +109,7 @@ func RefreshToken(w http.ResponseWriter, r *http.Request) { resJSON, err := res.ExportJSON() if err != nil { err := APIErrExportJSON() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -115,6 +120,8 @@ func RefreshToken(w http.ResponseWriter, r *http.Request) { // UserUpdate (PUT) updates the user information func UserUpdate(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) // Init output output := []byte("") @@ -135,7 +142,7 @@ func UserUpdate(w http.ResponseWriter, r *http.Request) { body, err := ioutil.ReadAll(r.Body) if err != nil { err := APIErrorInvalidRequestBody() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -143,14 +150,14 @@ func UserUpdate(w http.ResponseWriter, r *http.Request) { postBody, err := auth.GetUserFromJSON(body) if err != nil { err := APIErrorInvalidArgument("User") - respondErr(w, err) + respondErr(rCTX, w, err) return } // Get Result Object - userUUID := auth.GetUUIDByName(urlUser, refStr) + userUUID := auth.GetUUIDByName(rCTX, urlUser, refStr) modified := time.Now().UTC() - res, err := auth.UpdateUser(userUUID, postBody.FirstName, postBody.LastName, postBody.Organization, postBody.Description, + res, err := auth.UpdateUser(rCTX, userUUID, postBody.FirstName, postBody.LastName, postBody.Organization, postBody.Description, postBody.Name, postBody.Projects, postBody.Email, postBody.ServiceRoles, modified, true, refStr) if err != nil { @@ -159,24 +166,24 @@ func UserUpdate(w http.ResponseWriter, r *http.Request) { if err.Error() == "not found" { err := APIErrorNotFound("User") - respondErr(w, err) + respondErr(rCTX, w, err) return } if strings.HasPrefix(err.Error(), "invalid") { err := APIErrorInvalidData(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } if strings.HasPrefix(err.Error(), "duplicate") { err := APIErrorInvalidData(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } err := APIErrGenericInternal(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -184,7 +191,7 @@ func UserUpdate(w http.ResponseWriter, r *http.Request) { resJSON, err := res.ExportJSON() if err != nil { err := APIErrExportJSON() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -195,6 +202,8 @@ func UserUpdate(w http.ResponseWriter, r *http.Request) { // UserCreate (POST) creates a new user inside a project func UserCreate(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) // Init output output := []byte("") @@ -216,7 +225,7 @@ func UserCreate(w http.ResponseWriter, r *http.Request) { body, err := ioutil.ReadAll(r.Body) if err != nil { err := APIErrorInvalidRequestBody() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -224,8 +233,7 @@ func UserCreate(w http.ResponseWriter, r *http.Request) { postBody, err := auth.GetUserFromJSON(body) if err != nil { err := APIErrorInvalidArgument("User") - respondErr(w, err) - log.Error(string(body[:])) + respondErr(rCTX, w, err) return } @@ -233,30 +241,30 @@ func UserCreate(w http.ResponseWriter, r *http.Request) { token, err := auth.GenToken() // generate a new user token created := time.Now().UTC() // Get Result Object - res, err := auth.CreateUser(uuid, urlUser, postBody.FirstName, postBody.LastName, postBody.Organization, postBody.Description, + res, err := auth.CreateUser(rCTX, uuid, urlUser, postBody.FirstName, postBody.LastName, postBody.Organization, postBody.Description, postBody.Projects, token, postBody.Email, postBody.ServiceRoles, created, refUserUUID, refStr) if err != nil { if err.Error() == "exists" { err := APIErrorConflict("User") - respondErr(w, err) + respondErr(rCTX, w, err) return } if strings.HasPrefix(err.Error(), "duplicate") { err := APIErrorInvalidData(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } if strings.HasPrefix(err.Error(), "invalid") { err := APIErrorInvalidData(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } err := APIErrGenericInternal(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -264,7 +272,7 @@ func UserCreate(w http.ResponseWriter, r *http.Request) { resJSON, err := res.ExportJSON() if err != nil { err := APIErrExportJSON() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -275,6 +283,8 @@ func UserCreate(w http.ResponseWriter, r *http.Request) { // UserListByToken (GET) one user by his token func UserListByToken(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) // Init output output := []byte("") @@ -292,16 +302,16 @@ func UserListByToken(w http.ResponseWriter, r *http.Request) { refStr := gorillaContext.Get(r, "str").(stores.Store) // Get Results Object - result, err := auth.GetUserByToken(urlToken, refStr) + result, err := auth.GetUserByToken(rCTX, urlToken, refStr) if err != nil { if err.Error() == "not found" { err := APIErrorNotFound("User") - respondErr(w, err) + respondErr(rCTX, w, err) return } err := APIErrQueryDatastore() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -310,7 +320,7 @@ func UserListByToken(w http.ResponseWriter, r *http.Request) { if err != nil { err := APIErrExportJSON() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -322,6 +332,8 @@ func UserListByToken(w http.ResponseWriter, r *http.Request) { // UserListOne (GET) one user func UserListOne(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) // Init output output := []byte("") @@ -339,17 +351,17 @@ func UserListOne(w http.ResponseWriter, r *http.Request) { refStr := gorillaContext.Get(r, "str").(stores.Store) // Get Results Object - results, err := auth.FindUsers("", "", urlUser, true, refStr) + results, err := auth.FindUsers(rCTX, "", "", urlUser, true, refStr) if err != nil { if err.Error() == "not found" { err := APIErrorNotFound("User") - respondErr(w, err) + respondErr(rCTX, w, err) return } err := APIErrQueryDatastore() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -360,7 +372,7 @@ func UserListOne(w http.ResponseWriter, r *http.Request) { if err != nil { err := APIErrExportJSON() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -371,6 +383,8 @@ func UserListOne(w http.ResponseWriter, r *http.Request) { // UserListByUUID (GET) one user by uuid func UserListByUUID(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) // Init output output := []byte("") @@ -387,23 +401,23 @@ func UserListByUUID(w http.ResponseWriter, r *http.Request) { refStr := gorillaContext.Get(r, "str").(stores.Store) // Get Results Object - result, err := auth.GetUserByUUID(urlVars["uuid"], refStr) + result, err := auth.GetUserByUUID(rCTX, urlVars["uuid"], refStr) if err != nil { if err.Error() == "not found" { err := APIErrorNotFound("User") - respondErr(w, err) + respondErr(rCTX, w, err) return } if err.Error() == "multiple uuids" { err := APIErrGenericInternal("Multiple users found with the same uuid") - respondErr(w, err) + respondErr(rCTX, w, err) return } err := APIErrQueryDatastore() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -412,7 +426,7 @@ func UserListByUUID(w http.ResponseWriter, r *http.Request) { if err != nil { err := APIErrExportJSON() - respondErr(w, err) + respondErr(rCTX, w, err) } // Write response @@ -422,6 +436,8 @@ func UserListByUUID(w http.ResponseWriter, r *http.Request) { // UserListAll (GET) all users - or users belonging to a project func UserListAll(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) var err error var pageSize int @@ -453,19 +469,26 @@ func UserListAll(w http.ResponseWriter, r *http.Request) { } if projectName != "" { - projectUUID = projects.GetUUIDByName(projectName, refStr) + projectUUID = projects.GetUUIDByName(rCTX, projectName, refStr) if projectUUID == "" { err := APIErrorNotFound("ProjectUUID") - respondErr(w, err) + respondErr(rCTX, w, err) return } } if strPageSize != "" { if pageSize, err = strconv.Atoi(strPageSize); err != nil { - log.Errorf("Pagesize %v produced an error while being converted to int: %v", strPageSize, err.Error()) + log.WithFields( + log.Fields{ + "trace_id": rCTX.Value("trace_id"), + "type": "request_log", + "page_size": pageSize, + "error": err.Error(), + }, + ).Error("error while converting page size to int") err := APIErrorInvalidData("Invalid page size") - respondErr(w, err) + respondErr(rCTX, w, err) return } } @@ -475,11 +498,11 @@ func UserListAll(w http.ResponseWriter, r *http.Request) { // Get Results Object - call is always privileged because this handler is only accessible by service admins paginatedUsers, err = - auth.PaginatedFindUsers(pageToken, int64(pageSize), projectUUID, privileged, usersDetailedView, refStr) + auth.PaginatedFindUsers(rCTX, pageToken, int64(pageSize), projectUUID, privileged, usersDetailedView, refStr) if err != nil { err := APIErrorInvalidData("Invalid page token") - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -488,7 +511,7 @@ func UserListAll(w http.ResponseWriter, r *http.Request) { if err != nil { err := APIErrExportJSON() - respondErr(w, err) + respondErr(rCTX, w, err) return } @@ -499,6 +522,8 @@ func UserListAll(w http.ResponseWriter, r *http.Request) { // UserDelete (DEL) deletes an existing user func UserDelete(w http.ResponseWriter, r *http.Request) { + traceId := gorillaContext.Get(r, "trace_id").(string) + rCTX := context.WithValue(context.Background(), "trace_id", traceId) // Init output output := []byte("") @@ -514,17 +539,17 @@ func UserDelete(w http.ResponseWriter, r *http.Request) { urlVars := mux.Vars(r) urlUser := urlVars["user"] - userUUID := auth.GetUUIDByName(urlUser, refStr) + userUUID := auth.GetUUIDByName(rCTX, urlUser, refStr) - err := auth.RemoveUser(userUUID, refStr) + err := auth.RemoveUser(rCTX, userUUID, refStr) if err != nil { if err.Error() == "not found" { err := APIErrorNotFound("User") - respondErr(w, err) + respondErr(rCTX, w, err) return } err := APIErrGenericInternal(err.Error()) - respondErr(w, err) + respondErr(rCTX, w, err) return } diff --git a/metrics/metrics_test.go b/metrics/metrics_test.go index 0ec2d45a..a27bb5fe 100644 --- a/metrics/metrics_test.go +++ b/metrics/metrics_test.go @@ -1,6 +1,7 @@ package metrics import ( + "context" "io/ioutil" "strconv" "strings" @@ -17,9 +18,11 @@ import ( type MetricsTestSuite struct { suite.Suite cfgStr string + ctx context.Context } func (suite *MetricsTestSuite) SetupTest() { + suite.ctx = context.Background() suite.cfgStr = `{ "broker_host":"localhost:9092", "store_host":"localhost", @@ -115,7 +118,7 @@ func (suite *MetricsTestSuite) TestOperational() { APIcfg := config.NewAPICfg() APIcfg.LoadStrJSON(suite.cfgStr) store := stores.NewMockStore(APIcfg.StoreHost, APIcfg.StoreDB) - ml, _ := GetUsageCpuMem(store) + ml, _ := GetUsageCpuMem(suite.ctx, store) outJSON, _ := ml.ExportJSON() ts1 := ml.Metrics[0].Timeseries[0].Timestamp @@ -138,7 +141,7 @@ func (suite *MetricsTestSuite) TestGetTopics() { APIcfg := config.NewAPICfg() APIcfg.LoadStrJSON(suite.cfgStr) store := stores.NewMockStore(APIcfg.StoreHost, APIcfg.StoreDB) - n, _ := GetProjectTopics("argo_uuid", store) + n, _ := GetProjectTopics(suite.ctx, "argo_uuid", store) suite.Equal(int64(4), n) } @@ -147,7 +150,7 @@ func (suite *MetricsTestSuite) TestGetTopicsACL() { APIcfg := config.NewAPICfg() APIcfg.LoadStrJSON(suite.cfgStr) store := stores.NewMockStore(APIcfg.StoreHost, APIcfg.StoreDB) - n, _ := GetProjectTopicsACL("argo_uuid", "uuid1", store) + n, _ := GetProjectTopicsACL(suite.ctx, "argo_uuid", "uuid1", store) suite.Equal(int64(2), n) } @@ -157,7 +160,7 @@ func (suite *MetricsTestSuite) TestGetSubs() { APIcfg := config.NewAPICfg() APIcfg.LoadStrJSON(suite.cfgStr) store := stores.NewMockStore(APIcfg.StoreHost, APIcfg.StoreDB) - n, _ := GetProjectSubs("argo_uuid", store) + n, _ := GetProjectSubs(suite.ctx, "argo_uuid", store) suite.Equal(int64(4), n) } @@ -166,7 +169,7 @@ func (suite *MetricsTestSuite) TestGetSubsACL() { APIcfg := config.NewAPICfg() APIcfg.LoadStrJSON(suite.cfgStr) store := stores.NewMockStore(APIcfg.StoreHost, APIcfg.StoreDB) - n, _ := GetProjectSubsACL("argo_uuid", "uuid1", store) + n, _ := GetProjectSubsACL(suite.ctx, "argo_uuid", "uuid1", store) suite.Equal(int64(3), n) } @@ -176,7 +179,7 @@ func (suite *MetricsTestSuite) TestGetSubsByTopic() { APIcfg := config.NewAPICfg() APIcfg.LoadStrJSON(suite.cfgStr) store := stores.NewMockStore(APIcfg.StoreHost, APIcfg.StoreDB) - n, _ := GetProjectSubsByTopic("argo_uuid", "topic1", store) + n, _ := GetProjectSubsByTopic(suite.ctx, "argo_uuid", "topic1", store) suite.Equal(int64(1), n) } @@ -247,7 +250,7 @@ func (suite *MetricsTestSuite) TestAggrProjectUserSubTest() { APIcfg := config.NewAPICfg() APIcfg.LoadStrJSON(suite.cfgStr) store := stores.NewMockStore(APIcfg.StoreHost, APIcfg.StoreDB) - ml, _ := AggrProjectUserSubs("argo_uuid", store) + ml, _ := AggrProjectUserSubs(suite.ctx, "argo_uuid", store) ts1 := ml.Metrics[0].Timeseries[0].Timestamp ts2 := ml.Metrics[1].Timeseries[0].Timestamp @@ -331,7 +334,7 @@ func (suite *MetricsTestSuite) TestAggrProjectUserTopicsTest() { APIcfg := config.NewAPICfg() APIcfg.LoadStrJSON(suite.cfgStr) store := stores.NewMockStore(APIcfg.StoreHost, APIcfg.StoreDB) - ml, _ := AggrProjectUserTopics("argo_uuid", store) + ml, _ := AggrProjectUserTopics(suite.ctx, "argo_uuid", store) ts1 := ml.Metrics[0].Timeseries[0].Timestamp ts2 := ml.Metrics[0].Timeseries[0].Timestamp @@ -374,6 +377,7 @@ func (suite *MetricsTestSuite) TestGetProjectsMessageCount() { } tmpc, tmpcerr := GenerateVAReport( + suite.ctx, []string{"ARGO"}, time.Date(2018, 10, 1, 0, 0, 0, 0, time.UTC), time.Date(2018, 10, 4, 0, 0, 0, 0, time.UTC), @@ -416,6 +420,7 @@ func (suite *MetricsTestSuite) TestGetVAReport() { } va, tmpcerr := GetVAReport( + suite.ctx, []string{"ARGO"}, time.Date(2007, 10, 1, 0, 0, 0, 0, time.UTC), time.Date(2020, 20, 4, 0, 0, 0, 0, time.UTC), diff --git a/metrics/operational.go b/metrics/operational.go index 86702715..2de73081 100644 --- a/metrics/operational.go +++ b/metrics/operational.go @@ -1,6 +1,7 @@ package metrics import ( + "context" "os" "os/exec" "strconv" @@ -10,43 +11,77 @@ import ( log "github.com/sirupsen/logrus" ) -func GetUsageCpuMem(store stores.Store) (MetricList, error) { +func GetUsageCpuMem(ctx context.Context, store stores.Store) (MetricList, error) { pid := os.Getpid() pidstr := strconv.FormatInt(int64(pid), 10) out, err := exec.Command("ps", "-p", pidstr, "-o", "%cpu").Output() if err != nil { - log.Error(err) - + log.WithFields( + log.Fields{ + "trace_id": ctx.Value("trace_id"), + "type": "service_log", + }, + ).Error(err.Error()) } // Take cli output and split it by new line chars cpuOut := strings.Split(string(out[:len(out)]), "\n") - log.Info("CPU extracted value:", cpuOut[1]) + log.WithFields( + log.Fields{ + "trace_id": ctx.Value("trace_id"), + "type": "service_log", + }, + ).Info("CPU extracted value:", cpuOut[1]) cpuVal, err := strconv.ParseFloat(strings.TrimSpace(cpuOut[1]), 64) if err != nil { - log.Error(err) + log.WithFields( + log.Fields{ + "trace_id": ctx.Value("trace_id"), + "type": "service_log", + }, + ).Error(err.Error()) } out2, err := exec.Command("ps", "-p", pidstr, "-o", "%mem").Output() if err != nil { - log.Error(err) + log.WithFields( + log.Fields{ + "trace_id": ctx.Value("trace_id"), + "type": "service_log", + }, + ).Error(err.Error()) } // Take cli output and split it by new line chars memOut := strings.Split(string(out2[:len(out2)]), "\n") - log.Info("MEM extracted value:", memOut[1]) + log.WithFields( + log.Fields{ + "trace_id": ctx.Value("trace_id"), + "type": "service_log", + }, + ).Info("MEM extracted value:", memOut[1]) memVal, err := strconv.ParseFloat(strings.TrimSpace(memOut[1]), 64) if err != nil { - log.Error(err) + log.WithFields( + log.Fields{ + "trace_id": ctx.Value("trace_id"), + "type": "service_log", + }, + ).Error(err.Error()) } host, err := os.Hostname() if err != nil { - log.Error(err) + log.WithFields( + log.Fields{ + "trace_id": ctx.Value("trace_id"), + "type": "service_log", + }, + ).Error(err.Error()) } - store.InsertOpMetric(host, cpuVal, memVal) - result := store.GetOpMetrics() + store.InsertOpMetric(ctx, host, cpuVal, memVal) + result := store.GetOpMetrics(ctx) ml := MetricList{Metrics: []Metric{}} for _, v := range result { m := NewOpNodeCPU(v.Hostname, v.CPU, GetTimeNowZulu()) diff --git a/metrics/queries.go b/metrics/queries.go index 855b4a61..f8fc5341 100644 --- a/metrics/queries.go +++ b/metrics/queries.go @@ -1,6 +1,7 @@ package metrics import ( + "context" "fmt" amsProjects "github.com/ARGOeu/argo-messaging/projects" "github.com/ARGOeu/argo-messaging/stores" @@ -8,39 +9,39 @@ import ( "time" ) -func GetProjectTopics(projectUUID string, store stores.Store) (int64, error) { - topics, _, _, err := store.QueryTopics(projectUUID, "", "", "", 0) +func GetProjectTopics(ctx context.Context, projectUUID string, store stores.Store) (int64, error) { + topics, _, _, err := store.QueryTopics(ctx, projectUUID, "", "", "", 0) return int64(len(topics)), err } -func GetProjectSubsByTopic(projectUUID string, topic string, store stores.Store) (int64, error) { - subs, err := store.QuerySubsByTopic(projectUUID, topic) +func GetProjectSubsByTopic(ctx context.Context, projectUUID string, topic string, store stores.Store) (int64, error) { + subs, err := store.QuerySubsByTopic(ctx, projectUUID, topic) return int64(len(subs)), err } -func GetProjectTopicsACL(projectUUID string, username string, store stores.Store) (int64, error) { - topics, err := store.QueryTopicsByACL(projectUUID, username) +func GetProjectTopicsACL(ctx context.Context, projectUUID string, username string, store stores.Store) (int64, error) { + topics, err := store.QueryTopicsByACL(ctx, projectUUID, username) return int64(len(topics)), err } -func GetProjectSubs(projectUUID string, store stores.Store) (int64, error) { - subs, _, _, err := store.QuerySubs(projectUUID, "", "", "", 0) +func GetProjectSubs(ctx context.Context, projectUUID string, store stores.Store) (int64, error) { + subs, _, _, err := store.QuerySubs(ctx, projectUUID, "", "", "", 0) return int64(len(subs)), err } -func GetProjectSubsACL(projectUUID string, username string, store stores.Store) (int64, error) { - subs, err := store.QuerySubsByACL(projectUUID, username) +func GetProjectSubsACL(ctx context.Context, projectUUID string, username string, store stores.Store) (int64, error) { + subs, err := store.QuerySubsByACL(ctx, projectUUID, username) return int64(len(subs)), err } -func GetDailyTopicMsgCount(projectUUID string, topicName string, store stores.Store) ([]Timepoint, error) { +func GetDailyTopicMsgCount(ctx context.Context, projectUUID string, topicName string, store stores.Store) ([]Timepoint, error) { var err error var qDtmc []stores.QDailyTopicMsgCount timePoints := []Timepoint{} - if qDtmc, err = store.QueryDailyTopicMsgCount(projectUUID, topicName, time.Time{}); err != nil { + if qDtmc, err = store.QueryDailyTopicMsgCount(ctx, projectUUID, topicName, time.Time{}); err != nil { return timePoints, err } for _, qd := range qDtmc { @@ -50,14 +51,14 @@ func GetDailyTopicMsgCount(projectUUID string, topicName string, store stores.St return timePoints, err } -func GetDailyProjectMsgCount(projectUUID string, store stores.Store) ([]Timepoint, error) { +func GetDailyProjectMsgCount(ctx context.Context, projectUUID string, store stores.Store) ([]Timepoint, error) { var err error var qDpmc []stores.QDailyProjectMsgCount timePoints := []Timepoint{} - if qDpmc, err = store.QueryDailyProjectMsgCount(projectUUID); err != nil { + if qDpmc, err = store.QueryDailyProjectMsgCount(ctx, projectUUID); err != nil { return timePoints, err } @@ -68,18 +69,18 @@ func GetDailyProjectMsgCount(projectUUID string, store stores.Store) ([]Timepoin return timePoints, err } -func AggrProjectUserSubs(projectUUID string, store stores.Store) (MetricList, error) { - pr, err := store.QueryProjects(projectUUID, "") +func AggrProjectUserSubs(ctx context.Context, projectUUID string, store stores.Store) (MetricList, error) { + pr, err := store.QueryProjects(ctx, projectUUID, "") if err != nil { return MetricList{}, err } prName := pr[0].Name - users, err := store.QueryUsers(projectUUID, "", "") + users, err := store.QueryUsers(ctx, projectUUID, "", "") ml := MetricList{} for _, item := range users { username := item.Name userUUID := item.UUID - numSubs, _ := GetProjectSubsACL(projectUUID, userUUID, store) + numSubs, _ := GetProjectSubsACL(ctx, projectUUID, userUUID, store) if numSubs > 0 { m := NewProjectUserSubs(prName, username, numSubs, GetTimeNowZulu()) ml.Metrics = append(ml.Metrics, m) @@ -89,18 +90,18 @@ func AggrProjectUserSubs(projectUUID string, store stores.Store) (MetricList, er return ml, err } -func AggrProjectUserTopics(projectUUID string, store stores.Store) (MetricList, error) { - pr, err := store.QueryProjects(projectUUID, "") +func AggrProjectUserTopics(ctx context.Context, projectUUID string, store stores.Store) (MetricList, error) { + pr, err := store.QueryProjects(ctx, projectUUID, "") if err != nil { return MetricList{}, err } prName := pr[0].Name - users, err := store.QueryUsers(projectUUID, "", "") + users, err := store.QueryUsers(ctx, projectUUID, "", "") ml := MetricList{} for _, item := range users { username := item.Name userUUID := item.UUID - numSubs, _ := GetProjectTopicsACL(projectUUID, userUUID, store) + numSubs, _ := GetProjectTopicsACL(ctx, projectUUID, userUUID, store) if numSubs > 0 { m := NewProjectUserTopics(prName, username, numSubs, GetTimeNowZulu()) ml.Metrics = append(ml.Metrics, m) @@ -111,7 +112,7 @@ func AggrProjectUserTopics(projectUUID string, store stores.Store) (MetricList, } // GetVAReport returns a VAReport populated with the needed metrics -func GetVAReport(projects []string, startDate time.Time, endDate time.Time, str stores.Store) (VAReport, error) { +func GetVAReport(ctx context.Context, projects []string, startDate time.Time, endDate time.Time, str stores.Store) (VAReport, error) { vaReport := VAReport{} @@ -119,7 +120,7 @@ func GetVAReport(projects []string, startDate time.Time, endDate time.Time, str // if some gives 2020-15-01 we need to get all counters up to 2020-15-01T23:59:59 endDate = time.Date(endDate.Year(), endDate.Month(), endDate.Day(), 23, 59, 59, 0, endDate.Location()) - tpm, err := GenerateVAReport(projects, startDate, endDate, str) + tpm, err := GenerateVAReport(ctx, projects, startDate, endDate, str) if err != nil { return vaReport, err } @@ -130,7 +131,7 @@ func GetVAReport(projects []string, startDate time.Time, endDate time.Time, str // GenerateVAReport returns per project metrics regarding users,topics subscriptions for the given time period // It also includes various totals that derive from the each individual's project metrics. // The generated result is called a VA Report -func GenerateVAReport(projects []string, startDate time.Time, endDate time.Time, str stores.Store) (VAReport, error) { +func GenerateVAReport(ctx context.Context, projects []string, startDate time.Time, endDate time.Time, str stores.Store) (VAReport, error) { tpj := TotalProjectsMessageCount{ Projects: []ProjectMetrics{}, @@ -150,7 +151,7 @@ func GenerateVAReport(projects []string, startDate time.Time, endDate time.Time, // translate the project NAMES to their respective UUIDs projectUUIDs := make([]string, 0) for _, prj := range projects { - projectUUID := amsProjects.GetUUIDByName(prj, str) + projectUUID := amsProjects.GetUUIDByName(ctx, prj, str) if projectUUID == "" { return VAReport{}, fmt.Errorf("Project %v", prj) } @@ -158,22 +159,22 @@ func GenerateVAReport(projects []string, startDate time.Time, endDate time.Time, projectsUUIDNames[projectUUID] = prj } - qtpj, err = str.QueryTotalMessagesPerProject(projectUUIDs, startDate, endDate) + qtpj, err = str.QueryTotalMessagesPerProject(ctx, projectUUIDs, startDate, endDate) if err != nil { return VAReport{}, err } - topicsCount, err := str.TopicsCount(startDate, endDate, projectUUIDs) + topicsCount, err := str.TopicsCount(ctx, startDate, endDate, projectUUIDs) if err != nil { return VAReport{}, err } - subCount, err := str.SubscriptionsCount(startDate, endDate, projectUUIDs) + subCount, err := str.SubscriptionsCount(ctx, startDate, endDate, projectUUIDs) if err != nil { return VAReport{}, err } - userCount, err := str.UsersCount(startDate, endDate, projectUUIDs) + userCount, err := str.UsersCount(ctx, startDate, endDate, projectUUIDs) if err != nil { return VAReport{}, err } @@ -188,7 +189,7 @@ func GenerateVAReport(projects []string, startDate time.Time, endDate time.Time, // if no project names were provided we have to do the mapping between name and uuid if len(projects) == 0 { - projectName = amsProjects.GetNameByUUID(prj.ProjectUUID, str) + projectName = amsProjects.GetNameByUUID(ctx, prj.ProjectUUID, str) } else { projectName = projectsUUIDNames[prj.ProjectUUID] } @@ -227,14 +228,14 @@ func GenerateVAReport(projects []string, startDate time.Time, endDate time.Time, } // GetUserUsageReport returns a VAReport populated with the needed metrics alongside service operational metrics -func GetUserUsageReport(projects []string, startDate time.Time, endDate time.Time, str stores.Store) (UserUsageReport, error) { +func GetUserUsageReport(ctx context.Context, projects []string, startDate time.Time, endDate time.Time, str stores.Store) (UserUsageReport, error) { - vr, err := GetVAReport(projects, startDate, endDate, str) + vr, err := GetVAReport(ctx, projects, startDate, endDate, str) if err != nil { return UserUsageReport{}, err } - om, err := GetUsageCpuMem(str) + om, err := GetUsageCpuMem(ctx, str) if err != nil { return UserUsageReport{}, err } diff --git a/projects/project.go b/projects/project.go index e90f0949..0f2e75fa 100644 --- a/projects/project.go +++ b/projects/project.go @@ -1,6 +1,7 @@ package projects import ( + "context" "encoding/json" "errors" @@ -9,7 +10,7 @@ import ( "github.com/ARGOeu/argo-messaging/stores" ) -// ProjectUUID is the struct that holds ProjectUUID information +// Project is the struct that holds ProjectUUID information type Project struct { UUID string `json:"-"` Name string `json:"name,omitempty"` @@ -67,16 +68,16 @@ func NewProject(uuid string, name string, createdOn time.Time, modifiedOn time.T // Find returns a specific project or a list of all available projects in the datastore. // To return all projects use an empty project string parameter -func Find(uuid string, name string, store stores.Store) (Projects, error) { +func Find(ctx context.Context, uuid string, name string, store stores.Store) (Projects, error) { result := Projects{} // if project string empty, returns all projects - projects, err := store.QueryProjects(uuid, name) + projects, err := store.QueryProjects(ctx, uuid, name) for _, item := range projects { // Get Username from user uuid username := "" if item.CreatedBy != "" { - usr, err := store.QueryUsers("", item.CreatedBy, "") + usr, err := store.QueryUsers(ctx, "", item.CreatedBy, "") if err == nil && len(usr) > 0 { username = usr[0].Name } @@ -89,11 +90,11 @@ func Find(uuid string, name string, store stores.Store) (Projects, error) { } // GetNameByUUID queries projects by UUID and returns the project name. If not found, returns an empty string -func GetNameByUUID(uuid string, store stores.Store) string { +func GetNameByUUID(ctx context.Context, uuid string, store stores.Store) string { result := "" if uuid != "" { - projects, err := store.QueryProjects(uuid, "") + projects, err := store.QueryProjects(ctx, uuid, "") if len(projects) > 0 && err == nil { result = projects[0].Name } @@ -103,11 +104,11 @@ func GetNameByUUID(uuid string, store stores.Store) string { } // GetUUIDByName queries project by name and returns the corresponding UUID -func GetUUIDByName(name string, store stores.Store) string { +func GetUUIDByName(ctx context.Context, name string, store stores.Store) string { result := "" if name != "" { - projects, err := store.QueryProjects("", name) + projects, err := store.QueryProjects(ctx, "", name) if len(projects) > 0 && err == nil { result = projects[0].UUID } @@ -117,14 +118,14 @@ func GetUUIDByName(name string, store stores.Store) string { } // ExistsWithName returns true if a project with name exists -func ExistsWithName(name string, store stores.Store) bool { +func ExistsWithName(ctx context.Context, name string, store stores.Store) bool { if name == "" { return false } result := false - projects, err := store.QueryProjects("", name) + projects, err := store.QueryProjects(ctx, "", name) if len(projects) > 0 && err == nil { result = true } @@ -134,14 +135,14 @@ func ExistsWithName(name string, store stores.Store) bool { } // ExistsWithUUID return true if a project with uuid exists -func ExistsWithUUID(uuid string, store stores.Store) bool { +func ExistsWithUUID(ctx context.Context, uuid string, store stores.Store) bool { if uuid == "" { return false } result := false - projects, err := store.QueryProjects(uuid, "") + projects, err := store.QueryProjects(ctx, uuid, "") if len(projects) > 0 && err == nil { result = true } @@ -150,59 +151,59 @@ func ExistsWithUUID(uuid string, store stores.Store) bool { } // HasProject if store contains a project with the specific name -func HasProject(name string, store stores.Store) bool { - projects, _ := store.QueryProjects("", name) +func HasProject(ctx context.Context, name string, store stores.Store) bool { + projects, _ := store.QueryProjects(ctx, "", name) return len(projects) > 0 } // CreateProject creates a new project -func CreateProject(uuid string, name string, createdOn time.Time, createdBy string, description string, store stores.Store) (Project, error) { +func CreateProject(ctx context.Context, uuid string, name string, createdOn time.Time, createdBy string, description string, store stores.Store) (Project, error) { // check if project with the same name exists - if ExistsWithName(name, store) { + if ExistsWithName(ctx, name, store) { return Project{}, errors.New("exists") } - if err := store.InsertProject(uuid, name, createdOn, createdOn, createdBy, description); err != nil { + if err := store.InsertProject(ctx, uuid, name, createdOn, createdOn, createdBy, description); err != nil { return Project{}, errors.New("backend error") } // reflect stored object - stored, err := Find("", name, store) + stored, err := Find(ctx, "", name, store) return stored.One(), err } // UpdateProject creates a new project -func UpdateProject(uuid string, name string, description string, modifiedOn time.Time, store stores.Store) (Project, error) { +func UpdateProject(ctx context.Context, uuid string, name string, description string, modifiedOn time.Time, store stores.Store) (Project, error) { // ProjectUUID with uuid should exist to be updated // check if project with the same name exists - if ExistsWithUUID(uuid, store) == false { + if ExistsWithUUID(ctx, uuid, store) == false { return Project{}, errors.New("not found") } - if err := store.UpdateProject(uuid, name, description, modifiedOn); err != nil { + if err := store.UpdateProject(ctx, uuid, name, description, modifiedOn); err != nil { return Project{}, err } // reflect stored object - stored, err := Find(uuid, name, store) + stored, err := Find(ctx, uuid, name, store) return stored.One(), err } // RemoveProject removes project -func RemoveProject(uuid string, store stores.Store) error { +func RemoveProject(ctx context.Context, uuid string, store stores.Store) error { // ProjectUUID with uuid should exist to be updated // check if project with the same name exists - if ExistsWithUUID(uuid, store) == false { + if ExistsWithUUID(ctx, uuid, store) == false { return errors.New("not found") } // Remove project it self - if err := store.RemoveProject(uuid); err != nil { + if err := store.RemoveProject(ctx, uuid); err != nil { if err.Error() == "not found" { return err @@ -212,7 +213,7 @@ func RemoveProject(uuid string, store stores.Store) error { } // Remove topics attached to this project - if err := store.RemoveProjectTopics(uuid); err != nil { + if err := store.RemoveProjectTopics(ctx, uuid); err != nil { if err.Error() == "not found" { return err @@ -222,7 +223,7 @@ func RemoveProject(uuid string, store stores.Store) error { } // Remove subscriptions attached to this project - if err := store.RemoveProjectSubs(uuid); err != nil { + if err := store.RemoveProjectSubs(ctx, uuid); err != nil { if err.Error() == "not found" { return err @@ -231,7 +232,7 @@ func RemoveProject(uuid string, store stores.Store) error { return errors.New("backend error") } - if err := store.RemoveProjectDailyMessageCounters(uuid); err != nil { + if err := store.RemoveProjectDailyMessageCounters(ctx, uuid); err != nil { if err.Error() == "not found" { return err } diff --git a/projects/project_test.go b/projects/project_test.go index 1f150deb..3a5331ff 100644 --- a/projects/project_test.go +++ b/projects/project_test.go @@ -1,6 +1,7 @@ package projects import ( + "context" "errors" "testing" "time" @@ -11,6 +12,7 @@ import ( type ProjectsTestSuite struct { suite.Suite + ctx context.Context } func (suite *ProjectsTestSuite) TestProjects() { @@ -24,16 +26,16 @@ func (suite *ProjectsTestSuite) TestProjects() { ep3 := Projects{List: []Project{item1, item2}} ep4 := Projects{} - p1, err := Find("", "ARGO", store) + p1, err := Find(suite.ctx, "", "ARGO", store) suite.Equal(ep1, p1) suite.Equal(nil, err) - p2, err := Find("", "ARGO2", store) + p2, err := Find(suite.ctx, "", "ARGO2", store) suite.Equal(ep2, p2) suite.Equal(nil, err) - p3, err := Find("", "", store) + p3, err := Find(suite.ctx, "", "", store) suite.Equal(ep3, p3) suite.Equal(nil, err) - p4, err := Find("", "FOO", store) + p4, err := Find(suite.ctx, "", "FOO", store) suite.Equal(ep4, p4) suite.Equal(errors.New("not found"), err) @@ -41,26 +43,26 @@ func (suite *ProjectsTestSuite) TestProjects() { // Create new project itemNew := NewProject("uuid_new", "BRAND_NEW", tm, tm, "UserA", "brand new project") - reflect, err := CreateProject("uuid_new", "BRAND_NEW", tm, "uuid1", "brand new project", store) + reflect, err := CreateProject(suite.ctx, "uuid_new", "BRAND_NEW", tm, "uuid1", "brand new project", store) expNew := Projects{List: []Project{itemNew}} expAllNew := Projects{List: []Project{item1, item2, itemNew}} - pNew, err := Find("", "BRAND_NEW", store) + pNew, err := Find(suite.ctx, "", "BRAND_NEW", store) suite.Equal(expNew.List[0], reflect) suite.Equal(expNew, pNew) suite.Equal(nil, err) // Test GetNameByUUID - suite.Equal("BRAND_NEW", GetNameByUUID("uuid_new", store)) - suite.Equal("", GetNameByUUID("", store)) + suite.Equal("BRAND_NEW", GetNameByUUID(suite.ctx, "uuid_new", store)) + suite.Equal("", GetNameByUUID(suite.ctx, "", store)) // Test GetUUIDByName - suite.Equal("uuid_new", GetUUIDByName("BRAND_NEW", store)) - suite.Equal("", GetUUIDByName("", store)) + suite.Equal("uuid_new", GetUUIDByName(suite.ctx, "BRAND_NEW", store)) + suite.Equal("", GetUUIDByName(suite.ctx, "", store)) - pAllNew, err := Find("", "", store) + pAllNew, err := Find(suite.ctx, "", "", store) suite.Equal(expAllNew, pAllNew) suite.Equal(nil, err) @@ -159,28 +161,30 @@ func (suite *ProjectsTestSuite) TestProjects() { ] }` - UpdateProject("argo_uuid", "NEW_ARGO", "a new description and name for project", tm, store) - UpdateProject("argo_uuid2", "", "this project has only description changed", tm, store) - UpdateProject("uuid_new", "ONLY_NAME_CHANGED", "", tm, store) + UpdateProject(suite.ctx, "argo_uuid", "NEW_ARGO", "a new description and name for project", tm, store) + UpdateProject(suite.ctx, "argo_uuid2", "", "this project has only description changed", tm, store) + UpdateProject(suite.ctx, "uuid_new", "ONLY_NAME_CHANGED", "", tm, store) - pAllUpdated, _ := Find("", "", store) + pAllUpdated, _ := Find(suite.ctx, "", "", store) outAllUpdJSON, _ := pAllUpdated.ExportJSON() suite.Equal(expUpdJSON, outAllUpdJSON) // Test removing project - RemoveProject("argo_uuid", store) - pRemoved, err := Find("argo_uuid", "", store) + RemoveProject(suite.ctx, "argo_uuid", store) + pRemoved, err := Find(suite.ctx, "argo_uuid", "", store) suite.Equal(Projects{}, pRemoved) suite.Equal(errors.New("not found"), err) // Check to see that also projects topics and subscriptions have been removed from the store - resTop, _, _, _ := store.QueryTopics("argo_uuid", "", "", "", 0) + resTop, _, _, _ := store.QueryTopics(suite.ctx, "argo_uuid", "", "", "", 0) suite.Equal(0, len(resTop)) - resSub, _, _, _ := store.QuerySubs("argo_uuid", "", "", "", 0) + resSub, _, _, _ := store.QuerySubs(suite.ctx, "argo_uuid", "", "", "", 0) suite.Equal(0, len(resSub)) } func TestProjectsTestSuite(t *testing.T) { - suite.Run(t, new(ProjectsTestSuite)) + suite.Run(t, &ProjectsTestSuite{ + ctx: context.Background(), + }) } diff --git a/push/push.go b/push/push.go index ceadba71..9bd2b855 100644 --- a/push/push.go +++ b/push/push.go @@ -97,7 +97,7 @@ func (p *Pusher) push(brk brokers.Broker, store stores.Store) { log.Debug("pid ", p.id, "pushing") // update sub details - subs, err := subscriptions.Find(p.sub.ProjectUUID, "", p.sub.Name, "", 0, store) + subs, err := subscriptions.Find(context.Background(), p.sub.ProjectUUID, "", p.sub.Name, "", 0, store) // If subscription doesn't exist in store stop and remove it from manager if err == nil && len(subs.Subscriptions) == 0 { @@ -113,7 +113,7 @@ func (p *Pusher) push(brk brokers.Broker, store stores.Store) { // If tracked offset is off, update it to the latest min offset if err == brokers.ErrOffsetOff { // Get Current Min Offset and advanced tracked one - p.sub.Offset = brk.GetMinOffset(fullTopic) + p.sub.Offset = brk.GetMinOffset(context.Background(), fullTopic) msgs, err = brk.Consume(context.Background(), fullTopic, p.sub.Offset, true, 1) if err != nil { log.Error("Unable to consume after updating offset") @@ -132,10 +132,10 @@ func (p *Pusher) push(brk brokers.Broker, store stores.Store) { if err == nil { // Advance the offset - store.UpdateSubOffset(p.sub.ProjectUUID, p.sub.Name, 1+p.sub.Offset) + store.UpdateSubOffset(context.Background(), p.sub.ProjectUUID, p.sub.Name, 1+p.sub.Offset) // Update subscription's metrics - store.IncrementSubMsgNum(p.sub.ProjectUUID, p.sub.Name, int64(1)) - store.IncrementSubBytes(p.sub.ProjectUUID, p.sub.Name, pMsg.Msg.Size()) + store.IncrementSubMsgNum(context.Background(), p.sub.ProjectUUID, p.sub.Name, int64(1)) + store.IncrementSubBytes(context.Background(), p.sub.ProjectUUID, p.sub.Name, pMsg.Msg.Size()) log.Debug("offset updated") } } else { @@ -252,7 +252,7 @@ func (mgr *Manager) Refresh(projectUUID string, sub string) error { if p, err := mgr.Get(projectUUID + "/" + sub); err == nil { - subs, err := subscriptions.Find(projectUUID, "", sub, "", 0, mgr.store) + subs, err := subscriptions.Find(context.Background(), projectUUID, "", sub, "", 0, mgr.store) if err != nil { return errors.New("backend error") @@ -279,7 +279,7 @@ func (mgr *Manager) Add(projectUUID string, subName string) error { return errors.New("Push Manager not set") } // Check if subscription exists - subs, err := subscriptions.Find(projectUUID, "", subName, "", 0, mgr.store) + subs, err := subscriptions.Find(context.Background(), projectUUID, "", subName, "", 0, mgr.store) if err != nil { return errors.New("Backend error") @@ -341,7 +341,7 @@ func (p *Pusher) launch(brk brokers.Broker, store stores.Store) { } -//LinearActivity implements a linear retry push +// LinearActivity implements a linear retry push func LinearActivity(p *Pusher, brk brokers.Broker, store stores.Store) error { defer store.Close() diff --git a/schemas/schema.go b/schemas/schema.go index bb358562..5c91fb77 100644 --- a/schemas/schema.go +++ b/schemas/schema.go @@ -1,6 +1,7 @@ package schemas import ( + "context" "encoding/base64" "encoding/json" "errors" @@ -167,13 +168,13 @@ func FormatSchemaRef(projectName, schemaName string) string { } // Find retrieves a specific schema or all the schemas under a project -func Find(projectUUID, schemaUUID, schemaName string, str stores.Store) (SchemaList, error) { +func Find(ctx context.Context, projectUUID, schemaUUID, schemaName string, str stores.Store) (SchemaList, error) { schemaList := SchemaList{ Schemas: []Schema{}, } - qSchemas, err := str.QuerySchemas(projectUUID, schemaUUID, schemaName) + qSchemas, err := str.QuerySchemas(ctx, projectUUID, schemaUUID, schemaName) if err != nil { return schemaList, err } @@ -211,7 +212,7 @@ func Find(projectUUID, schemaUUID, schemaName string, str stores.Store) (SchemaL return SchemaList{}, errors.New("Could not load the schema") } - projectName := projects.GetNameByUUID(projectUUID, str) + projectName := projects.GetNameByUUID(ctx, projectUUID, str) _schema.FullName = FormatSchemaRef(projectName, s.Name) @@ -222,19 +223,19 @@ func Find(projectUUID, schemaUUID, schemaName string, str stores.Store) (SchemaL } // Delete wraps the store's method for removing a schema -func Delete(schemaUUID string, str stores.Store) error { - return str.DeleteSchema(schemaUUID) +func Delete(ctx context.Context, schemaUUID string, str stores.Store) error { + return str.DeleteSchema(ctx, schemaUUID) } // Update updates the provided schema , validates its content and saves it to the store -func Update(existingSchema Schema, newSchemaName, newSchemaType string, newRawSchema map[string]interface{}, str stores.Store) (Schema, error) { +func Update(ctx context.Context, existingSchema Schema, newSchemaName, newSchemaType string, newRawSchema map[string]interface{}, str stores.Store) (Schema, error) { newSchema := Schema{} if newSchemaName != "" { // if the name has changed check that is not already taken by another schema under the given project if existingSchema.Name != newSchemaName { - exists, err := ExistsWithName(existingSchema.ProjectUUID, newSchemaName, str) + exists, err := ExistsWithName(ctx, existingSchema.ProjectUUID, newSchemaName, str) if err != nil { return Schema{}, err } @@ -288,12 +289,12 @@ func Update(existingSchema Schema, newSchemaName, newSchemaType string, newRawSc } - err := str.UpdateSchema(existingSchema.UUID, newSchema.Name, newSchema.Type, rawSchemaString) + err := str.UpdateSchema(ctx, existingSchema.UUID, newSchema.Name, newSchema.Type, rawSchemaString) if err != nil { return Schema{}, err } - projectName := projects.GetNameByUUID(existingSchema.ProjectUUID, str) + projectName := projects.GetNameByUUID(ctx, existingSchema.ProjectUUID, str) existingSchema.FullName = FormatSchemaRef(projectName, existingSchema.Name) @@ -301,9 +302,9 @@ func Update(existingSchema Schema, newSchemaName, newSchemaType string, newRawSc } // Create checks the validity of the schema to be created and then saves it to the store -func Create(projectUUID, schemaUUID, name, schemaType string, rawSchema map[string]interface{}, str stores.Store) (Schema, error) { +func Create(ctx context.Context, projectUUID, schemaUUID, name, schemaType string, rawSchema map[string]interface{}, str stores.Store) (Schema, error) { - exists, err := ExistsWithName(projectUUID, name, str) + exists, err := ExistsWithName(ctx, projectUUID, name, str) if err != nil { return Schema{}, err } @@ -326,12 +327,12 @@ func Create(projectUUID, schemaUUID, name, schemaType string, rawSchema map[stri return Schema{}, err } - err = str.InsertSchema(projectUUID, schemaUUID, name, schemaType, b64SchemaString) + err = str.InsertSchema(ctx, projectUUID, schemaUUID, name, schemaType, b64SchemaString) if err != nil { return Schema{}, err } - projectName := projects.GetNameByUUID(projectUUID, str) + projectName := projects.GetNameByUUID(ctx, projectUUID, str) schema := Schema{ UUID: schemaUUID, @@ -376,8 +377,8 @@ func checkSchema(schemaType string, schemaContent map[string]interface{}) error } // ExistsWithName checks if a schema with the given name exists under the given project -func ExistsWithName(projectUUID string, schemaName string, str stores.Store) (bool, error) { - qSchemas, err := str.QuerySchemas(projectUUID, "", schemaName) +func ExistsWithName(ctx context.Context, projectUUID string, schemaName string, str stores.Store) (bool, error) { + qSchemas, err := str.QuerySchemas(ctx, projectUUID, "", schemaName) if err != nil { return false, errors.New("backend error") diff --git a/schemas/schema_test.go b/schemas/schema_test.go index cd105df0..a50f16e6 100644 --- a/schemas/schema_test.go +++ b/schemas/schema_test.go @@ -1,6 +1,7 @@ package schemas import ( + "context" "errors" "github.com/ARGOeu/argo-messaging/messages" "github.com/ARGOeu/argo-messaging/stores" @@ -10,6 +11,7 @@ import ( type SchemasTestSuite struct { suite.Suite + ctx context.Context } func (suite *SchemasTestSuite) TestExtractSchema() { @@ -176,7 +178,7 @@ func (suite *SchemasTestSuite) TestFind() { } for _, t := range testData { - s, e := Find(t.projectUUID, t.schemaUUID, t.schemaName, store) + s, e := Find(suite.ctx, t.projectUUID, t.schemaUUID, t.schemaName, store) suite.Equal(t.err, e, t.msg) suite.Equal(t.schemaList, s, t.msg) } @@ -229,7 +231,7 @@ func (suite *SchemasTestSuite) TestUpdate() { }, err: nil, queryFunc: func() interface{} { - qs, _ := store.QuerySchemas("argo_uuid", "schema_uuid_1", "new-schema-name") + qs, _ := store.QuerySchemas(suite.ctx, "argo_uuid", "schema_uuid_1", "new-schema-name") return qs[0] }, returnQuery: stores.QSchema{ @@ -328,7 +330,7 @@ func (suite *SchemasTestSuite) TestUpdate() { } for _, t := range testData { - s, e := Update(t.existingSchema, t.newName, t.newType, t.newSchema, store) + s, e := Update(suite.ctx, t.existingSchema, t.newName, t.newType, t.newSchema, store) suite.Equal(t.expectedSchema, s, t.msg) suite.Equal(t.err, e, t.msg) suite.Equal(t.returnQuery, t.queryFunc(), t.msg) @@ -474,9 +476,9 @@ func (suite *SchemasTestSuite) TestDelete() { store := stores.NewMockStore("", "") - e1 := Delete("schema_uuid_1", store) - sl, _ := Find("argo_uuid", "schema_uuid_1", "", store) - qtd, _, _, _ := store.QueryTopics("argo_uuid", "", "topic2", "", 1) + e1 := Delete(suite.ctx, "schema_uuid_1", store) + sl, _ := Find(suite.ctx, "argo_uuid", "schema_uuid_1", "", store) + qtd, _, _, _ := store.QueryTopics(suite.ctx, "argo_uuid", "", "topic2", "", 1) suite.Equal([]Schema{}, sl.Schemas) suite.Equal("", qtd[0].SchemaUUID) suite.Nil(e1) @@ -515,7 +517,7 @@ func (suite *SchemasTestSuite) TestCreate() { RawSchema: map[string]interface{}{"type": "string"}, }, queryFunc: func() interface{} { - qs, _ := store.QuerySchemas("argo_uuid", "suuid", "s1") + qs, _ := store.QuerySchemas(suite.ctx, "argo_uuid", "suuid", "s1") return qs[0] }, returnQuery: stores.QSchema{ @@ -544,7 +546,7 @@ func (suite *SchemasTestSuite) TestCreate() { } for _, t := range testData { - s, e := Create(t.projectUUID, t.uuid, t.name, t.schemaType, t.rawSchema, store) + s, e := Create(suite.ctx, t.projectUUID, t.uuid, t.name, t.schemaType, t.rawSchema, store) suite.Equal(t.err, e, t.msg) suite.Equal(t.returnedSchema, s, t.msg) suite.Equal(t.returnQuery, t.queryFunc()) @@ -576,7 +578,7 @@ func (suite *SchemasTestSuite) TestExistsWithName() { store := stores.NewMockStore("", "") for _, t := range testData { - b, _ := ExistsWithName("argo_uuid", t.schemaName, store) + b, _ := ExistsWithName(suite.ctx, "argo_uuid", t.schemaName, store) suite.Equal(t.exists, b, t.msg) } } @@ -648,5 +650,7 @@ func (suite *SchemasTestSuite) TestCheckSchema() { } func TestSchemasTestSuite(t *testing.T) { - suite.Run(t, new(SchemasTestSuite)) + suite.Run(t, &SchemasTestSuite{ + ctx: context.Background(), + }) } diff --git a/stores/mock.go b/stores/mock.go index b1b156f9..7f26bd31 100644 --- a/stores/mock.go +++ b/stores/mock.go @@ -1,6 +1,7 @@ package stores import ( + "context" "errors" "sort" "strconv" @@ -25,7 +26,7 @@ type MockStore struct { OpMetrics map[string]QopMetric } -func (mk *MockStore) TopicsCount(startDate, endDate time.Time, projectUUIDs []string) (map[string]int64, error) { +func (mk *MockStore) TopicsCount(ctx context.Context, startDate, endDate time.Time, projectUUIDs []string) (map[string]int64, error) { res := map[string]int64{} @@ -51,7 +52,7 @@ func (mk *MockStore) TopicsCount(startDate, endDate time.Time, projectUUIDs []st return res, nil } -func (mk *MockStore) SubscriptionsCount(startDate, endDate time.Time, projectUUIDs []string) (map[string]int64, error) { +func (mk *MockStore) SubscriptionsCount(ctx context.Context, startDate, endDate time.Time, projectUUIDs []string) (map[string]int64, error) { res := map[string]int64{} @@ -77,7 +78,7 @@ func (mk *MockStore) SubscriptionsCount(startDate, endDate time.Time, projectUUI return res, nil } -func (mk *MockStore) UsersCount(startDate, endDate time.Time, projectUUIDs []string) (map[string]int64, error) { +func (mk *MockStore) UsersCount(ctx context.Context, startDate, endDate time.Time, projectUUIDs []string) (map[string]int64, error) { res := map[string]int64{} @@ -108,7 +109,7 @@ func (mk *MockStore) UsersCount(startDate, endDate time.Time, projectUUIDs []str } // QueryACL Topic/Subscription ACL -func (mk *MockStore) QueryACL(projectUUID string, resource string, name string) (QAcl, error) { +func (mk *MockStore) QueryACL(ctx context.Context, projectUUID string, resource string, name string) (QAcl, error) { if resource == "topics" { if _, exists := mk.TopicsACL[name]; exists { return mk.TopicsACL[name], nil @@ -133,7 +134,7 @@ func NewMockStore(server string, database string) *MockStore { } // InsertOpMetric inserts a new operation metric -func (mk *MockStore) InsertOpMetric(hostname string, cpu float64, mem float64) error { +func (mk *MockStore) InsertOpMetric(ctx context.Context, hostname string, cpu float64, mem float64) error { qOp := QopMetric{hostname, cpu, mem} mk.OpMetrics[hostname] = qOp return nil @@ -145,7 +146,7 @@ func (mk *MockStore) Close() { } // InsertUser inserts a new user to the store -func (mk *MockStore) InsertUser(uuid string, projects []QProjectRoles, name string, fname string, lname string, org string, desc string, token string, email string, serviceRoles []string, createdOn time.Time, modifiedOn time.Time, createdBy string) error { +func (mk *MockStore) InsertUser(ctx context.Context, uuid string, projects []QProjectRoles, name string, fname string, lname string, org string, desc string, token string, email string, serviceRoles []string, createdOn time.Time, modifiedOn time.Time, createdBy string) error { user := QUser{ UUID: uuid, Name: name, @@ -165,7 +166,7 @@ func (mk *MockStore) InsertUser(uuid string, projects []QProjectRoles, name stri return nil } -func (mk *MockStore) RegisterUser(uuid, name, firstName, lastName, email, org, desc, registeredAt, atkn, status string) error { +func (mk *MockStore) RegisterUser(ctx context.Context, uuid, name, firstName, lastName, email, org, desc, registeredAt, atkn, status string) error { ur := QUserRegistration{ UUID: uuid, @@ -184,7 +185,7 @@ func (mk *MockStore) RegisterUser(uuid, name, firstName, lastName, email, org, d return nil } -func (mk *MockStore) QueryRegistrations(regUUID, status, activationToken, name, email, org string) ([]QUserRegistration, error) { +func (mk *MockStore) QueryRegistrations(ctx context.Context, regUUID, status, activationToken, name, email, org string) ([]QUserRegistration, error) { if regUUID == "" && status == "" && activationToken == "" && name == "" && email == "" && org == "" { return mk.UserRegistrations, nil @@ -256,7 +257,7 @@ func (mk *MockStore) QueryRegistrations(regUUID, status, activationToken, name, return reqs, nil } -func (mk *MockStore) UpdateRegistration(regUUID, status, declineComment, modifiedBy, modifiedAt string) error { +func (mk *MockStore) UpdateRegistration(ctx context.Context, regUUID, status, declineComment, modifiedBy, modifiedAt string) error { for idx, ur := range mk.UserRegistrations { if ur.UUID == regUUID { @@ -271,13 +272,13 @@ func (mk *MockStore) UpdateRegistration(regUUID, status, declineComment, modifie return nil } -//GetAllRoles returns a list of all available roles -func (mk *MockStore) GetAllRoles() []string { +// GetAllRoles returns a list of all available roles +func (mk *MockStore) GetAllRoles(ctx context.Context) []string { return []string{"service_admin", "admin", "project_admin", "viewer", "consumer", "producer", "publisher", "push_worker"} } // UpdateUserToken updates user's token -func (mk *MockStore) UpdateUserToken(uuid string, token string) error { +func (mk *MockStore) UpdateUserToken(ctx context.Context, uuid string, token string) error { for i, item := range mk.UserList { if item.UUID == uuid { mk.UserList[i].Token = token @@ -290,7 +291,7 @@ func (mk *MockStore) UpdateUserToken(uuid string, token string) error { } // GetOpMetrics returns operation metrics -func (mk *MockStore) GetOpMetrics() []QopMetric { +func (mk *MockStore) GetOpMetrics(ctx context.Context) []QopMetric { results := []QopMetric{} for _, v := range mk.OpMetrics { results = append(results, v) @@ -299,7 +300,7 @@ func (mk *MockStore) GetOpMetrics() []QopMetric { } // AppendToUserProjects adds project and specific roles to a users role list -func (mk *MockStore) AppendToUserProjects(userUUID string, projectUUID string, pRoles ...string) error { +func (mk *MockStore) AppendToUserProjects(ctx context.Context, userUUID string, projectUUID string, pRoles ...string) error { for idx, user := range mk.UserList { @@ -330,7 +331,7 @@ func (mk *MockStore) AppendToUserProjects(userUUID string, projectUUID string, p } // UpdateUser updates user information -func (mk *MockStore) UpdateUser(uuid, fname, lname, org, desc string, projects []QProjectRoles, name string, email string, serviceRoles []string, modifiedOn time.Time) error { +func (mk *MockStore) UpdateUser(ctx context.Context, uuid, fname, lname, org, desc string, projects []QProjectRoles, name string, email string, serviceRoles []string, modifiedOn time.Time) error { for i, item := range mk.UserList { if item.UUID == uuid { @@ -374,7 +375,7 @@ func (mk *MockStore) UpdateUser(uuid, fname, lname, org, desc string, projects [ } // HasUsers accepts a user array of usernames and returns the not found -func (mk *MockStore) HasUsers(projectUUID string, users []string) (bool, []string) { +func (mk *MockStore) HasUsers(ctx context.Context, projectUUID string, users []string) (bool, []string) { var notFound []string @@ -398,7 +399,7 @@ func (mk *MockStore) HasUsers(projectUUID string, users []string) (bool, []strin } // ModACL changes the acl in a function -func (mk *MockStore) ModACL(projectUUID string, resource string, name string, acl []string) error { +func (mk *MockStore) ModACL(ctx context.Context, projectUUID string, resource string, name string, acl []string) error { newACL := QAcl{ACL: acl} if resource == "topics" { if _, exists := mk.TopicsACL[name]; exists { @@ -416,7 +417,7 @@ func (mk *MockStore) ModACL(projectUUID string, resource string, name string, ac } // AppendToACL adds given users to an existing ACL -func (mk *MockStore) AppendToACL(projectUUID string, resource string, name string, acl []string) error { +func (mk *MockStore) AppendToACL(ctx context.Context, projectUUID string, resource string, name string, acl []string) error { if resource == "topics" { if qACL, exists := mk.TopicsACL[name]; exists { qACL.ACL = appendUniqueValues(qACL.ACL, acl...) @@ -453,7 +454,7 @@ func appendUniqueValues(existingValues []string, newValues ...string) []string { } // RemoveFromACL removes given users from an existing acl -func (mk *MockStore) RemoveFromACL(projectUUID string, resource string, name string, acl []string) error { +func (mk *MockStore) RemoveFromACL(ctx context.Context, projectUUID string, resource string, name string, acl []string) error { if resource == "topics" { if qACL, exists := mk.TopicsACL[name]; exists { qACL.ACL = removeValues(qACL.ACL, acl...) @@ -494,7 +495,7 @@ func removeSingleValue(existingValues []string, valueToRemove string) []string { } // UpdateProject updates project information -func (mk *MockStore) UpdateProject(projectUUID string, name string, description string, modifiedOn time.Time) error { +func (mk *MockStore) UpdateProject(ctx context.Context, projectUUID string, name string, description string, modifiedOn time.Time) error { for i, item := range mk.ProjectList { if item.UUID == projectUUID { @@ -515,7 +516,7 @@ func (mk *MockStore) UpdateProject(projectUUID string, name string, description } // QueryDailyProjectMsgCount retrieves the number of total messages that have been published to all project's topics daily -func (mk *MockStore) QueryDailyProjectMsgCount(projectUUID string) ([]QDailyProjectMsgCount, error) { +func (mk *MockStore) QueryDailyProjectMsgCount(ctx context.Context, projectUUID string) ([]QDailyProjectMsgCount, error) { var qDps []QDailyProjectMsgCount var ok bool @@ -549,8 +550,8 @@ func (mk *MockStore) QueryDailyProjectMsgCount(projectUUID string) ([]QDailyProj return qDps, nil } -//IncrementTopicMsgNum increase number of messages published in a topic -func (mk *MockStore) IncrementTopicMsgNum(projectUUID string, name string, num int64) error { +// IncrementTopicMsgNum increase number of messages published in a topic +func (mk *MockStore) IncrementTopicMsgNum(ctx context.Context, projectUUID string, name string, num int64) error { for i, item := range mk.TopicList { if item.ProjectUUID == projectUUID && item.Name == name { @@ -562,8 +563,8 @@ func (mk *MockStore) IncrementTopicMsgNum(projectUUID string, name string, num i return errors.New("not found") } -//IncrementDailyTopicMsgCount increase number of messages published in a topic -func (mk *MockStore) IncrementDailyTopicMsgCount(projectUUID string, topicName string, num int64, date time.Time) error { +// IncrementDailyTopicMsgCount increase number of messages published in a topic +func (mk *MockStore) IncrementDailyTopicMsgCount(ctx context.Context, projectUUID string, topicName string, num int64, date time.Time) error { for i, item := range mk.DailyTopicMsgCount { if item.ProjectUUID == projectUUID && item.TopicName == topicName && item.Date.Equal(date) { @@ -576,8 +577,8 @@ func (mk *MockStore) IncrementDailyTopicMsgCount(projectUUID string, topicName s return nil } -//IncrementTopicBytes increases the total number of bytes published in a topic -func (mk *MockStore) IncrementTopicBytes(projectUUID string, name string, totalBytes int64) error { +// IncrementTopicBytes increases the total number of bytes published in a topic +func (mk *MockStore) IncrementTopicBytes(ctx context.Context, projectUUID string, name string, totalBytes int64) error { for i, item := range mk.TopicList { if item.ProjectUUID == projectUUID && item.Name == name { mk.TopicList[i].TotalBytes += totalBytes @@ -588,8 +589,8 @@ func (mk *MockStore) IncrementTopicBytes(projectUUID string, name string, totalB return errors.New("not found") } -//IncrementSubBytes increases the total number of bytes published in a subscription -func (mk *MockStore) IncrementSubBytes(projectUUID string, name string, totalBytes int64) error { +// IncrementSubBytes increases the total number of bytes published in a subscription +func (mk *MockStore) IncrementSubBytes(ctx context.Context, projectUUID string, name string, totalBytes int64) error { for i, item := range mk.SubList { if item.ProjectUUID == projectUUID && item.Name == name { mk.SubList[i].TotalBytes += totalBytes @@ -600,8 +601,8 @@ func (mk *MockStore) IncrementSubBytes(projectUUID string, name string, totalByt return errors.New("not found") } -//IncrementSubMsgNum increase number of messages pulled in a subscription -func (mk *MockStore) IncrementSubMsgNum(projectUUID string, name string, num int64) error { +// IncrementSubMsgNum increase number of messages pulled in a subscription +func (mk *MockStore) IncrementSubMsgNum(ctx context.Context, projectUUID string, name string, num int64) error { for i, item := range mk.SubList { if item.ProjectUUID == projectUUID && item.Name == name { @@ -614,12 +615,12 @@ func (mk *MockStore) IncrementSubMsgNum(projectUUID string, name string, num int } // UpdateSubOffset updates the offset of the current subscription -func (mk *MockStore) UpdateSubOffset(projectUUID string, name string, offset int64) { +func (mk *MockStore) UpdateSubOffset(ctx context.Context, projectUUID string, name string, offset int64) { } // ModAck modifies the subscription ack -func (mk *MockStore) ModAck(projectUUID string, name string, ack int) error { +func (mk *MockStore) ModAck(ctx context.Context, projectUUID string, name string, ack int) error { for i, item := range mk.SubList { if item.ProjectUUID == projectUUID && item.Name == name { mk.SubList[i].Ack = ack @@ -632,7 +633,7 @@ func (mk *MockStore) ModAck(projectUUID string, name string, ack int) error { } // ModSubPush modifies the subscription push configuration -func (mk *MockStore) ModSubPush(projectUUID string, name string, config QPushConfig) error { +func (mk *MockStore) ModSubPush(ctx context.Context, projectUUID string, name string, config QPushConfig) error { for i, item := range mk.SubList { if item.ProjectUUID == projectUUID && item.Name == name { mk.SubList[i].PushType = config.Type @@ -654,7 +655,7 @@ func (mk *MockStore) ModSubPush(projectUUID string, name string, config QPushCon } // UpdateSubOffsetAck updates the offset of the current subscription -func (mk *MockStore) UpdateSubOffsetAck(projectUUID string, name string, offset int64, ts string) error { +func (mk *MockStore) UpdateSubOffsetAck(ctx context.Context, projectUUID string, name string, offset int64, ts string) error { // find sub sub := QSub{} @@ -688,7 +689,7 @@ func (mk *MockStore) UpdateSubOffsetAck(projectUUID string, name string, offset } // QueryProjects function queries for a specific project or for a list of all projects -func (mk *MockStore) QueryProjects(uuid string, name string) ([]QProject, error) { +func (mk *MockStore) QueryProjects(ctx context.Context, uuid string, name string) ([]QProject, error) { result := []QProject{} if name == "" && uuid == "" { @@ -719,7 +720,7 @@ func (mk *MockStore) QueryProjects(uuid string, name string) ([]QProject, error) } // QueryUsers queries the datastore for user information -func (mk *MockStore) QueryUsers(projectUUID string, uuid string, name string) ([]QUser, error) { +func (mk *MockStore) QueryUsers(ctx context.Context, projectUUID string, uuid string, name string) ([]QUser, error) { result := []QUser{} if name == "" && uuid == "" && projectUUID == "" { @@ -756,7 +757,7 @@ func (mk *MockStore) QueryUsers(projectUUID string, uuid string, name string) ([ } // PaginatedQueryUsers provides query to the list of users using pagination parameters -func (mk *MockStore) PaginatedQueryUsers(pageToken string, pageSize int64, projectUUID string) ([]QUser, int64, string, error) { +func (mk *MockStore) PaginatedQueryUsers(ctx context.Context, pageToken string, pageSize int64, projectUUID string) ([]QUser, int64, string, error) { var qUsers []QUser var nextPageToken string @@ -844,7 +845,7 @@ func (mk *MockStore) PaginatedQueryUsers(pageToken string, pageSize int64, proje } // UpdateSubPull updates next offset info after a pull -func (mk *MockStore) UpdateSubPull(projectUUID string, name string, offset int64, ts string) error { +func (mk *MockStore) UpdateSubPull(ctx context.Context, projectUUID string, name string, offset int64, ts string) error { for i, item := range mk.SubList { if item.ProjectUUID == projectUUID && item.Name == name { mk.SubList[i].NextOffset = offset @@ -1007,7 +1008,7 @@ func (mk *MockStore) Initialize() { mk.UserRegistrations = append(mk.UserRegistrations, ur1) } -func (mk *MockStore) QueryTotalMessagesPerProject(projectUUIDs []string, startDate time.Time, endDate time.Time) ([]QProjectMessageCount, error) { +func (mk *MockStore) QueryTotalMessagesPerProject(ctx context.Context, projectUUIDs []string, startDate time.Time, endDate time.Time) ([]QProjectMessageCount, error) { projectCount := make(map[string]int64) @@ -1063,7 +1064,7 @@ func (mk *MockStore) QueryTotalMessagesPerProject(projectUUIDs []string, startDa } // QueryOneSub returns one sub exactly -func (mk *MockStore) QueryOneSub(projectUUID string, name string) (QSub, error) { +func (mk *MockStore) QueryOneSub(ctx context.Context, projectUUID string, name string) (QSub, error) { for _, item := range mk.SubList { if item.Name == name && item.ProjectUUID == projectUUID { return item, nil @@ -1079,7 +1080,7 @@ func (mk *MockStore) Clone() Store { } // GetUserFromToken retrieves specific user info from a given token -func (mk *MockStore) GetUserFromToken(token string) (QUser, error) { +func (mk *MockStore) GetUserFromToken(ctx context.Context, token string) (QUser, error) { for _, item := range mk.UserList { if item.Token == token { @@ -1093,7 +1094,7 @@ func (mk *MockStore) GetUserFromToken(token string) (QUser, error) { } // GetUserRoles returns the roles of a user in a project -func (mk *MockStore) GetUserRoles(projectUUID string, token string) ([]string, string) { +func (mk *MockStore) GetUserRoles(ctx context.Context, projectUUID string, token string) ([]string, string) { for _, item := range mk.UserList { if item.Token == token { @@ -1105,8 +1106,8 @@ func (mk *MockStore) GetUserRoles(projectUUID string, token string) ([]string, s return []string{}, "" } -//HasResourceRoles returns the roles of a user in a project -func (mk *MockStore) HasResourceRoles(resource string, roles []string) bool { +// HasResourceRoles returns the roles of a user in a project +func (mk *MockStore) HasResourceRoles(ctx context.Context, resource string, roles []string) bool { for _, item := range mk.RoleList { if item.Name == resource { @@ -1126,7 +1127,7 @@ func (mk *MockStore) HasResourceRoles(resource string, roles []string) bool { } // HasProject returns true if project exists in store -func (mk *MockStore) HasProject(name string) bool { +func (mk *MockStore) HasProject(ctx context.Context, name string) bool { for _, item := range mk.ProjectList { if item.Name == name { return true @@ -1137,7 +1138,7 @@ func (mk *MockStore) HasProject(name string) bool { } // InsertTopic inserts a new topic object to the store -func (mk *MockStore) InsertTopic(projectUUID string, name string, schemaUUID string, createdOn time.Time) error { +func (mk *MockStore) InsertTopic(ctx context.Context, projectUUID string, name string, schemaUUID string, createdOn time.Time) error { topic := QTopic{ ID: len(mk.TopicList), ProjectUUID: projectUUID, @@ -1154,7 +1155,7 @@ func (mk *MockStore) InsertTopic(projectUUID string, name string, schemaUUID str return nil } -func (mk *MockStore) LinkTopicSchema(projectUUID, name, schemaUUID string) error { +func (mk *MockStore) LinkTopicSchema(ctx context.Context, projectUUID, name, schemaUUID string) error { for idx, topic := range mk.TopicList { if topic.Name == name && topic.ProjectUUID == projectUUID { mk.TopicList[idx].SchemaUUID = schemaUUID @@ -1165,7 +1166,7 @@ func (mk *MockStore) LinkTopicSchema(projectUUID, name, schemaUUID string) error } // InsertSub inserts a new sub object to the store -func (mk *MockStore) InsertSub(projectUUID string, name string, topic string, +func (mk *MockStore) InsertSub(ctx context.Context, projectUUID string, name string, topic string, offset int64, ack int, pushCfg QPushConfig, createdOn time.Time) error { sub := QSub{ ID: len(mk.SubList), @@ -1201,14 +1202,14 @@ func (mk *MockStore) InsertSub(projectUUID string, name string, topic string, } // InsertProject inserts a project to the store -func (mk *MockStore) InsertProject(uuid string, name string, createdOn time.Time, modifiedOn time.Time, createdBy string, description string) error { +func (mk *MockStore) InsertProject(ctx context.Context, uuid string, name string, createdOn time.Time, modifiedOn time.Time, createdBy string, description string) error { project := QProject{UUID: uuid, Name: name, CreatedOn: createdOn, ModifiedOn: modifiedOn, CreatedBy: createdBy, Description: description} mk.ProjectList = append(mk.ProjectList, project) return nil } // RemoveProject removes an existing project -func (mk *MockStore) RemoveProject(uuid string) error { +func (mk *MockStore) RemoveProject(ctx context.Context, uuid string) error { for i, project := range mk.ProjectList { if project.UUID == uuid { // found item at i, remove it using index @@ -1221,7 +1222,7 @@ func (mk *MockStore) RemoveProject(uuid string) error { } // RemoveTopic removes an existing topic -func (mk *MockStore) RemoveTopic(projectUUID string, name string) error { +func (mk *MockStore) RemoveTopic(ctx context.Context, projectUUID string, name string) error { for i, topic := range mk.TopicList { if topic.Name == name && topic.ProjectUUID == projectUUID { // found item at i, remove it using index @@ -1234,7 +1235,7 @@ func (mk *MockStore) RemoveTopic(projectUUID string, name string) error { } // RemoveUser removes an existing user -func (mk *MockStore) RemoveUser(uuid string) error { +func (mk *MockStore) RemoveUser(ctx context.Context, uuid string) error { for i, user := range mk.UserList { if user.UUID == uuid { // found item at i, remove it using index @@ -1247,7 +1248,7 @@ func (mk *MockStore) RemoveUser(uuid string) error { } // RemoveProjectTopics removes all topics belonging to a specific project uuid -func (mk *MockStore) RemoveProjectTopics(projectUUID string) error { +func (mk *MockStore) RemoveProjectTopics(ctx context.Context, projectUUID string) error { found := false newList := []QTopic{} for _, topic := range mk.TopicList { @@ -1266,7 +1267,7 @@ func (mk *MockStore) RemoveProjectTopics(projectUUID string) error { } // RemoveProjectSubs removes all existing subs belonging to a specific project uuid -func (mk *MockStore) RemoveProjectSubs(projectUUID string) error { +func (mk *MockStore) RemoveProjectSubs(ctx context.Context, projectUUID string) error { found := false newList := []QSub{} for _, sub := range mk.SubList { @@ -1285,7 +1286,7 @@ func (mk *MockStore) RemoveProjectSubs(projectUUID string) error { } // RemoveProjectDailyMessageCounters removes all existing message counters belonging to a specific project uuid -func (mk *MockStore) RemoveProjectDailyMessageCounters(projectUUID string) error { +func (mk *MockStore) RemoveProjectDailyMessageCounters(ctx context.Context, projectUUID string) error { found := false newList := []QDailyTopicMsgCount{} for _, qc := range mk.DailyTopicMsgCount { @@ -1304,7 +1305,7 @@ func (mk *MockStore) RemoveProjectDailyMessageCounters(projectUUID string) error } // RemoveSub removes an existing sub from the store -func (mk *MockStore) RemoveSub(projectUUID string, name string) error { +func (mk *MockStore) RemoveSub(ctx context.Context, projectUUID string, name string) error { for i, sub := range mk.SubList { if sub.Name == name && sub.ProjectUUID == projectUUID { // found item at i, remove it using index @@ -1317,12 +1318,12 @@ func (mk *MockStore) RemoveSub(projectUUID string, name string) error { } // QueryPushSubs Query push Subscription info from store -func (mk *MockStore) QueryPushSubs() []QSub { +func (mk *MockStore) QueryPushSubs(ctx context.Context) []QSub { return mk.SubList } // QuerySubs Query Subscription info from store -func (mk *MockStore) QuerySubs(projectUUID, userUUID, name, pageToken string, pageSize int64) ([]QSub, int64, string, error) { +func (mk *MockStore) QuerySubs(ctx context.Context, projectUUID, userUUID, name, pageToken string, pageSize int64) ([]QSub, int64, string, error) { var qSubs []QSub var totalSize int64 @@ -1336,7 +1337,7 @@ func (mk *MockStore) QuerySubs(projectUUID, userUUID, name, pageToken string, pa if sub.ProjectUUID == projectUUID { if userUUID != "" { - if !mk.existsInACL("subscriptions", sub.Name, userUUID) { + if !mk.existsInACL(ctx, "subscriptions", sub.Name, userUUID) { continue } @@ -1378,7 +1379,7 @@ func (mk *MockStore) QuerySubs(projectUUID, userUUID, name, pageToken string, pa if sub.ID.(int) <= pg && sub.ProjectUUID == projectUUID { if userUUID != "" { - if !mk.existsInACL("subscriptions", sub.Name, userUUID) { + if !mk.existsInACL(ctx, "subscriptions", sub.Name, userUUID) { continue } } @@ -1412,7 +1413,7 @@ func (mk *MockStore) QuerySubs(projectUUID, userUUID, name, pageToken string, pa if sub.ProjectUUID == projectUUID && sub.Name == name { if userUUID != "" { - if !mk.existsInACL("subscriptions", sub.Name, userUUID) { + if !mk.existsInACL(ctx, "subscriptions", sub.Name, userUUID) { continue } } @@ -1428,7 +1429,7 @@ func (mk *MockStore) QuerySubs(projectUUID, userUUID, name, pageToken string, pa } // QuerySubsByTopic returns subscriptions attached to a given topic -func (mk *MockStore) QuerySubsByTopic(projectUUID, topic string) ([]QSub, error) { +func (mk *MockStore) QuerySubsByTopic(ctx context.Context, projectUUID, topic string) ([]QSub, error) { result := []QSub{} for _, item := range mk.SubList { if projectUUID == item.ProjectUUID && item.Topic == topic { @@ -1439,7 +1440,7 @@ func (mk *MockStore) QuerySubsByTopic(projectUUID, topic string) ([]QSub, error) } // QuerySubsByACL returns subscriptions that contain a specific user in their ACL -func (mk *MockStore) QuerySubsByACL(projectUUID, user string) ([]QSub, error) { +func (mk *MockStore) QuerySubsByACL(ctx context.Context, projectUUID, user string) ([]QSub, error) { result := []QSub{} for _, item := range mk.SubList { @@ -1456,7 +1457,7 @@ func (mk *MockStore) QuerySubsByACL(projectUUID, user string) ([]QSub, error) { } // QueryTopicsByACL returns topics that contain a specific user in their ACL -func (mk *MockStore) QueryTopicsByACL(projectUUID, user string) ([]QTopic, error) { +func (mk *MockStore) QueryTopicsByACL(ctx context.Context, projectUUID, user string) ([]QTopic, error) { result := []QTopic{} for _, item := range mk.TopicList { @@ -1473,7 +1474,7 @@ func (mk *MockStore) QueryTopicsByACL(projectUUID, user string) ([]QTopic, error } // QueryTopics Query Subscription info from store -func (mk *MockStore) QueryTopics(projectUUID, userUUID, name, pageToken string, pageSize int64) ([]QTopic, int64, string, error) { +func (mk *MockStore) QueryTopics(ctx context.Context, projectUUID, userUUID, name, pageToken string, pageSize int64) ([]QTopic, int64, string, error) { var qTopics []QTopic var totalSize int64 @@ -1487,7 +1488,7 @@ func (mk *MockStore) QueryTopics(projectUUID, userUUID, name, pageToken string, if topic.ProjectUUID == projectUUID { if userUUID != "" { - if !mk.existsInACL("topics", topic.Name, userUUID) { + if !mk.existsInACL(ctx, "topics", topic.Name, userUUID) { continue } @@ -1528,7 +1529,7 @@ func (mk *MockStore) QueryTopics(projectUUID, userUUID, name, pageToken string, if topic.ID.(int) <= pg && topic.ProjectUUID == projectUUID { if userUUID != "" { - if !mk.existsInACL("topics", topic.Name, userUUID) { + if !mk.existsInACL(ctx, "topics", topic.Name, userUUID) { continue } } @@ -1543,7 +1544,7 @@ func (mk *MockStore) QueryTopics(projectUUID, userUUID, name, pageToken string, if topic.ProjectUUID == projectUUID { if userUUID != "" { - if !mk.existsInACL("topics", topic.Name, userUUID) { + if !mk.existsInACL(ctx, "topics", topic.Name, userUUID) { continue } } @@ -1569,7 +1570,7 @@ func (mk *MockStore) QueryTopics(projectUUID, userUUID, name, pageToken string, if topic.ProjectUUID == projectUUID && topic.Name == name { if userUUID != "" { - if !mk.existsInACL("topics", topic.Name, userUUID) { + if !mk.existsInACL(ctx, "topics", topic.Name, userUUID) { continue } } @@ -1584,7 +1585,7 @@ func (mk *MockStore) QueryTopics(projectUUID, userUUID, name, pageToken string, return qTopics, totalSize, nextPageToken, nil } -func (mk *MockStore) existsInACL(resource, resourceName, userUUID string) bool { +func (mk *MockStore) existsInACL(ctx context.Context, resource, resourceName, userUUID string) bool { var acl QAcl @@ -1606,7 +1607,7 @@ func (mk *MockStore) existsInACL(resource, resourceName, userUUID string) bool { } // Checks if a users exists in an ACL resource (topic or subscription) -func (mk *MockStore) ExistsInACL(projectUUID string, resource string, resourceName string, userUUID string) error { +func (mk *MockStore) ExistsInACL(ctx context.Context, projectUUID string, resource string, resourceName string, userUUID string) error { var acl QAcl @@ -1627,7 +1628,7 @@ func (mk *MockStore) ExistsInACL(projectUUID string, resource string, resourceNa } // UpdateTopicLatestPublish updates the topic's latest publish time -func (mk *MockStore) UpdateTopicLatestPublish(projectUUID string, name string, date time.Time) error { +func (mk *MockStore) UpdateTopicLatestPublish(ctx context.Context, projectUUID string, name string, date time.Time) error { for idx, topic := range mk.TopicList { if topic.ProjectUUID == projectUUID && topic.Name == name { mk.TopicList[idx].LatestPublish = date @@ -1638,7 +1639,7 @@ func (mk *MockStore) UpdateTopicLatestPublish(projectUUID string, name string, d } // UpdateTopicPublishRate updates the topic's publishing rate -func (mk *MockStore) UpdateTopicPublishRate(projectUUID string, name string, rate float64) error { +func (mk *MockStore) UpdateTopicPublishRate(ctx context.Context, projectUUID string, name string, rate float64) error { for idx, topic := range mk.TopicList { if topic.ProjectUUID == projectUUID && topic.Name == name { mk.TopicList[idx].PublishRate = rate @@ -1649,7 +1650,7 @@ func (mk *MockStore) UpdateTopicPublishRate(projectUUID string, name string, rat } // UpdateSubLatestConsume updates the subscription's latest consume time -func (mk *MockStore) UpdateSubLatestConsume(projectUUID string, name string, date time.Time) error { +func (mk *MockStore) UpdateSubLatestConsume(ctx context.Context, projectUUID string, name string, date time.Time) error { for idx, topic := range mk.SubList { if topic.ProjectUUID == projectUUID && topic.Name == name { mk.SubList[idx].LatestConsume = date @@ -1660,7 +1661,7 @@ func (mk *MockStore) UpdateSubLatestConsume(projectUUID string, name string, dat } // UpdateSubConsumeRate updates the subscription's consume rate -func (mk *MockStore) UpdateSubConsumeRate(projectUUID string, name string, rate float64) error { +func (mk *MockStore) UpdateSubConsumeRate(ctx context.Context, projectUUID string, name string, rate float64) error { for idx, topic := range mk.SubList { if topic.ProjectUUID == projectUUID && topic.Name == name { mk.SubList[idx].ConsumeRate = rate @@ -1671,7 +1672,7 @@ func (mk *MockStore) UpdateSubConsumeRate(projectUUID string, name string, rate } // QueryDailyTopicMsgCount returns results regarding the number of messages published to a topic -func (mk *MockStore) QueryDailyTopicMsgCount(projectUUID string, topicName string, date time.Time) ([]QDailyTopicMsgCount, error) { +func (mk *MockStore) QueryDailyTopicMsgCount(ctx context.Context, projectUUID string, topicName string, date time.Time) ([]QDailyTopicMsgCount, error) { var qds []QDailyTopicMsgCount var zeroValueTime time.Time @@ -1703,7 +1704,7 @@ func (mk *MockStore) QueryDailyTopicMsgCount(projectUUID string, topicName strin return qds, nil } -func (mk *MockStore) InsertSchema(projectUUID, schemaUUID, name, schemaType, rawSchemaString string) error { +func (mk *MockStore) InsertSchema(ctx context.Context, projectUUID, schemaUUID, name, schemaType, rawSchemaString string) error { mk.SchemaList = append(mk.SchemaList, QSchema{ ProjectUUID: projectUUID, UUID: schemaUUID, @@ -1715,7 +1716,7 @@ func (mk *MockStore) InsertSchema(projectUUID, schemaUUID, name, schemaType, raw return nil } -func (mk *MockStore) QuerySchemas(projectUUID, schemaUUID, name string) ([]QSchema, error) { +func (mk *MockStore) QuerySchemas(ctx context.Context, projectUUID, schemaUUID, name string) ([]QSchema, error) { qSchemas := []QSchema{} @@ -1754,7 +1755,7 @@ func (mk *MockStore) QuerySchemas(projectUUID, schemaUUID, name string) ([]QSche return qSchemas, nil } -func (mk *MockStore) UpdateSchema(schemaUUID, name, schemaType, rawSchemaString string) error { +func (mk *MockStore) UpdateSchema(ctx context.Context, schemaUUID, name, schemaType, rawSchemaString string) error { for idx, s := range mk.SchemaList { if s.UUID == schemaUUID { @@ -1777,7 +1778,7 @@ func (mk *MockStore) UpdateSchema(schemaUUID, name, schemaType, rawSchemaString return errors.New("not found") } -func (mk *MockStore) DeleteSchema(schemaUUID string) error { +func (mk *MockStore) DeleteSchema(ctx context.Context, schemaUUID string) error { for idx, s := range mk.SchemaList { if s.UUID == schemaUUID { @@ -1796,7 +1797,7 @@ func (mk *MockStore) DeleteSchema(schemaUUID string) error { return errors.New("not found") } -func (mk *MockStore) DeleteRegistration(regUUID string) error { +func (mk *MockStore) DeleteRegistration(ctx context.Context, regUUID string) error { for idx, s := range mk.UserRegistrations { if s.UUID == regUUID { diff --git a/stores/mongo.go b/stores/mongo.go index 3913387e..228f22e8 100644 --- a/stores/mongo.go +++ b/stores/mongo.go @@ -1,6 +1,7 @@ package stores import ( + "context" "errors" "time" @@ -78,18 +79,30 @@ func (mong *MongoStore) Initialize() { } } +func (mong *MongoStore) logErrorAndCrash(ctx context.Context, funcName string, err error) { + log.WithFields( + log.Fields{ + "trace_id": ctx.Value("trace_id"), + "type": "backend_log", + "function": funcName, + "backend_service": "mongo", + "backend_hosts": mong.Server, + }, + ).Fatal(err.Error()) +} + // SubscriptionsCount returns the amount of subscriptions created in the given time period -func (mong *MongoStore) SubscriptionsCount(startDate, endDate time.Time, projectUUIDs []string) (map[string]int64, error) { - return mong.getDocCountForCollection(startDate, endDate, "subscriptions", projectUUIDs) +func (mong *MongoStore) SubscriptionsCount(ctx context.Context, startDate, endDate time.Time, projectUUIDs []string) (map[string]int64, error) { + return mong.getDocCountForCollection(ctx, startDate, endDate, "subscriptions", projectUUIDs) } // TopicsCount returns the amount of topics created in the given time period -func (mong *MongoStore) TopicsCount(startDate, endDate time.Time, projectUUIDs []string) (map[string]int64, error) { - return mong.getDocCountForCollection(startDate, endDate, "topics", projectUUIDs) +func (mong *MongoStore) TopicsCount(ctx context.Context, startDate, endDate time.Time, projectUUIDs []string) (map[string]int64, error) { + return mong.getDocCountForCollection(ctx, startDate, endDate, "topics", projectUUIDs) } -// UserCount returns the amount of users created in the given time period -func (mong *MongoStore) UsersCount(startDate, endDate time.Time, projectUUIDs []string) (map[string]int64, error) { +// UsersCount returns the amount of users created in the given time period +func (mong *MongoStore) UsersCount(ctx context.Context, startDate, endDate time.Time, projectUUIDs []string) (map[string]int64, error) { var resourceCounts []QProjectResourceCount condQuery := []bson.M{ @@ -145,13 +158,7 @@ func (mong *MongoStore) UsersCount(startDate, endDate time.Time, projectUUIDs [] c := mong.Session.DB(mong.Database).C("users") if err := c.Pipe(query).All(&resourceCounts); err != nil { - log.WithFields( - log.Fields{ - "type": "backend_log", - "backend_service": "mongo", - "backend_hosts": mong.Server, - }, - ).Fatal(err.Error()) + mong.logErrorAndCrash(ctx, "UsersCount", err) } res := map[string]int64{} @@ -165,7 +172,7 @@ func (mong *MongoStore) UsersCount(startDate, endDate time.Time, projectUUIDs [] // getDocCountForCollection returns the document count for a collection in a given time period // collection should support field created_on -func (mong *MongoStore) getDocCountForCollection(startDate, endDate time.Time, col string, projectUUIDs []string) (map[string]int64, error) { +func (mong *MongoStore) getDocCountForCollection(ctx context.Context, startDate, endDate time.Time, col string, projectUUIDs []string) (map[string]int64, error) { var resourceCounts []QProjectResourceCount @@ -219,13 +226,7 @@ func (mong *MongoStore) getDocCountForCollection(startDate, endDate time.Time, c c := mong.Session.DB(mong.Database).C(col) if err := c.Pipe(query).All(&resourceCounts); err != nil { - log.WithFields( - log.Fields{ - "type": "backend_log", - "backend_service": "mongo", - "backend_hosts": mong.Server, - }, - ).Fatal(err.Error()) + mong.logErrorAndCrash(ctx, "getDocCountForCollection", err) } res := map[string]int64{} @@ -238,7 +239,7 @@ func (mong *MongoStore) getDocCountForCollection(startDate, endDate time.Time, c } // QueryProjects queries the database for a specific project or a list of all projects -func (mong *MongoStore) QueryProjects(uuid string, name string) ([]QProject, error) { +func (mong *MongoStore) QueryProjects(ctx context.Context, uuid string, name string) ([]QProject, error) { query := bson.M{} @@ -255,13 +256,7 @@ func (mong *MongoStore) QueryProjects(uuid string, name string) ([]QProject, err var results []QProject err := c.Find(query).All(&results) if err != nil { - log.WithFields( - log.Fields{ - "type": "backend_log", - "backend_service": "mongo", - "backend_hosts": mong.Server, - }, - ).Fatal(err.Error()) + mong.logErrorAndCrash(ctx, "QueryProjects", err) } if len(results) > 0 { @@ -272,12 +267,12 @@ func (mong *MongoStore) QueryProjects(uuid string, name string) ([]QProject, err } // UpdateProject updates project information -func (mong *MongoStore) UpdateProject(projectUUID string, name string, description string, modifiedOn time.Time) error { +func (mong *MongoStore) UpdateProject(ctx context.Context, projectUUID string, name string, description string, modifiedOn time.Time) error { db := mong.Session.DB(mong.Database) c := db.C("projects") doc := bson.M{"uuid": projectUUID} - results, err := mong.QueryProjects(projectUUID, "") + results, err := mong.QueryProjects(ctx, projectUUID, "") if err != nil { return err } @@ -288,7 +283,7 @@ func (mong *MongoStore) UpdateProject(projectUUID string, name string, descripti if name != "" { // Check if name is going to change and if that name already exists if name != curPr.Name { - if sameRes, _ := mong.QueryProjects("", name); len(sameRes) > 0 { + if sameRes, _ := mong.QueryProjects(ctx, "", name); len(sameRes) > 0 { return errors.New("invalid project name change, name already exists") } } @@ -303,12 +298,12 @@ func (mong *MongoStore) UpdateProject(projectUUID string, name string, descripti err = c.Update(doc, change) - return err + return nil } // RegisterUser inserts a new user registration to the database -func (mong *MongoStore) RegisterUser(uuid, name, firstName, lastName, email, org, desc, registeredAt, atkn, status string) error { +func (mong *MongoStore) RegisterUser(ctx context.Context, uuid, name, firstName, lastName, email, org, desc, registeredAt, atkn, status string) error { ur := QUserRegistration{ UUID: uuid, @@ -323,15 +318,15 @@ func (mong *MongoStore) RegisterUser(uuid, name, firstName, lastName, email, org Status: status, } - return mong.InsertResource("user_registrations", ur) + return mong.InsertResource(ctx, "user_registrations", ur) } // DeleteRegistration removes the respective registration from the -func (mong *MongoStore) DeleteRegistration(uuid string) error { - return mong.RemoveResource("user_registrations", bson.M{"uuid": uuid}) +func (mong *MongoStore) DeleteRegistration(ctx context.Context, uuid string) error { + return mong.RemoveResource(ctx, "user_registrations", bson.M{"uuid": uuid}) } -func (mong *MongoStore) QueryRegistrations(regUUID, status, activationToken, name, email, org string) ([]QUserRegistration, error) { +func (mong *MongoStore) QueryRegistrations(ctx context.Context, regUUID, status, activationToken, name, email, org string) ([]QUserRegistration, error) { query := bson.M{} @@ -371,7 +366,7 @@ func (mong *MongoStore) QueryRegistrations(regUUID, status, activationToken, nam return qur, nil } -func (mong *MongoStore) UpdateRegistration(regUUID, status, declineComment, modifiedBy, modifiedAt string) error { +func (mong *MongoStore) UpdateRegistration(ctx context.Context, regUUID, status, declineComment, modifiedBy, modifiedAt string) error { db := mong.Session.DB(mong.Database) c := db.C("user_registrations") @@ -390,7 +385,7 @@ func (mong *MongoStore) UpdateRegistration(regUUID, status, declineComment, modi } // UpdateUserToken updates user's token -func (mong *MongoStore) UpdateUserToken(uuid string, token string) error { +func (mong *MongoStore) UpdateUserToken(ctx context.Context, uuid string, token string) error { db := mong.Session.DB(mong.Database) c := db.C("users") @@ -405,7 +400,7 @@ func (mong *MongoStore) UpdateUserToken(uuid string, token string) error { } // AppendToUserProjects appends a new unique project to the user's projects -func (mong *MongoStore) AppendToUserProjects(userUUID string, projectUUID string, pRoles ...string) error { +func (mong *MongoStore) AppendToUserProjects(ctx context.Context, userUUID string, projectUUID string, pRoles ...string) error { db := mong.Session.DB(mong.Database) c := db.C("users") @@ -423,25 +418,19 @@ func (mong *MongoStore) AppendToUserProjects(userUUID string, projectUUID string ) if err != nil { - log.WithFields( - log.Fields{ - "type": "backend_log", - "backend_service": "mongo", - "backend_hosts": mong.Server, - }, - ).Fatal(err.Error()) + mong.logErrorAndCrash(ctx, "AppendToUserProjects", err) } return nil } // UpdateUser updates user information -func (mong *MongoStore) UpdateUser(uuid, fname, lname, org, desc string, projects []QProjectRoles, name string, email string, serviceRoles []string, modifiedOn time.Time) error { +func (mong *MongoStore) UpdateUser(ctx context.Context, uuid, fname, lname, org, desc string, projects []QProjectRoles, name string, email string, serviceRoles []string, modifiedOn time.Time) error { db := mong.Session.DB(mong.Database) c := db.C("users") doc := bson.M{"uuid": uuid} - results, err := mong.QueryUsers("", uuid, "") + results, err := mong.QueryUsers(ctx, "", uuid, "") if err != nil { return err } @@ -451,7 +440,7 @@ func (mong *MongoStore) UpdateUser(uuid, fname, lname, org, desc string, project if name != "" { // Check if name is going to change and if that name already exists if name != curUsr.Name { - if sameRes, _ := mong.QueryUsers("", "", name); len(sameRes) > 0 { + if sameRes, _ := mong.QueryUsers(ctx, "", "", name); len(sameRes) > 0 { return errors.New("invalid user name change, name already exists") } } @@ -497,7 +486,7 @@ func (mong *MongoStore) UpdateUser(uuid, fname, lname, org, desc string, project } // UpdateSubPull updates next offset and sets timestamp for Ack -func (mong *MongoStore) UpdateSubPull(projectUUID string, name string, nextOff int64, ts string) error { +func (mong *MongoStore) UpdateSubPull(ctx context.Context, projectUUID string, name string, nextOff int64, ts string) error { db := mong.Session.DB(mong.Database) c := db.C("subscriptions") @@ -506,13 +495,7 @@ func (mong *MongoStore) UpdateSubPull(projectUUID string, name string, nextOff i change := bson.M{"$set": bson.M{"next_offset": nextOff, "pending_ack": ts}} err := c.Update(doc, change) if err != nil && err != mgo.ErrNotFound { - log.WithFields( - log.Fields{ - "type": "backend_log", - "backend_service": "mongo", - "backend_hosts": mong.Server, - }, - ).Fatal(err.Error()) + mong.logErrorAndCrash(ctx, "UpdateSubPull", err) } return err @@ -520,7 +503,7 @@ func (mong *MongoStore) UpdateSubPull(projectUUID string, name string, nextOff i } // UpdateSubOffsetAck updates a subscription offset after Ack -func (mong *MongoStore) UpdateSubOffsetAck(projectUUID string, name string, offset int64, ts string) error { +func (mong *MongoStore) UpdateSubOffsetAck(ctx context.Context, projectUUID string, name string, offset int64, ts string) error { db := mong.Session.DB(mong.Database) c := db.C("subscriptions") @@ -553,13 +536,7 @@ func (mong *MongoStore) UpdateSubOffsetAck(projectUUID string, name string, offs change := bson.M{"$set": bson.M{"offset": offset, "next_offset": 0, "pending_ack": ""}} err = c.Update(doc, change) if err != nil && err != mgo.ErrNotFound { - log.WithFields( - log.Fields{ - "type": "backend_log", - "backend_service": "mongo", - "backend_hosts": mong.Server, - }, - ).Fatal(err.Error()) + mong.logErrorAndCrash(ctx, "UpdateSubOffsetAck", err) } return nil @@ -567,7 +544,7 @@ func (mong *MongoStore) UpdateSubOffsetAck(projectUUID string, name string, offs } // UpdateSubOffset updates a subscription offset -func (mong *MongoStore) UpdateSubOffset(projectUUID string, name string, offset int64) { +func (mong *MongoStore) UpdateSubOffset(ctx context.Context, projectUUID string, name string, offset int64) { db := mong.Session.DB(mong.Database) c := db.C("subscriptions") @@ -576,19 +553,13 @@ func (mong *MongoStore) UpdateSubOffset(projectUUID string, name string, offset change := bson.M{"$set": bson.M{"offset": offset, "next_offset": 0, "pending_ack": ""}} err := c.Update(doc, change) if err != nil && err != mgo.ErrNotFound { - log.WithFields( - log.Fields{ - "type": "backend_log", - "backend_service": "mongo", - "backend_hosts": mong.Server, - }, - ).Fatal(err.Error()) + mong.logErrorAndCrash(ctx, "UpdateSubOffset", err) } } // HasUsers accepts a user array of usernames and returns the not found -func (mong *MongoStore) HasUsers(projectUUID string, users []string) (bool, []string) { +func (mong *MongoStore) HasUsers(ctx context.Context, projectUUID string, users []string) (bool, []string) { db := mong.Session.DB(mong.Database) var results []QUser var notFound []string @@ -596,13 +567,7 @@ func (mong *MongoStore) HasUsers(projectUUID string, users []string) (bool, []st err := c.Find(bson.M{"projects": bson.M{"$elemMatch": bson.M{"project_uuid": projectUUID}}, "name": bson.M{"$in": users}}).All(&results) if err != nil { - log.WithFields( - log.Fields{ - "type": "backend_log", - "backend_service": "mongo", - "backend_hosts": mong.Server, - }, - ).Fatal(err.Error()) + mong.logErrorAndCrash(ctx, "HasUsers", err) } // for each given username @@ -625,7 +590,7 @@ func (mong *MongoStore) HasUsers(projectUUID string, users []string) (bool, []st } // QueryACL queries topic or subscription for a list of authorized users -func (mong *MongoStore) QueryACL(projectUUID string, resource string, name string) (QAcl, error) { +func (mong *MongoStore) QueryACL(ctx context.Context, projectUUID string, resource string, name string) (QAcl, error) { db := mong.Session.DB(mong.Database) var results []QAcl @@ -639,13 +604,7 @@ func (mong *MongoStore) QueryACL(projectUUID string, resource string, name strin err := c.Find(bson.M{"project_uuid": projectUUID, "name": name}).All(&results) if err != nil { - log.WithFields( - log.Fields{ - "type": "backend_log", - "backend_service": "mongo", - "backend_hosts": mong.Server, - }, - ).Fatal(err.Error()) + mong.logErrorAndCrash(ctx, "QueryACL", err) } if len(results) > 0 { @@ -656,9 +615,9 @@ func (mong *MongoStore) QueryACL(projectUUID string, resource string, name strin } // QueryUsers queries user(s) information belonging to a project -func (mong *MongoStore) QueryUsers(projectUUID string, uuid string, name string) ([]QUser, error) { +func (mong *MongoStore) QueryUsers(ctx context.Context, projectUUID string, uuid string, name string) ([]QUser, error) { - // By default return all users + // By default, return all users query := bson.M{} // If project UUID is given return users that belong to the project if projectUUID != "" { @@ -684,20 +643,14 @@ func (mong *MongoStore) QueryUsers(projectUUID string, uuid string, name string) err := c.Find(query).All(&results) if err != nil { - log.WithFields( - log.Fields{ - "type": "backend_log", - "backend_service": "mongo", - "backend_hosts": mong.Server, - }, - ).Fatal(err.Error()) + mong.logErrorAndCrash(ctx, "QueryUsers", err) } return results, err } // PaginatedQueryUsers returns a page of users -func (mong *MongoStore) PaginatedQueryUsers(pageToken string, pageSize int64, projectUUID string) ([]QUser, int64, string, error) { +func (mong *MongoStore) PaginatedQueryUsers(ctx context.Context, pageToken string, pageSize int64, projectUUID string) ([]QUser, int64, string, error) { var qUsers []QUser var totalSize int64 @@ -732,24 +685,19 @@ func (mong *MongoStore) PaginatedQueryUsers(pageToken string, pageSize int64, pr // check the total of the users selected by the query not taking into account pagination if size, err = c.Find(query).Count(); err != nil { - log.WithFields( - log.Fields{ - "type": "backend_log", - "backend_service": "mongo", - "backend_hosts": mong.Server, - }, - ).Fatal(err.Error()) + mong.logErrorAndCrash(ctx, "PaginatedQueryUsers", err) } totalSize = int64(size) // now take into account if pagination is enabled and change the query accordingly - // first check if an pageToken is provided and whether or not is a valid bson ID + // first check if an pageToken is provided and whether is a valid bson ID if pageToken != "" { if ok = bson.IsObjectIdHex(pageToken); !ok { err = fmt.Errorf("Page token %v is not a valid bson ObjectId", pageToken) log.WithFields( log.Fields{ "type": "backend_log", + "trace_id": ctx.Value("trace_id"), "backend_service": "mongo", "page_token": pageToken, }, @@ -784,13 +732,7 @@ func (mong *MongoStore) PaginatedQueryUsers(pageToken string, pageSize int64, pr } if err = c.Find(query).Sort("-_id").Limit(int(limit)).All(&qUsers); err != nil { - log.WithFields( - log.Fields{ - "type": "backend_log", - "backend_service": "mongo", - "backend_hosts": mong.Server, - }, - ).Fatal(err.Error()) + mong.logErrorAndCrash(ctx, "PaginatedQueryUsers-2", err) } // if the amount of users that were found was equal to the limit, its a sign that there are users to populate the next page @@ -805,9 +747,9 @@ func (mong *MongoStore) PaginatedQueryUsers(pageToken string, pageSize int64, pr return qUsers, totalSize, nextPageToken, err } -//QuerySubsByTopic returns subscriptions of a specific topic -func (mong *MongoStore) QuerySubsByTopic(projectUUID, topic string) ([]QSub, error) { - // By default return all subs of a given project +// QuerySubsByTopic returns subscriptions of a specific topic +func (mong *MongoStore) QuerySubsByTopic(ctx context.Context, projectUUID, topic string) ([]QSub, error) { + // By default, return all subs of a given project query := bson.M{"project_uuid": projectUUID} // If topic is given return only the specific topic @@ -820,21 +762,15 @@ func (mong *MongoStore) QuerySubsByTopic(projectUUID, topic string) ([]QSub, err err := c.Find(query).All(&results) if err != nil { - log.WithFields( - log.Fields{ - "type": "backend_log", - "backend_service": "mongo", - "backend_hosts": mong.Server, - }, - ).Fatal(err.Error()) + mong.logErrorAndCrash(ctx, "QuerySubsByTopic", err) } return results, err } -//QuerySubsByACL returns subscriptions that a specific username has access to -func (mong *MongoStore) QuerySubsByACL(projectUUID, user string) ([]QSub, error) { - // By default return all subs of a given project +// QuerySubsByACL returns subscriptions that a specific username has access to +func (mong *MongoStore) QuerySubsByACL(ctx context.Context, projectUUID, user string) ([]QSub, error) { + // By default, return all subs of a given project query := bson.M{"project_uuid": projectUUID} // If name is given return only the specific topic @@ -847,21 +783,15 @@ func (mong *MongoStore) QuerySubsByACL(projectUUID, user string) ([]QSub, error) err := c.Find(query).All(&results) if err != nil { - log.WithFields( - log.Fields{ - "type": "backend_log", - "backend_service": "mongo", - "backend_hosts": mong.Server, - }, - ).Fatal(err.Error()) + mong.logErrorAndCrash(ctx, "QuerySubsByACL", err) } return results, err } -//QueryTopicsByACL returns topics that a specific username has access to -func (mong *MongoStore) QueryTopicsByACL(projectUUID, user string) ([]QTopic, error) { - // By default return all topics of a given project +// QueryTopicsByACL returns topics that a specific username has access to +func (mong *MongoStore) QueryTopicsByACL(ctx context.Context, projectUUID, user string) ([]QTopic, error) { + // By default, return all topics of a given project query := bson.M{"project_uuid": projectUUID} // If name is given return only the specific topic @@ -874,20 +804,14 @@ func (mong *MongoStore) QueryTopicsByACL(projectUUID, user string) ([]QTopic, er err := c.Find(query).All(&results) if err != nil { - log.WithFields( - log.Fields{ - "type": "backend_log", - "backend_service": "mongo", - "backend_hosts": mong.Server, - }, - ).Fatal(err.Error()) + mong.logErrorAndCrash(ctx, "QueryTopicsByACL", err) } return results, err } // QueryTopics Query Subscription info from store -func (mong *MongoStore) QueryTopics(projectUUID, userUUID, name, pageToken string, pageSize int64) ([]QTopic, int64, string, error) { +func (mong *MongoStore) QueryTopics(ctx context.Context, projectUUID, userUUID, name, pageToken string, pageSize int64) ([]QTopic, int64, string, error) { var err error var totalSize int64 @@ -920,6 +844,7 @@ func (mong *MongoStore) QueryTopics(projectUUID, userUUID, name, pageToken strin log.WithFields( log.Fields{ "type": "backend_log", + "trace_id": ctx.Value("trace_id"), "backend_service": "mongo", "page_token": pageToken, }, @@ -940,13 +865,7 @@ func (mong *MongoStore) QueryTopics(projectUUID, userUUID, name, pageToken strin c := db.C("topics") if err = c.Find(query).Sort("-_id").Limit(int(limit)).All(&qTopics); err != nil { - log.WithFields( - log.Fields{ - "type": "backend_log", - "backend_service": "mongo", - "backend_hosts": mong.Server, - }, - ).Fatal(err.Error()) + mong.logErrorAndCrash(ctx, "QueryTopics", err) } if name == "" { @@ -957,13 +876,8 @@ func (mong *MongoStore) QueryTopics(projectUUID, userUUID, name, pageToken strin } if size, err = c.Find(countQuery).Count(); err != nil { - log.WithFields( - log.Fields{ - "type": "backend_log", - "backend_service": "mongo", - "backend_hosts": mong.Server, - }, - ).Fatal(err.Error()) + mong.logErrorAndCrash(ctx, "QueryTopics-2", err) + } totalSize = int64(size) @@ -983,7 +897,7 @@ func (mong *MongoStore) QueryTopics(projectUUID, userUUID, name, pageToken strin } // UpdateTopicLatestPublish updates the topic's latest publish time -func (mong *MongoStore) UpdateTopicLatestPublish(projectUUID string, name string, date time.Time) error { +func (mong *MongoStore) UpdateTopicLatestPublish(ctx context.Context, projectUUID string, name string, date time.Time) error { db := mong.Session.DB(mong.Database) c := db.C("topics") @@ -1003,7 +917,7 @@ func (mong *MongoStore) UpdateTopicLatestPublish(projectUUID string, name string } // UpdateTopicPublishRate updates the topic's publishing rate -func (mong *MongoStore) UpdateTopicPublishRate(projectUUID string, name string, rate float64) error { +func (mong *MongoStore) UpdateTopicPublishRate(ctx context.Context, projectUUID string, name string, rate float64) error { db := mong.Session.DB(mong.Database) c := db.C("topics") @@ -1023,7 +937,7 @@ func (mong *MongoStore) UpdateTopicPublishRate(projectUUID string, name string, } // UpdateSubLatestConsume updates the subscription's latest consume time -func (mong *MongoStore) UpdateSubLatestConsume(projectUUID string, name string, date time.Time) error { +func (mong *MongoStore) UpdateSubLatestConsume(ctx context.Context, projectUUID string, name string, date time.Time) error { db := mong.Session.DB(mong.Database) c := db.C("subscriptions") @@ -1043,7 +957,7 @@ func (mong *MongoStore) UpdateSubLatestConsume(projectUUID string, name string, } // UpdateSubConsumeRate updates the subscription's consume rate -func (mong *MongoStore) UpdateSubConsumeRate(projectUUID string, name string, rate float64) error { +func (mong *MongoStore) UpdateSubConsumeRate(ctx context.Context, projectUUID string, name string, rate float64) error { db := mong.Session.DB(mong.Database) c := db.C("subscriptions") @@ -1063,7 +977,7 @@ func (mong *MongoStore) UpdateSubConsumeRate(projectUUID string, name string, ra } // QueryDailyTopicMsgCount returns results regarding the number of messages published to a topic -func (mong *MongoStore) QueryDailyTopicMsgCount(projectUUID string, topicName string, date time.Time) ([]QDailyTopicMsgCount, error) { +func (mong *MongoStore) QueryDailyTopicMsgCount(ctx context.Context, projectUUID string, topicName string, date time.Time) ([]QDailyTopicMsgCount, error) { var err error var qDailyTopicMsgCount []QDailyTopicMsgCount @@ -1089,20 +1003,14 @@ func (mong *MongoStore) QueryDailyTopicMsgCount(projectUUID string, topicName st err = c.Find(query).Sort("-date").Limit(30).All(&qDailyTopicMsgCount) if err != nil { - log.WithFields( - log.Fields{ - "type": "backend_log", - "backend_service": "mongo", - "backend_hosts": mong.Server, - }, - ).Fatal(err.Error()) + mong.logErrorAndCrash(ctx, "QueryDailyTopicMsgCount", err) } return qDailyTopicMsgCount, err } // IncrementTopicMsgNum increments the number of messages published in a topic -func (mong *MongoStore) IncrementTopicMsgNum(projectUUID string, name string, num int64) error { +func (mong *MongoStore) IncrementTopicMsgNum(ctx context.Context, projectUUID string, name string, num int64) error { db := mong.Session.DB(mong.Database) c := db.C("topics") @@ -1117,7 +1025,7 @@ func (mong *MongoStore) IncrementTopicMsgNum(projectUUID string, name string, nu } // IncrementDailyTopicMsgCount increments the daily count of published messages to a specific topic -func (mong *MongoStore) IncrementDailyTopicMsgCount(projectUUID string, topicName string, num int64, date time.Time) error { +func (mong *MongoStore) IncrementDailyTopicMsgCount(ctx context.Context, projectUUID string, topicName string, num int64, date time.Time) error { db := mong.Session.DB(mong.Database) c := db.C("daily_topic_msg_count") @@ -1131,8 +1039,8 @@ func (mong *MongoStore) IncrementDailyTopicMsgCount(projectUUID string, topicNam } -//IncrementTopicBytes increases the total number of bytes published in a topic -func (mong *MongoStore) IncrementTopicBytes(projectUUID string, name string, totalBytes int64) error { +// IncrementTopicBytes increases the total number of bytes published in a topic +func (mong *MongoStore) IncrementTopicBytes(ctx context.Context, projectUUID string, name string, totalBytes int64) error { db := mong.Session.DB(mong.Database) c := db.C("topics") @@ -1145,7 +1053,7 @@ func (mong *MongoStore) IncrementTopicBytes(projectUUID string, name string, tot } // IncrementSubMsgNum increments the number of messages pulled in a subscription -func (mong *MongoStore) IncrementSubMsgNum(projectUUID string, name string, num int64) error { +func (mong *MongoStore) IncrementSubMsgNum(ctx context.Context, projectUUID string, name string, num int64) error { db := mong.Session.DB(mong.Database) c := db.C("subscriptions") @@ -1159,8 +1067,8 @@ func (mong *MongoStore) IncrementSubMsgNum(projectUUID string, name string, num } -//IncrementSubBytes increases the total number of bytes consumed from a subscripion -func (mong *MongoStore) IncrementSubBytes(projectUUID string, name string, totalBytes int64) error { +// IncrementSubBytes increases the total number of bytes consumed from a subscription +func (mong *MongoStore) IncrementSubBytes(ctx context.Context, projectUUID string, name string, totalBytes int64) error { db := mong.Session.DB(mong.Database) c := db.C("subscriptions") @@ -1172,21 +1080,15 @@ func (mong *MongoStore) IncrementSubBytes(projectUUID string, name string, total return err } -//HasResourceRoles returns the roles of a user in a project -func (mong *MongoStore) HasResourceRoles(resource string, roles []string) bool { +// HasResourceRoles returns the roles of a user in a project +func (mong *MongoStore) HasResourceRoles(ctx context.Context, resource string, roles []string) bool { db := mong.Session.DB(mong.Database) c := db.C("roles") var results []QRole err := c.Find(bson.M{"resource": resource, "roles": bson.M{"$in": roles}}).All(&results) if err != nil { - log.WithFields( - log.Fields{ - "type": "backend_log", - "backend_service": "mongo", - "backend_hosts": mong.Server, - }, - ).Fatal(err.Error()) + mong.logErrorAndCrash(ctx, "HasResourceRoles", err) } if len(results) > 0 { @@ -1197,27 +1099,21 @@ func (mong *MongoStore) HasResourceRoles(resource string, roles []string) bool { } -//GetAllRoles returns a list of all available roles -func (mong *MongoStore) GetAllRoles() []string { +// GetAllRoles returns a list of all available roles +func (mong *MongoStore) GetAllRoles(ctx context.Context) []string { db := mong.Session.DB(mong.Database) c := db.C("roles") var results []string err := c.Find(nil).Distinct("roles", &results) if err != nil { - log.WithFields( - log.Fields{ - "type": "backend_log", - "backend_service": "mongo", - "backend_hosts": mong.Server, - }, - ).Fatal(err.Error()) + mong.logErrorAndCrash(ctx, "GetAllRoles", err) } return results } -//GetOpMetrics returns the operational metrics from datastore -func (mong *MongoStore) GetOpMetrics() []QopMetric { +// GetOpMetrics returns the operational metrics from datastore +func (mong *MongoStore) GetOpMetrics(ctx context.Context) []QopMetric { db := mong.Session.DB(mong.Database) c := db.C("op_metrics") @@ -1226,21 +1122,15 @@ func (mong *MongoStore) GetOpMetrics() []QopMetric { err := c.Find(bson.M{}).All(&results) if err != nil { - log.WithFields( - log.Fields{ - "type": "backend_log", - "backend_service": "mongo", - "backend_hosts": mong.Server, - }, - ).Fatal(err.Error()) + mong.logErrorAndCrash(ctx, "GetOpMetrics", err) } return results } -//GetUserRoles returns the roles of a user in a project -func (mong *MongoStore) GetUserRoles(projectUUID string, token string) ([]string, string) { +// GetUserRoles returns the roles of a user in a project +func (mong *MongoStore) GetUserRoles(ctx context.Context, projectUUID string, token string) ([]string, string) { db := mong.Session.DB(mong.Database) c := db.C("users") @@ -1249,13 +1139,7 @@ func (mong *MongoStore) GetUserRoles(projectUUID string, token string) ([]string err := c.Find(bson.M{"token": token}).All(&results) if err != nil { - log.WithFields( - log.Fields{ - "type": "backend_log", - "backend_service": "mongo", - "backend_hosts": mong.Server, - }, - ).Fatal(err.Error()) + mong.logErrorAndCrash(ctx, "GetUserRoles", err) } if len(results) == 0 { @@ -1266,6 +1150,7 @@ func (mong *MongoStore) GetUserRoles(projectUUID string, token string) ([]string log.WithFields( log.Fields{ "type": "backend_log", + "trace_id": ctx.Value("trace_id"), "token": token, "backend_service": "mongo", "backend_hosts": mong.Server, @@ -1278,8 +1163,8 @@ func (mong *MongoStore) GetUserRoles(projectUUID string, token string) ([]string } -//GetUserFromToken returns user information from a specific token -func (mong *MongoStore) GetUserFromToken(token string) (QUser, error) { +// GetUserFromToken returns user information from a specific token +func (mong *MongoStore) GetUserFromToken(ctx context.Context, token string) (QUser, error) { db := mong.Session.DB(mong.Database) c := db.C("users") @@ -1288,13 +1173,8 @@ func (mong *MongoStore) GetUserFromToken(token string) (QUser, error) { err := c.Find(bson.M{"token": token}).All(&results) if err != nil { - log.WithFields( - log.Fields{ - "type": "backend_log", - "backend_service": "mongo", - "backend_hosts": mong.Server, - }, - ).Fatal(err.Error()) + mong.logErrorAndCrash(ctx, "GetUserFromToken", err) + } if len(results) == 0 { @@ -1305,6 +1185,7 @@ func (mong *MongoStore) GetUserFromToken(token string) (QUser, error) { log.WithFields( log.Fields{ "type": "backend_log", + "trace_id": ctx.Value("trace_id"), "token": token, "backend_service": "mongo", "backend_hosts": mong.Server, @@ -1318,20 +1199,15 @@ func (mong *MongoStore) GetUserFromToken(token string) (QUser, error) { } // QueryOneSub queries and returns specific sub of project -func (mong *MongoStore) QueryOneSub(projectUUID string, name string) (QSub, error) { +func (mong *MongoStore) QueryOneSub(ctx context.Context, projectUUID string, name string) (QSub, error) { db := mong.Session.DB(mong.Database) c := db.C("subscriptions") var results []QSub err := c.Find(bson.M{"project_uuid": projectUUID, "name": name}).All(&results) if err != nil { - log.WithFields( - log.Fields{ - "type": "backend_log", - "backend_service": "mongo", - "backend_hosts": mong.Server, - }, - ).Fatal(err.Error()) + mong.logErrorAndCrash(ctx, "QueryOneSub", err) + } if len(results) > 0 { @@ -1342,20 +1218,15 @@ func (mong *MongoStore) QueryOneSub(projectUUID string, name string) (QSub, erro } // HasProject Returns true if project exists -func (mong *MongoStore) HasProject(name string) bool { +func (mong *MongoStore) HasProject(ctx context.Context, name string) bool { db := mong.Session.DB(mong.Database) c := db.C("projects") var results []QProject err := c.Find(bson.M{"name": name}).All(&results) if err != nil { - log.WithFields( - log.Fields{ - "type": "backend_log", - "backend_service": "mongo", - "backend_hosts": mong.Server, - }, - ).Fatal(err.Error()) + mong.logErrorAndCrash(ctx, "HasProject", err) + } if len(results) > 0 { @@ -1365,7 +1236,7 @@ func (mong *MongoStore) HasProject(name string) bool { } // InsertTopic inserts a topic to the store -func (mong *MongoStore) InsertTopic(projectUUID string, name string, schemaUUID string, createdOn time.Time) error { +func (mong *MongoStore) InsertTopic(ctx context.Context, projectUUID string, name string, schemaUUID string, createdOn time.Time) error { topic := QTopic{ ProjectUUID: projectUUID, @@ -1379,30 +1250,25 @@ func (mong *MongoStore) InsertTopic(projectUUID string, name string, schemaUUID ACL: []string{}, } - return mong.InsertResource("topics", topic) + return mong.InsertResource(ctx, "topics", topic) } // LinkTopicSchema links the topic with a schema -func (mong *MongoStore) LinkTopicSchema(projectUUID, name, schemaUUID string) error { +func (mong *MongoStore) LinkTopicSchema(ctx context.Context, projectUUID, name, schemaUUID string) error { db := mong.Session.DB(mong.Database) c := db.C("topics") err := c.Update(bson.M{"project_uuid": projectUUID, "name": name}, bson.M{"$set": bson.M{"schema_uuid": schemaUUID}}) if err != nil { - log.WithFields( - log.Fields{ - "type": "backend_log", - "backend_service": "mongo", - "backend_hosts": mong.Server, - }, - ).Fatal(err.Error()) + mong.logErrorAndCrash(ctx, "LinkTopicSchema", err) + } return nil } // InsertOpMetric inserts an operational metric -func (mong *MongoStore) InsertOpMetric(hostname string, cpu float64, mem float64) error { +func (mong *MongoStore) InsertOpMetric(ctx context.Context, hostname string, cpu float64, mem float64) error { opMetric := QopMetric{Hostname: hostname, CPU: cpu, MEM: mem} db := mong.Session.DB(mong.Database) c := db.C("op_metrics") @@ -1411,20 +1277,15 @@ func (mong *MongoStore) InsertOpMetric(hostname string, cpu float64, mem float64 _, err := c.UpsertId(opMetric.Hostname, upsertdata) if err != nil { - log.WithFields( - log.Fields{ - "type": "backend_log", - "backend_service": "mongo", - "backend_hosts": mong.Server, - }, - ).Error(err.Error()) + mong.logErrorAndCrash(ctx, "InsertOpMetric", err) + } return err } // InsertUser inserts a new user to the store -func (mong *MongoStore) InsertUser(uuid string, projects []QProjectRoles, name string, fname string, lname string, org string, desc string, token string, email string, serviceRoles []string, createdOn time.Time, modifiedOn time.Time, createdBy string) error { +func (mong *MongoStore) InsertUser(ctx context.Context, uuid string, projects []QProjectRoles, name string, fname string, lname string, org string, desc string, token string, email string, serviceRoles []string, createdOn time.Time, modifiedOn time.Time, createdBy string) error { user := QUser{ UUID: uuid, Name: name, @@ -1440,17 +1301,17 @@ func (mong *MongoStore) InsertUser(uuid string, projects []QProjectRoles, name s ModifiedOn: modifiedOn, CreatedBy: createdBy, } - return mong.InsertResource("users", user) + return mong.InsertResource(ctx, "users", user) } // InsertProject inserts a project to the store -func (mong *MongoStore) InsertProject(uuid string, name string, createdOn time.Time, modifiedOn time.Time, createdBy string, description string) error { +func (mong *MongoStore) InsertProject(ctx context.Context, uuid string, name string, createdOn time.Time, modifiedOn time.Time, createdBy string, description string) error { project := QProject{UUID: uuid, Name: name, CreatedOn: createdOn, ModifiedOn: modifiedOn, CreatedBy: createdBy, Description: description} - return mong.InsertResource("projects", project) + return mong.InsertResource(ctx, "projects", project) } // InsertSub inserts a subscription to the store -func (mong *MongoStore) InsertSub(projectUUID string, name string, topic string, +func (mong *MongoStore) InsertSub(ctx context.Context, projectUUID string, name string, topic string, offset int64, ack int, pushCfg QPushConfig, createdOn time.Time) error { sub := QSub{ ProjectUUID: projectUUID, @@ -1478,28 +1339,28 @@ func (mong *MongoStore) InsertSub(projectUUID string, name string, topic string, CreatedOn: createdOn, ACL: []string{}, } - return mong.InsertResource("subscriptions", sub) + return mong.InsertResource(ctx, "subscriptions", sub) } // RemoveProjectTopics removes all topics related to a project UUID -func (mong *MongoStore) RemoveProjectTopics(projectUUID string) error { +func (mong *MongoStore) RemoveProjectTopics(ctx context.Context, projectUUID string) error { topicMatch := bson.M{"project_uuid": projectUUID} - return mong.RemoveAll("topics", topicMatch) + return mong.RemoveAll(ctx, "topics", topicMatch) } // RemoveProjectSubs removes all subscriptions related to a project UUID -func (mong *MongoStore) RemoveProjectSubs(projectUUID string) error { +func (mong *MongoStore) RemoveProjectSubs(ctx context.Context, projectUUID string) error { subMatch := bson.M{"project_uuid": projectUUID} - return mong.RemoveAll("subscriptions", subMatch) + return mong.RemoveAll(ctx, "subscriptions", subMatch) } -// RemoveProjectDailyMessageCount removes all message counts related to a project UUID -func (mong *MongoStore) RemoveProjectDailyMessageCounters(projectUUID string) error { - return mong.RemoveAll("daily_topic_msg_count", bson.M{"project_uuid": projectUUID}) +// RemoveProjectDailyMessageCounters removes all message counts related to a project UUID +func (mong *MongoStore) RemoveProjectDailyMessageCounters(ctx context.Context, projectUUID string) error { + return mong.RemoveAll(ctx, "daily_topic_msg_count", bson.M{"project_uuid": projectUUID}) } // QueryTotalMessagesPerProject returns the total amount of messages per project for the given time window -func (mong *MongoStore) QueryTotalMessagesPerProject(projectUUIDs []string, startDate time.Time, endDate time.Time) ([]QProjectMessageCount, error) { +func (mong *MongoStore) QueryTotalMessagesPerProject(ctx context.Context, projectUUIDs []string, startDate time.Time, endDate time.Time) ([]QProjectMessageCount, error) { var err error var qdp []QProjectMessageCount @@ -1569,20 +1430,14 @@ func (mong *MongoStore) QueryTotalMessagesPerProject(projectUUIDs []string, star } if err = c.Pipe(query).All(&qdp); err != nil { - log.WithFields( - log.Fields{ - "type": "backend_log", - "backend_service": "mongo", - "backend_hosts": mong.Server, - }, - ).Fatal(err.Error()) + mong.logErrorAndCrash(ctx, "QueryTotalMessagesPerProject", err) } return qdp, err } // QueryDailyProjectMsgCount queries the total messages per day for a given project -func (mong *MongoStore) QueryDailyProjectMsgCount(projectUUID string) ([]QDailyProjectMsgCount, error) { +func (mong *MongoStore) QueryDailyProjectMsgCount(ctx context.Context, projectUUID string) ([]QDailyProjectMsgCount, error) { var err error var qdp []QDailyProjectMsgCount @@ -1625,13 +1480,7 @@ func (mong *MongoStore) QueryDailyProjectMsgCount(projectUUID string) ([]QDailyP fmt.Println(query) fmt.Println("2") if err = c.Pipe(query).All(&qdp); err != nil { - log.WithFields( - log.Fields{ - "type": "backend_log", - "backend_service": "mongo", - "backend_hosts": mong.Server, - }, - ).Fatal(err.Error()) + mong.logErrorAndCrash(ctx, "QueryDailyProjectMsgCount", err) } return qdp, err @@ -1639,31 +1488,31 @@ func (mong *MongoStore) QueryDailyProjectMsgCount(projectUUID string) ([]QDailyP } // RemoveProject removes a project from the store -func (mong *MongoStore) RemoveProject(uuid string) error { +func (mong *MongoStore) RemoveProject(ctx context.Context, uuid string) error { project := bson.M{"uuid": uuid} - return mong.RemoveResource("projects", project) + return mong.RemoveResource(ctx, "projects", project) } // RemoveTopic removes a topic from the store -func (mong *MongoStore) RemoveTopic(projectUUID string, name string) error { +func (mong *MongoStore) RemoveTopic(ctx context.Context, projectUUID string, name string) error { topic := bson.M{"project_uuid": projectUUID, "name": name} - return mong.RemoveResource("topics", topic) + return mong.RemoveResource(ctx, "topics", topic) } // RemoveUser removes a user entry from the store -func (mong *MongoStore) RemoveUser(uuid string) error { +func (mong *MongoStore) RemoveUser(ctx context.Context, uuid string) error { user := bson.M{"uuid": uuid} - return mong.RemoveResource("users", user) + return mong.RemoveResource(ctx, "users", user) } // RemoveSub removes a subscription from the store -func (mong *MongoStore) RemoveSub(projectUUID string, name string) error { +func (mong *MongoStore) RemoveSub(ctx context.Context, projectUUID string, name string) error { sub := bson.M{"project_uuid": projectUUID, "name": name} - return mong.RemoveResource("subscriptions", sub) + return mong.RemoveResource(ctx, "subscriptions", sub) } // ExistsInACL checks if a user is part of a topic's or sub's acl -func (mong *MongoStore) ExistsInACL(projectUUID string, resource string, resourceName string, userUUID string) error { +func (mong *MongoStore) ExistsInACL(ctx context.Context, projectUUID string, resource string, resourceName string, userUUID string) error { db := mong.Session.DB(mong.Database) @@ -1686,7 +1535,7 @@ func (mong *MongoStore) ExistsInACL(projectUUID string, resource string, resourc } // ModACL modifies the push configuration -func (mong *MongoStore) ModACL(projectUUID string, resource string, name string, acl []string) error { +func (mong *MongoStore) ModACL(ctx context.Context, projectUUID string, resource string, name string, acl []string) error { db := mong.Session.DB(mong.Database) if resource != "topics" && resource != "subscriptions" { @@ -1700,7 +1549,7 @@ func (mong *MongoStore) ModACL(projectUUID string, resource string, name string, } // AppendToACL adds additional users to an existing ACL -func (mong *MongoStore) AppendToACL(projectUUID string, resource string, name string, acl []string) error { +func (mong *MongoStore) AppendToACL(ctx context.Context, projectUUID string, resource string, name string, acl []string) error { db := mong.Session.DB(mong.Database) @@ -1725,7 +1574,7 @@ func (mong *MongoStore) AppendToACL(projectUUID string, resource string, name st } // RemoveFromACL removes users for a given ACL -func (mong *MongoStore) RemoveFromACL(projectUUID string, resource string, name string, acl []string) error { +func (mong *MongoStore) RemoveFromACL(ctx context.Context, projectUUID string, resource string, name string, acl []string) error { db := mong.Session.DB(mong.Database) @@ -1750,8 +1599,7 @@ func (mong *MongoStore) RemoveFromACL(projectUUID string, resource string, name } // ModAck modifies the subscription's ack timeout field in mongodb -func (mong *MongoStore) ModAck(projectUUID string, name string, ack int) error { - log.Info("Modifying Ack Deadline", ack) +func (mong *MongoStore) ModAck(ctx context.Context, projectUUID string, name string, ack int) error { db := mong.Session.DB(mong.Database) c := db.C("subscriptions") err := c.Update(bson.M{"project_uuid": projectUUID, "name": name}, bson.M{"$set": bson.M{"ack": ack}}) @@ -1759,7 +1607,7 @@ func (mong *MongoStore) ModAck(projectUUID string, name string, ack int) error { } // ModSubPush modifies the push configuration -func (mong *MongoStore) ModSubPush(projectUUID string, name string, pushCfg QPushConfig) error { +func (mong *MongoStore) ModSubPush(ctx context.Context, projectUUID string, name string, pushCfg QPushConfig) error { db := mong.Session.DB(mong.Database) c := db.C("subscriptions") @@ -1787,7 +1635,7 @@ func (mong *MongoStore) ModSubPush(projectUUID string, name string, pushCfg QPus } // InsertResource inserts a new topic object to the datastore -func (mong *MongoStore) InsertResource(col string, res interface{}) error { +func (mong *MongoStore) InsertResource(ctx context.Context, col string, res interface{}) error { db := mong.Session.DB(mong.Database) c := db.C(col) @@ -1801,7 +1649,7 @@ func (mong *MongoStore) InsertResource(col string, res interface{}) error { } // RemoveAll removes all occurrences matched with a resource from the store -func (mong *MongoStore) RemoveAll(col string, res interface{}) error { +func (mong *MongoStore) RemoveAll(ctx context.Context, col string, res interface{}) error { db := mong.Session.DB(mong.Database) c := db.C(col) @@ -1811,7 +1659,7 @@ func (mong *MongoStore) RemoveAll(col string, res interface{}) error { } // RemoveResource removes a resource from the store -func (mong *MongoStore) RemoveResource(col string, res interface{}) error { +func (mong *MongoStore) RemoveResource(ctx context.Context, col string, res interface{}) error { db := mong.Session.DB(mong.Database) c := db.C(col) @@ -1822,7 +1670,7 @@ func (mong *MongoStore) RemoveResource(col string, res interface{}) error { } // QuerySubs Query Subscription info from store -func (mong *MongoStore) QuerySubs(projectUUID, userUUID, name, pageToken string, pageSize int64) ([]QSub, int64, string, error) { +func (mong *MongoStore) QuerySubs(ctx context.Context, projectUUID, userUUID, name, pageToken string, pageSize int64) ([]QSub, int64, string, error) { var err error var totalSize int64 @@ -1832,7 +1680,7 @@ func (mong *MongoStore) QuerySubs(projectUUID, userUUID, name, pageToken string, var ok bool var size int - // By default return all subs of a given project + // By default, return all subs of a given project query := bson.M{"project_uuid": projectUUID} // find all the subscriptions for a specific user @@ -1848,13 +1696,14 @@ func (mong *MongoStore) QuerySubs(projectUUID, userUUID, name, pageToken string, } - // first check if an pageToken is provided and whether or not is a valid bson ID + // first check if an pageToken is provided and whether is a valid bson ID if pageToken != "" { if ok = bson.IsObjectIdHex(pageToken); !ok { err = fmt.Errorf("Page token %v is not a valid bson ObjectId", pageToken) log.WithFields( log.Fields{ "type": "backend_log", + "trace_id": ctx.Value("trace_id"), "backend_service": "mongo", "page_token": pageToken, }, @@ -1875,13 +1724,7 @@ func (mong *MongoStore) QuerySubs(projectUUID, userUUID, name, pageToken string, c := db.C("subscriptions") if err = c.Find(query).Sort("-_id").Limit(int(limit)).All(&qSubs); err != nil { - log.WithFields( - log.Fields{ - "type": "backend_log", - "backend_service": "mongo", - "backend_hosts": mong.Server, - }, - ).Fatal(err.Error()) + mong.logErrorAndCrash(ctx, "QuerySubs", err) } if name == "" { @@ -1892,13 +1735,8 @@ func (mong *MongoStore) QuerySubs(projectUUID, userUUID, name, pageToken string, } if size, err = c.Find(countQuery).Count(); err != nil { - log.WithFields( - log.Fields{ - "type": "backend_log", - "backend_service": "mongo", - "backend_hosts": mong.Server, - }, - ).Fatal(err.Error()) + mong.logErrorAndCrash(ctx, "QuerySubs-2", err) + } totalSize = int64(size) @@ -1918,26 +1756,21 @@ func (mong *MongoStore) QuerySubs(projectUUID, userUUID, name, pageToken string, } // QueryPushSubs retrieves subscriptions that have a push_endpoint defined -func (mong *MongoStore) QueryPushSubs() []QSub { +func (mong *MongoStore) QueryPushSubs(ctx context.Context) []QSub { db := mong.Session.DB(mong.Database) c := db.C("subscriptions") var results []QSub err := c.Find(bson.M{"push_endpoint": bson.M{"$ne": ""}}).All(&results) if err != nil { - log.WithFields( - log.Fields{ - "type": "backend_log", - "backend_service": "mongo", - "backend_hosts": mong.Server, - }, - ).Fatal(err.Error()) + mong.logErrorAndCrash(ctx, "QueryPushSubs", err) + } return results } -func (mong *MongoStore) InsertSchema(projectUUID, schemaUUID, name, schemaType, rawSchemaString string) error { +func (mong *MongoStore) InsertSchema(ctx context.Context, projectUUID, schemaUUID, name, schemaType, rawSchemaString string) error { sub := QSchema{ ProjectUUID: projectUUID, UUID: schemaUUID, @@ -1945,10 +1778,10 @@ func (mong *MongoStore) InsertSchema(projectUUID, schemaUUID, name, schemaType, Type: schemaType, RawSchema: rawSchemaString, } - return mong.InsertResource("schemas", sub) + return mong.InsertResource(ctx, "schemas", sub) } -func (mong *MongoStore) QuerySchemas(projectUUID, schemaUUID, name string) ([]QSchema, error) { +func (mong *MongoStore) QuerySchemas(ctx context.Context, projectUUID, schemaUUID, name string) ([]QSchema, error) { db := mong.Session.DB(mong.Database) c := db.C("schemas") @@ -1967,20 +1800,15 @@ func (mong *MongoStore) QuerySchemas(projectUUID, schemaUUID, name string) ([]QS err := c.Find(query).All(&results) if err != nil { - log.WithFields( - log.Fields{ - "type": "backend_log", - "backend_service": "mongo", - "backend_hosts": mong.Server, - }, - ).Fatal(err.Error()) + mong.logErrorAndCrash(ctx, "QuerySchemas", err) + } return results, nil } // UpdateSchema updates the fields of a schema -func (mong *MongoStore) UpdateSchema(schemaUUID, name, schemaType, rawSchemaString string) error { +func (mong *MongoStore) UpdateSchema(ctx context.Context, schemaUUID, name, schemaType, rawSchemaString string) error { db := mong.Session.DB(mong.Database) c := db.C("schemas") @@ -2008,7 +1836,7 @@ func (mong *MongoStore) UpdateSchema(schemaUUID, name, schemaType, rawSchemaStri // DeleteSchema removes the schema from the store // It also clears all the respective topics from the schema_uuid of the deleted schema -func (mong *MongoStore) DeleteSchema(schemaUUID string) error { +func (mong *MongoStore) DeleteSchema(ctx context.Context, schemaUUID string) error { db := mong.Session.DB(mong.Database) c := db.C("schemas") @@ -2018,13 +1846,8 @@ func (mong *MongoStore) DeleteSchema(schemaUUID string) error { err := c.Remove(selector) if err != nil { - log.WithFields( - log.Fields{ - "type": "backend_log", - "backend_service": "mongo", - "backend_hosts": mong.Server, - }, - ).Fatal(err.Error()) + mong.logErrorAndCrash(ctx, "DeleteSchema", err) + } topics := db.C("topics") @@ -2035,16 +1858,10 @@ func (mong *MongoStore) DeleteSchema(schemaUUID string) error { "schema_uuid": "", }, } - topics.UpdateAll(topicSelector, change) - + _, err = topics.UpdateAll(topicSelector, change) if err != nil { - log.WithFields( - log.Fields{ - "type": "backend_log", - "backend_service": "mongo", - "backend_hosts": mong.Server, - }, - ).Fatal(err.Error()) + mong.logErrorAndCrash(ctx, "DeleteSchema-2", err) + } return nil diff --git a/stores/store.go b/stores/store.go index c4f190c9..a97b48d1 100644 --- a/stores/store.go +++ b/stores/store.go @@ -1,79 +1,80 @@ package stores import ( + "context" "time" ) // Store encapsulates the generic store interface type Store interface { Initialize() - QuerySubsByTopic(projectUUID, topic string) ([]QSub, error) - QueryTopicsByACL(projectUUID, user string) ([]QTopic, error) - QuerySubsByACL(projectUUID, user string) ([]QSub, error) - QuerySubs(projectUUID string, userUUID string, name string, pageToken string, pageSize int64) ([]QSub, int64, string, error) - QueryTopics(projectUUID string, userUUID string, name string, pageToken string, pageSize int64) ([]QTopic, int64, string, error) - QueryDailyTopicMsgCount(projectUUID string, name string, date time.Time) ([]QDailyTopicMsgCount, error) - UpdateTopicLatestPublish(projectUUID string, name string, date time.Time) error - UpdateTopicPublishRate(projectUUID string, name string, rate float64) error - UpdateSubLatestConsume(projectUUID string, name string, date time.Time) error - UpdateSubConsumeRate(projectUUID string, name string, rate float64) error - RemoveTopic(projectUUID string, name string) error - RemoveSub(projectUUID string, name string) error - PaginatedQueryUsers(pageToken string, pageSize int64, projectUUID string) ([]QUser, int64, string, error) - QueryUsers(projectUUID string, uuid string, name string) ([]QUser, error) - UpdateUser(uuid, fname, lname, org, desc string, projects []QProjectRoles, name string, email string, serviceRoles []string, modifiedOn time.Time) error - AppendToUserProjects(userUUID string, projectUUID string, pRoles ...string) error - UpdateUserToken(uuid string, token string) error - RemoveUser(uuid string) error - QueryProjects(uuid string, name string) ([]QProject, error) - UpdateProject(projectUUID string, name string, description string, modifiedOn time.Time) error - RemoveProject(uuid string) error - RemoveProjectTopics(projectUUID string) error - RemoveProjectSubs(projectUUID string) error - RemoveProjectDailyMessageCounters(projectUUID string) error - QueryDailyProjectMsgCount(projectUUID string) ([]QDailyProjectMsgCount, error) - QueryTotalMessagesPerProject(projectUUIDs []string, startDate time.Time, endDate time.Time) ([]QProjectMessageCount, error) - RegisterUser(uuid, name, firstName, lastName, email, org, desc, registeredAt, atkn, status string) error - DeleteRegistration(uuid string) error - QueryRegistrations(regUUID, status, activationToken, name, email, org string) ([]QUserRegistration, error) - UpdateRegistration(regUUID, status, declineComment, modifiedBy, modifiedAt string) error - InsertUser(uuid string, projects []QProjectRoles, name string, firstName string, lastName string, org string, desc string, token string, email string, serviceRoles []string, createdOn time.Time, modifiedOn time.Time, createdBy string) error - InsertProject(uuid string, name string, createdOn time.Time, modifiedOn time.Time, createdBy string, description string) error - InsertOpMetric(hostname string, cpu float64, mem float64) error - InsertTopic(projectUUID string, name string, schemaUUID string, createdOn time.Time) error - LinkTopicSchema(projectUUID, name, schemaUUID string) error - IncrementTopicMsgNum(projectUUID string, name string, num int64) error - IncrementDailyTopicMsgCount(projectUUID string, topicName string, num int64, date time.Time) error - IncrementTopicBytes(projectUUID string, name string, totalBytes int64) error - IncrementSubBytes(projectUUID string, name string, totalBytes int64) error - IncrementSubMsgNum(projectUUID string, name string, num int64) error - InsertSub(projectUUID string, name string, topic string, offest int64, ack int, pushCfg QPushConfig, createdOn time.Time) error - HasProject(name string) bool - HasUsers(projectUUID string, users []string) (bool, []string) - QueryOneSub(projectUUID string, name string) (QSub, error) - QueryPushSubs() []QSub - HasResourceRoles(resource string, roles []string) bool - GetOpMetrics() []QopMetric - GetUserRoles(projectUUID string, token string) ([]string, string) - GetUserFromToken(token string) (QUser, error) - UpdateSubOffset(projectUUID string, name string, offset int64) - UpdateSubPull(projectUUID string, name string, offset int64, ts string) error - UpdateSubOffsetAck(projectUUID string, name string, offset int64, ts string) error - ModSubPush(projectUUID string, name string, pushCfg QPushConfig) error - QueryACL(projectUUID string, resource string, name string) (QAcl, error) - ExistsInACL(projectUUID string, resource string, resourceName string, userUUID string) error - ModACL(projectUUID string, resource string, name string, acl []string) error - AppendToACL(projectUUID string, resource string, name string, acl []string) error - RemoveFromACL(projectUUID string, resource string, name string, acl []string) error - ModAck(projectUUID string, name string, ack int) error - GetAllRoles() []string - InsertSchema(projectUUID, schemaUUID, name, schemaType, rawSchemaString string) error - QuerySchemas(projectUUID, schemaUUID, name string) ([]QSchema, error) - UpdateSchema(schemaUUID, name, schemaType, rawSchemaString string) error - DeleteSchema(schemaUUID string) error - UsersCount(startDate, endDate time.Time, projectUUIDs []string) (map[string]int64, error) - TopicsCount(startDate, endDate time.Time, projectUUIDs []string) (map[string]int64, error) - SubscriptionsCount(startDate, endDate time.Time, projectUUIDs []string) (map[string]int64, error) + QuerySubsByTopic(ctx context.Context, projectUUID, topic string) ([]QSub, error) + QueryTopicsByACL(ctx context.Context, projectUUID, user string) ([]QTopic, error) + QuerySubsByACL(ctx context.Context, projectUUID, user string) ([]QSub, error) + QuerySubs(ctx context.Context, projectUUID string, userUUID string, name string, pageToken string, pageSize int64) ([]QSub, int64, string, error) + QueryTopics(ctx context.Context, projectUUID string, userUUID string, name string, pageToken string, pageSize int64) ([]QTopic, int64, string, error) + QueryDailyTopicMsgCount(ctx context.Context, projectUUID string, name string, date time.Time) ([]QDailyTopicMsgCount, error) + UpdateTopicLatestPublish(ctx context.Context, projectUUID string, name string, date time.Time) error + UpdateTopicPublishRate(ctx context.Context, projectUUID string, name string, rate float64) error + UpdateSubLatestConsume(ctx context.Context, projectUUID string, name string, date time.Time) error + UpdateSubConsumeRate(ctx context.Context, projectUUID string, name string, rate float64) error + RemoveTopic(ctx context.Context, projectUUID string, name string) error + RemoveSub(ctx context.Context, projectUUID string, name string) error + PaginatedQueryUsers(ctx context.Context, pageToken string, pageSize int64, projectUUID string) ([]QUser, int64, string, error) + QueryUsers(ctx context.Context, projectUUID string, uuid string, name string) ([]QUser, error) + UpdateUser(ctx context.Context, uuid, fname, lname, org, desc string, projects []QProjectRoles, name string, email string, serviceRoles []string, modifiedOn time.Time) error + AppendToUserProjects(ctx context.Context, userUUID string, projectUUID string, pRoles ...string) error + UpdateUserToken(ctx context.Context, uuid string, token string) error + RemoveUser(ctx context.Context, uuid string) error + QueryProjects(ctx context.Context, uuid string, name string) ([]QProject, error) + UpdateProject(ctx context.Context, projectUUID string, name string, description string, modifiedOn time.Time) error + RemoveProject(ctx context.Context, uuid string) error + RemoveProjectTopics(ctx context.Context, projectUUID string) error + RemoveProjectSubs(ctx context.Context, projectUUID string) error + RemoveProjectDailyMessageCounters(ctx context.Context, projectUUID string) error + QueryDailyProjectMsgCount(ctx context.Context, projectUUID string) ([]QDailyProjectMsgCount, error) + QueryTotalMessagesPerProject(ctx context.Context, projectUUIDs []string, startDate time.Time, endDate time.Time) ([]QProjectMessageCount, error) + RegisterUser(ctx context.Context, uuid, name, firstName, lastName, email, org, desc, registeredAt, atkn, status string) error + DeleteRegistration(ctx context.Context, uuid string) error + QueryRegistrations(ctx context.Context, regUUID, status, activationToken, name, email, org string) ([]QUserRegistration, error) + UpdateRegistration(ctx context.Context, regUUID, status, declineComment, modifiedBy, modifiedAt string) error + InsertUser(ctx context.Context, uuid string, projects []QProjectRoles, name string, firstName string, lastName string, org string, desc string, token string, email string, serviceRoles []string, createdOn time.Time, modifiedOn time.Time, createdBy string) error + InsertProject(ctx context.Context, uuid string, name string, createdOn time.Time, modifiedOn time.Time, createdBy string, description string) error + InsertOpMetric(ctx context.Context, hostname string, cpu float64, mem float64) error + InsertTopic(ctx context.Context, projectUUID string, name string, schemaUUID string, createdOn time.Time) error + LinkTopicSchema(ctx context.Context, projectUUID, name, schemaUUID string) error + IncrementTopicMsgNum(ctx context.Context, projectUUID string, name string, num int64) error + IncrementDailyTopicMsgCount(ctx context.Context, projectUUID string, topicName string, num int64, date time.Time) error + IncrementTopicBytes(ctx context.Context, projectUUID string, name string, totalBytes int64) error + IncrementSubBytes(ctx context.Context, projectUUID string, name string, totalBytes int64) error + IncrementSubMsgNum(ctx context.Context, projectUUID string, name string, num int64) error + InsertSub(ctx context.Context, projectUUID string, name string, topic string, offest int64, ack int, pushCfg QPushConfig, createdOn time.Time) error + HasProject(ctx context.Context, name string) bool + HasUsers(ctx context.Context, projectUUID string, users []string) (bool, []string) + QueryOneSub(ctx context.Context, projectUUID string, name string) (QSub, error) + QueryPushSubs(ctx context.Context) []QSub + HasResourceRoles(ctx context.Context, resource string, roles []string) bool + GetOpMetrics(ctx context.Context) []QopMetric + GetUserRoles(ctx context.Context, projectUUID string, token string) ([]string, string) + GetUserFromToken(ctx context.Context, token string) (QUser, error) + UpdateSubOffset(ctx context.Context, projectUUID string, name string, offset int64) + UpdateSubPull(ctx context.Context, projectUUID string, name string, offset int64, ts string) error + UpdateSubOffsetAck(ctx context.Context, projectUUID string, name string, offset int64, ts string) error + ModSubPush(ctx context.Context, projectUUID string, name string, pushCfg QPushConfig) error + QueryACL(ctx context.Context, projectUUID string, resource string, name string) (QAcl, error) + ExistsInACL(ctx context.Context, projectUUID string, resource string, resourceName string, userUUID string) error + ModACL(ctx context.Context, projectUUID string, resource string, name string, acl []string) error + AppendToACL(ctx context.Context, projectUUID string, resource string, name string, acl []string) error + RemoveFromACL(ctx context.Context, projectUUID string, resource string, name string, acl []string) error + ModAck(ctx context.Context, projectUUID string, name string, ack int) error + GetAllRoles(ctx context.Context) []string + InsertSchema(ctx context.Context, projectUUID, schemaUUID, name, schemaType, rawSchemaString string) error + QuerySchemas(ctx context.Context, projectUUID, schemaUUID, name string) ([]QSchema, error) + UpdateSchema(ctx context.Context, schemaUUID, name, schemaType, rawSchemaString string) error + DeleteSchema(ctx context.Context, schemaUUID string) error + UsersCount(ctx context.Context, startDate, endDate time.Time, projectUUIDs []string) (map[string]int64, error) + TopicsCount(ctx context.Context, startDate, endDate time.Time, projectUUIDs []string) (map[string]int64, error) + SubscriptionsCount(ctx context.Context, startDate, endDate time.Time, projectUUIDs []string) (map[string]int64, error) Clone() Store Close() } diff --git a/stores/store_test.go b/stores/store_test.go index ff6a5a24..f82d6dcb 100644 --- a/stores/store_test.go +++ b/stores/store_test.go @@ -1,6 +1,7 @@ package stores import ( + "context" "errors" "testing" "time" @@ -14,6 +15,8 @@ type StoreTestSuite struct { func (suite *StoreTestSuite) TestMockStore() { + ctx := context.Background() + store := NewMockStore("mockhost", "mockbase") suite.Equal("mockhost", store.Server) suite.Equal("mockbase", store.Database) @@ -32,7 +35,7 @@ func (suite *StoreTestSuite) TestMockStore() { {0, "argo_uuid", "sub1", "topic1", 0, 0, "", "", "", 0, "", "", 10, "", 0, 0, 0, "", false, "", "", "", false, time.Date(2019, 5, 6, 0, 0, 0, 0, time.UTC), 10, time.Date(2020, 11, 19, 0, 0, 0, 0, time.UTC), []string{}}, } // retrieve all topics - tpList, ts1, pg1, _ := store.QueryTopics("argo_uuid", "", "", "", 0) + tpList, ts1, pg1, _ := store.QueryTopics(ctx, "argo_uuid", "", "", "", 0) suite.Equal(eTopList, tpList) suite.Equal(int64(4), ts1) suite.Equal("", pg1) @@ -42,7 +45,7 @@ func (suite *StoreTestSuite) TestMockStore() { {3, "argo_uuid", "topic4", 0, 0, time.Date(0, 0, 0, 0, 0, 0, 0, time.UTC), 0, "", time.Date(2020, 11, 19, 0, 0, 0, 0, time.UTC), []string{}}, {2, "argo_uuid", "topic3", 0, 0, time.Date(2019, 5, 7, 0, 0, 0, 0, time.UTC), 8.99, "schema_uuid_3", time.Date(2020, 11, 20, 0, 0, 0, 0, time.UTC), []string{}}, } - tpList2, ts2, pg2, _ := store.QueryTopics("argo_uuid", "", "", "", 2) + tpList2, ts2, pg2, _ := store.QueryTopics(ctx, "argo_uuid", "", "", "", 2) suite.Equal(eTopList1st2, tpList2) suite.Equal(int64(4), ts2) suite.Equal("1", pg2) @@ -51,7 +54,7 @@ func (suite *StoreTestSuite) TestMockStore() { eTopList3 := []QTopic{ {0, "argo_uuid", "topic1", 0, 0, time.Date(2019, 5, 6, 0, 0, 0, 0, time.UTC), 10, "", time.Date(2020, 11, 22, 0, 0, 0, 0, time.UTC), []string{}}, } - tpList3, ts3, pg3, _ := store.QueryTopics("argo_uuid", "", "", "0", 1) + tpList3, ts3, pg3, _ := store.QueryTopics(ctx, "argo_uuid", "", "", "0", 1) suite.Equal(eTopList3, tpList3) suite.Equal(int64(4), ts3) suite.Equal("", pg3) @@ -60,26 +63,26 @@ func (suite *StoreTestSuite) TestMockStore() { eTopList4 := []QTopic{ {0, "argo_uuid", "topic1", 0, 0, time.Date(2019, 5, 6, 0, 0, 0, 0, time.UTC), 10, "", time.Date(2020, 11, 22, 0, 0, 0, 0, time.UTC), []string{}}, } - tpList4, ts4, pg4, _ := store.QueryTopics("argo_uuid", "", "topic1", "", 0) + tpList4, ts4, pg4, _ := store.QueryTopics(ctx, "argo_uuid", "", "topic1", "", 0) suite.Equal(eTopList4, tpList4) suite.Equal(int64(0), ts4) suite.Equal("", pg4) // retrieve a single topic - store.LinkTopicSchema("argo_uuid", "topic1", "schema_uuid_1") + store.LinkTopicSchema(ctx, "argo_uuid", "topic1", "schema_uuid_1") eTopListSchema := []QTopic{ {0, "argo_uuid", "topic1", 0, 0, time.Date(2019, 5, 6, 0, 0, 0, 0, time.UTC), 10, "schema_uuid_1", time.Date(2020, 11, 22, 0, 0, 0, 0, time.UTC), []string{}}, } - tpListSchema, _, _, _ := store.QueryTopics("argo_uuid", "", "topic1", "", 0) + tpListSchema, _, _, _ := store.QueryTopics(ctx, "argo_uuid", "", "topic1", "", 0) suite.Equal(eTopListSchema, tpListSchema) - store.LinkTopicSchema("argo_uuid", "topic1", "") + store.LinkTopicSchema(ctx, "argo_uuid", "topic1", "") // retrieve user's topics eTopList5 := []QTopic{ {1, "argo_uuid", "topic2", 0, 0, time.Date(2019, 5, 8, 0, 0, 0, 0, time.UTC), 5.45, "schema_uuid_1", time.Date(2020, 11, 21, 0, 0, 0, 0, time.UTC), []string{}}, {0, "argo_uuid", "topic1", 0, 0, time.Date(2019, 5, 6, 0, 0, 0, 0, time.UTC), 10, "", time.Date(2020, 11, 22, 0, 0, 0, 0, time.UTC), []string{}}, } - tpList5, ts5, pg5, _ := store.QueryTopics("argo_uuid", "uuid1", "", "", 0) + tpList5, ts5, pg5, _ := store.QueryTopics(ctx, "argo_uuid", "uuid1", "", "", 0) suite.Equal(eTopList5, tpList5) suite.Equal(int64(2), ts5) suite.Equal("", pg5) @@ -89,13 +92,13 @@ func (suite *StoreTestSuite) TestMockStore() { {1, "argo_uuid", "topic2", 0, 0, time.Date(2019, 5, 8, 0, 0, 0, 0, time.UTC), 5.45, "schema_uuid_1", time.Date(2020, 11, 21, 0, 0, 0, 0, time.UTC), []string{}}, } - tpList6, ts6, pg6, _ := store.QueryTopics("argo_uuid", "uuid1", "", "", 1) + tpList6, ts6, pg6, _ := store.QueryTopics(ctx, "argo_uuid", "uuid1", "", "", 1) suite.Equal(eTopList6, tpList6) suite.Equal(int64(2), ts6) suite.Equal("0", pg6) // retrieve all subs - subList, ts1, pg1, err1 := store.QuerySubs("argo_uuid", "", "", "", 0) + subList, ts1, pg1, err1 := store.QuerySubs(ctx, "argo_uuid", "", "", "", 0) suite.Equal(eSubList, subList) suite.Equal(int64(4), ts1) suite.Equal("", pg3) @@ -105,7 +108,7 @@ func (suite *StoreTestSuite) TestMockStore() { {3, "argo_uuid", "sub4", "topic4", 0, 0, "", "http_endpoint", "endpoint.foo", 1, "autogen", "auth-header-1", 10, "linear", 300, 0, 0, "push-id-1", true, "", "", "", true, time.Date(0, 0, 0, 0, 0, 0, 0, time.UTC), 0, time.Date(2020, 11, 22, 0, 0, 0, 0, time.UTC), []string{}}, {2, "argo_uuid", "sub3", "topic3", 0, 0, "", "", "", 0, "", "", 10, "", 0, 0, 0, "", false, "", "", "", false, time.Date(2019, 5, 8, 0, 0, 0, 0, time.UTC), 5.45, time.Date(2020, 11, 21, 0, 0, 0, 0, time.UTC), []string{}}} - subList2, ts2, pg2, err2 := store.QuerySubs("argo_uuid", "", "", "", 2) + subList2, ts2, pg2, err2 := store.QuerySubs(ctx, "argo_uuid", "", "", "", 2) suite.Equal(eSubListFirstPage, subList2) suite.Equal(int64(4), ts2) suite.Equal("1", pg2) @@ -116,7 +119,7 @@ func (suite *StoreTestSuite) TestMockStore() { {0, "argo_uuid", "sub1", "topic1", 0, 0, "", "", "", 0, "", "", 10, "", 0, 0, 0, "", false, "", "", "", false, time.Date(2019, 5, 6, 0, 0, 0, 0, time.UTC), 10, time.Date(2020, 11, 19, 0, 0, 0, 0, time.UTC), []string{}}, } - subList3, ts3, pg3, err3 := store.QuerySubs("argo_uuid", "", "", "1", 2) + subList3, ts3, pg3, err3 := store.QuerySubs(ctx, "argo_uuid", "", "", "1", 2) suite.Equal(eSubListNextPage, subList3) suite.Equal(int64(4), ts3) suite.Equal("", pg3) @@ -128,7 +131,7 @@ func (suite *StoreTestSuite) TestMockStore() { {ID: 1, ProjectUUID: "argo_uuid", Name: "sub2", Topic: "topic2", Offset: 0, NextOffset: 0, PendingAck: "", PushEndpoint: "", MaxMessages: 0, Ack: 10, RetPolicy: "", RetPeriod: 0, MsgNum: 0, TotalBytes: 0, LatestConsume: time.Date(2019, 5, 7, 0, 0, 0, 0, time.UTC), ConsumeRate: 8.99, CreatedOn: time.Date(2020, 11, 20, 0, 0, 0, 0, time.UTC), ACL: []string{}}, } - subList4, ts4, pg4, err4 := store.QuerySubs("argo_uuid", "uuid1", "", "", 0) + subList4, ts4, pg4, err4 := store.QuerySubs(ctx, "argo_uuid", "uuid1", "", "", 0) suite.Equal(int64(3), ts4) suite.Equal("", pg4) @@ -139,7 +142,7 @@ func (suite *StoreTestSuite) TestMockStore() { {ID: 3, ProjectUUID: "argo_uuid", Name: "sub4", Topic: "topic4", Offset: 0, NextOffset: 0, PendingAck: "", PushType: "http_endpoint", PushEndpoint: "endpoint.foo", MaxMessages: 1, AuthorizationType: "autogen", AuthorizationHeader: "auth-header-1", Ack: 10, RetPolicy: "linear", RetPeriod: 300, MsgNum: 0, TotalBytes: 0, VerificationHash: "push-id-1", Verified: true, Base64Decode: true, LatestConsume: time.Date(0, 0, 0, 0, 0, 0, 0, time.UTC), ConsumeRate: 0, CreatedOn: time.Date(2020, 11, 22, 0, 0, 0, 0, time.UTC), ACL: []string{}}, {ID: 2, ProjectUUID: "argo_uuid", Name: "sub3", Topic: "topic3", Offset: 0, NextOffset: 0, PendingAck: "", PushEndpoint: "", MaxMessages: 0, Ack: 10, RetPolicy: "", RetPeriod: 0, MsgNum: 0, TotalBytes: 0, LatestConsume: time.Date(2019, 5, 8, 0, 0, 0, 0, time.UTC), ConsumeRate: 5.45, CreatedOn: time.Date(2020, 11, 21, 0, 0, 0, 0, time.UTC), ACL: []string{}}, } - subList5, ts5, pg5, err5 := store.QuerySubs("argo_uuid", "uuid1", "", "", 2) + subList5, ts5, pg5, err5 := store.QuerySubs(ctx, "argo_uuid", "uuid1", "", "", 2) suite.Equal(int64(3), ts5) suite.Equal("1", pg5) @@ -152,7 +155,7 @@ func (suite *StoreTestSuite) TestMockStore() { suite.Nil(err5) // test retrieve subs by topic - subListByTopic, errSublistByTopic := store.QuerySubsByTopic("argo_uuid", "topic1") + subListByTopic, errSublistByTopic := store.QuerySubsByTopic(ctx, "argo_uuid", "topic1") suite.Equal([]QSub{ { ID: 0, @@ -177,41 +180,41 @@ func (suite *StoreTestSuite) TestMockStore() { suite.Nil(errSublistByTopic) // Test ProjectUUID - suite.Equal(true, store.HasProject("ARGO")) - suite.Equal(false, store.HasProject("FOO")) + suite.Equal(true, store.HasProject(ctx, "ARGO")) + suite.Equal(false, store.HasProject(ctx, "FOO")) // check query all - qdsAll, _ := store.QueryDailyTopicMsgCount("", "", time.Time{}) + qdsAll, _ := store.QueryDailyTopicMsgCount(ctx, "", "", time.Time{}) suite.Equal(store.DailyTopicMsgCount, qdsAll) // test daily count - store.IncrementDailyTopicMsgCount("argo_uuid", "topic1", 40, time.Date(2018, 10, 1, 0, 0, 0, 0, time.UTC)) - qds, _ := store.QueryDailyTopicMsgCount("argo_uuid", "topic1", time.Date(2018, 10, 1, 0, 0, 0, 0, time.UTC)) + store.IncrementDailyTopicMsgCount(ctx, "argo_uuid", "topic1", 40, time.Date(2018, 10, 1, 0, 0, 0, 0, time.UTC)) + qds, _ := store.QueryDailyTopicMsgCount(ctx, "argo_uuid", "topic1", time.Date(2018, 10, 1, 0, 0, 0, 0, time.UTC)) suite.Equal(int64(80), qds[0].NumberOfMessages) // check if the it was inserted since it wasn't present - store.IncrementDailyTopicMsgCount("argo_uuid", "some_other_topic", 70, time.Date(2018, 10, 1, 0, 0, 0, 0, time.UTC)) - qds2, _ := store.QueryDailyTopicMsgCount("argo_uuid", "some_other_topic", time.Date(2018, 10, 1, 0, 0, 0, 0, time.UTC)) + store.IncrementDailyTopicMsgCount(ctx, "argo_uuid", "some_other_topic", 70, time.Date(2018, 10, 1, 0, 0, 0, 0, time.UTC)) + qds2, _ := store.QueryDailyTopicMsgCount(ctx, "argo_uuid", "some_other_topic", time.Date(2018, 10, 1, 0, 0, 0, 0, time.UTC)) suite.Equal(int64(70), qds2[0].NumberOfMessages) // Test user - roles01, _ := store.GetUserRoles("argo_uuid", "S3CR3T") - roles02, _ := store.GetUserRoles("argo_uuid", "SecretKey") + roles01, _ := store.GetUserRoles(ctx, "argo_uuid", "S3CR3T") + roles02, _ := store.GetUserRoles(ctx, "argo_uuid", "SecretKey") suite.Equal([]string{"consumer", "publisher"}, roles01) suite.Equal([]string{}, roles02) // Test roles - suite.Equal(true, store.HasResourceRoles("topics:list_all", []string{"admin"})) - suite.Equal(true, store.HasResourceRoles("topics:list_all", []string{"admin", "reader"})) - suite.Equal(true, store.HasResourceRoles("topics:list_all", []string{"admin", "foo"})) - suite.Equal(false, store.HasResourceRoles("topics:list_all", []string{"foo"})) - suite.Equal(false, store.HasResourceRoles("topics:publish", []string{"reader"})) - suite.Equal(true, store.HasResourceRoles("topics:list_all", []string{"admin"})) - suite.Equal(true, store.HasResourceRoles("topics:list_all", []string{"publisher"})) - suite.Equal(true, store.HasResourceRoles("topics:publish", []string{"publisher"})) - - store.InsertTopic("argo_uuid", "topicFresh", "", time.Date(2020, 9, 11, 0, 0, 0, 0, time.UTC)) - store.InsertSub("argo_uuid", "subFresh", "topicFresh", 0, 10, QPushConfig{}, time.Date(2020, 12, 19, 0, 0, 0, 0, time.UTC)) + suite.Equal(true, store.HasResourceRoles(ctx, "topics:list_all", []string{"admin"})) + suite.Equal(true, store.HasResourceRoles(ctx, "topics:list_all", []string{"admin", "reader"})) + suite.Equal(true, store.HasResourceRoles(ctx, "topics:list_all", []string{"admin", "foo"})) + suite.Equal(false, store.HasResourceRoles(ctx, "topics:list_all", []string{"foo"})) + suite.Equal(false, store.HasResourceRoles(ctx, "topics:publish", []string{"reader"})) + suite.Equal(true, store.HasResourceRoles(ctx, "topics:list_all", []string{"admin"})) + suite.Equal(true, store.HasResourceRoles(ctx, "topics:list_all", []string{"publisher"})) + suite.Equal(true, store.HasResourceRoles(ctx, "topics:publish", []string{"publisher"})) + + store.InsertTopic(ctx, "argo_uuid", "topicFresh", "", time.Date(2020, 9, 11, 0, 0, 0, 0, time.UTC)) + store.InsertSub(ctx, "argo_uuid", "subFresh", "topicFresh", 0, 10, QPushConfig{}, time.Date(2020, 12, 19, 0, 0, 0, 0, time.UTC)) eTopList2 := []QTopic{ {4, "argo_uuid", "topicFresh", 0, 0, time.Time{}, 0, "", time.Date(2020, 9, 11, 0, 0, 0, 0, time.UTC), []string{}}, @@ -228,34 +231,34 @@ func (suite *StoreTestSuite) TestMockStore() { {1, "argo_uuid", "sub2", "topic2", 0, 0, "", "", "", 0, "", "", 10, "", 0, 0, 0, "", false, "", "", "", false, time.Date(2019, 5, 7, 0, 0, 0, 0, time.UTC), 8.99, time.Date(2020, 11, 20, 0, 0, 0, 0, time.UTC), []string{}}, {0, "argo_uuid", "sub1", "topic1", 0, 0, "", "", "", 0, "", "", 10, "", 0, 0, 0, "", false, "", "", "", false, time.Date(2019, 5, 6, 0, 0, 0, 0, time.UTC), 10, time.Date(2020, 11, 19, 0, 0, 0, 0, time.UTC), []string{}}} - tpList, _, _, _ = store.QueryTopics("argo_uuid", "", "", "", 0) + tpList, _, _, _ = store.QueryTopics(ctx, "argo_uuid", "", "", "", 0) suite.Equal(eTopList2, tpList) - subList, _, _, _ = store.QuerySubs("argo_uuid", "", "", "", 0) + subList, _, _, _ = store.QuerySubs(ctx, "argo_uuid", "", "", "", 0) suite.Equal(eSubList2, subList) // Test delete on topic - err := store.RemoveTopic("argo_uuid", "topicFresh") + err := store.RemoveTopic(ctx, "argo_uuid", "topicFresh") suite.Equal(nil, err) - tpList, _, _, _ = store.QueryTopics("argo_uuid", "", "", "", 0) + tpList, _, _, _ = store.QueryTopics(ctx, "argo_uuid", "", "", "", 0) suite.Equal(eTopList, tpList) - err = store.RemoveTopic("argo_uuid", "topicFresh") + err = store.RemoveTopic(ctx, "argo_uuid", "topicFresh") suite.Equal("not found", err.Error()) // Test delete on subscription - err = store.RemoveSub("argo_uuid", "subFresh") + err = store.RemoveSub(ctx, "argo_uuid", "subFresh") suite.Equal(nil, err) - subList, _, _, _ = store.QuerySubs("argo_uuid", "", "", "", 0) + subList, _, _, _ = store.QuerySubs(ctx, "argo_uuid", "", "", "", 0) suite.Equal(eSubList, subList) - err = store.RemoveSub("argo_uuid", "subFresh") + err = store.RemoveSub(ctx, "argo_uuid", "subFresh") suite.Equal("not found", err.Error()) - sb, err := store.QueryOneSub("argo_uuid", "sub1") + sb, err := store.QueryOneSub(ctx, "argo_uuid", "sub1") esb := QSub{0, "argo_uuid", "sub1", "topic1", 0, 0, "", "", "", 0, "", "", 10, "", 0, 0, 0, "", false, "", "", "", false, time.Date(2019, 5, 6, 0, 0, 0, 0, time.UTC), 10, time.Date(2020, 11, 19, 0, 0, 0, 0, time.UTC), []string{}} suite.Equal(esb, sb) // Test modify ack deadline in store - store.ModAck("argo_uuid", "sub1", 66) - subAck, _ := store.QueryOneSub("argo_uuid", "sub1") + store.ModAck(ctx, "argo_uuid", "sub1", 66) + subAck, _ := store.QueryOneSub(ctx, "argo_uuid", "sub1") suite.Equal(66, subAck.Ack) // Test mod push sub @@ -273,8 +276,8 @@ func (suite *StoreTestSuite) TestMockStore() { MattermostUsername: "", MattermostChannel: "", } - e1 := store.ModSubPush("argo_uuid", "sub1", qCfg) - sub1, _ := store.QueryOneSub("argo_uuid", "sub1") + e1 := store.ModSubPush(ctx, "argo_uuid", "sub1", qCfg) + sub1, _ := store.QueryOneSub(ctx, "argo_uuid", "sub1") suite.Nil(e1) suite.Equal("example.com", sub1.PushEndpoint) suite.Equal(int64(3), sub1.MaxMessages) @@ -285,97 +288,97 @@ func (suite *StoreTestSuite) TestMockStore() { suite.Equal("auth-h-1", sub1.AuthorizationHeader) suite.True(sub1.Verified) - e2 := store.ModSubPush("argo_uuid", "unknown", QPushConfig{}) + e2 := store.ModSubPush(ctx, "argo_uuid", "unknown", QPushConfig{}) suite.Equal("not found", e2.Error()) // exists in acl - existsE1 := store.ExistsInACL("argo_uuid", "topics", "topic1", "uuid1") + existsE1 := store.ExistsInACL(ctx, "argo_uuid", "topics", "topic1", "uuid1") suite.Nil(existsE1) - existsE2 := store.ExistsInACL("argo_uuid", "topics", "topic1", "unknown") + existsE2 := store.ExistsInACL(ctx, "argo_uuid", "topics", "topic1", "unknown") suite.Equal("not found", existsE2.Error()) // Query ACLS ExpectedACL01 := QAcl{[]string{"uuid1", "uuid2"}} - QAcl01, _ := store.QueryACL("argo_uuid", "topics", "topic1") + QAcl01, _ := store.QueryACL(ctx, "argo_uuid", "topics", "topic1") suite.Equal(ExpectedACL01, QAcl01) ExpectedACL02 := QAcl{[]string{"uuid1", "uuid2", "uuid4"}} - QAcl02, _ := store.QueryACL("argo_uuid", "topics", "topic2") + QAcl02, _ := store.QueryACL(ctx, "argo_uuid", "topics", "topic2") suite.Equal(ExpectedACL02, QAcl02) ExpectedACL03 := QAcl{[]string{"uuid3"}} - QAcl03, _ := store.QueryACL("argo_uuid", "topics", "topic3") + QAcl03, _ := store.QueryACL(ctx, "argo_uuid", "topics", "topic3") suite.Equal(ExpectedACL03, QAcl03) ExpectedACL04 := QAcl{[]string{"uuid1", "uuid2"}} - QAcl04, _ := store.QueryACL("argo_uuid", "subscriptions", "sub1") + QAcl04, _ := store.QueryACL(ctx, "argo_uuid", "subscriptions", "sub1") suite.Equal(ExpectedACL04, QAcl04) ExpectedACL05 := QAcl{[]string{"uuid1", "uuid3"}} - QAcl05, _ := store.QueryACL("argo_uuid", "subscriptions", "sub2") + QAcl05, _ := store.QueryACL(ctx, "argo_uuid", "subscriptions", "sub2") suite.Equal(ExpectedACL05, QAcl05) ExpectedACL06 := QAcl{[]string{"uuid4", "uuid2", "uuid1"}} - QAcl06, _ := store.QueryACL("argo_uuid", "subscriptions", "sub3") + QAcl06, _ := store.QueryACL(ctx, "argo_uuid", "subscriptions", "sub3") suite.Equal(ExpectedACL06, QAcl06) ExpectedACL07 := QAcl{[]string{"uuid2", "uuid4", "uuid7"}} - QAcl07, _ := store.QueryACL("argo_uuid", "subscriptions", "sub4") + QAcl07, _ := store.QueryACL(ctx, "argo_uuid", "subscriptions", "sub4") suite.Equal(ExpectedACL07, QAcl07) - QAcl08, err08 := store.QueryACL("argo_uuid", "subscr", "sub4ss") + QAcl08, err08 := store.QueryACL(ctx, "argo_uuid", "subscr", "sub4ss") suite.Equal(QAcl{}, QAcl08) suite.Equal(errors.New("not found"), err08) // test mod acl - eModACL1 := store.ModACL("argo_uuid", "topics", "topic1", []string{"u1", "u2"}) + eModACL1 := store.ModACL(ctx, "argo_uuid", "topics", "topic1", []string{"u1", "u2"}) suite.Nil(eModACL1) tACL := store.TopicsACL["topic1"].ACL suite.Equal([]string{"u1", "u2"}, tACL) - eModACL2 := store.ModACL("argo_uuid", "subscriptions", "sub1", []string{"u1", "u2"}) + eModACL2 := store.ModACL(ctx, "argo_uuid", "subscriptions", "sub1", []string{"u1", "u2"}) suite.Nil(eModACL2) sACL := store.SubsACL["sub1"].ACL suite.Equal([]string{"u1", "u2"}, sACL) - eModACL3 := store.ModACL("argo_uuid", "mistype", "sub1", []string{"u1", "u2"}) + eModACL3 := store.ModACL(ctx, "argo_uuid", "mistype", "sub1", []string{"u1", "u2"}) suite.Equal("wrong resource type", eModACL3.Error()) // test append acl - eAppACL1 := store.AppendToACL("argo_uuid", "topics", "topic1", []string{"u3", "u4", "u4"}) + eAppACL1 := store.AppendToACL(ctx, "argo_uuid", "topics", "topic1", []string{"u3", "u4", "u4"}) suite.Nil(eAppACL1) tACLapp := store.TopicsACL["topic1"].ACL suite.Equal([]string{"u1", "u2", "u3", "u4"}, tACLapp) - eAppACL2 := store.AppendToACL("argo_uuid", "subscriptions", "sub1", []string{"u3", "u4", "u4"}) + eAppACL2 := store.AppendToACL(ctx, "argo_uuid", "subscriptions", "sub1", []string{"u3", "u4", "u4"}) suite.Nil(eAppACL2) sACLapp := store.SubsACL["sub1"].ACL suite.Equal([]string{"u1", "u2", "u3", "u4"}, sACLapp) - eAppACL3 := store.AppendToACL("argo_uuid", "mistype", "sub1", []string{"u3", "u4", "u4"}) + eAppACL3 := store.AppendToACL(ctx, "argo_uuid", "mistype", "sub1", []string{"u3", "u4", "u4"}) suite.Equal("wrong resource type", eAppACL3.Error()) // test remove acl - eRemACL1 := store.RemoveFromACL("argo_uuid", "topics", "topic1", []string{"u1", "u4", "u5"}) + eRemACL1 := store.RemoveFromACL(ctx, "argo_uuid", "topics", "topic1", []string{"u1", "u4", "u5"}) suite.Nil(eRemACL1) tACLRem := store.TopicsACL["topic1"].ACL suite.Equal([]string{"u2", "u3"}, tACLRem) - eRemACL2 := store.RemoveFromACL("argo_uuid", "subscriptions", "sub1", []string{"u1", "u4", "u5"}) + eRemACL2 := store.RemoveFromACL(ctx, "argo_uuid", "subscriptions", "sub1", []string{"u1", "u4", "u5"}) suite.Nil(eRemACL2) sACLRem := store.SubsACL["sub1"].ACL suite.Equal([]string{"u2", "u3"}, sACLRem) - eRemACL3 := store.RemoveFromACL("argo_uuid", "mistype", "sub1", []string{"u3", "u4", "u4"}) + eRemACL3 := store.RemoveFromACL(ctx, "argo_uuid", "mistype", "sub1", []string{"u3", "u4", "u4"}) suite.Equal("wrong resource type", eRemACL3.Error()) //Check has users - allFound, notFound := store.HasUsers("argo_uuid", []string{"UserA", "UserB", "FooUser"}) + allFound, notFound := store.HasUsers(ctx, "argo_uuid", []string{"UserA", "UserB", "FooUser"}) suite.Equal(false, allFound) suite.Equal([]string{"FooUser"}, notFound) - allFound, notFound = store.HasUsers("argo_uuid", []string{"UserA", "UserB"}) + allFound, notFound = store.HasUsers(ctx, "argo_uuid", []string{"UserA", "UserB"}) suite.Equal(true, allFound) suite.Equal([]string(nil), notFound) @@ -390,17 +393,17 @@ func (suite *StoreTestSuite) TestMockStore() { expProj3 := []QProject{qPr, qPr2} expProj4 := []QProject{} - projectOut1, err := store.QueryProjects("", "ARGO") + projectOut1, err := store.QueryProjects(ctx, "", "ARGO") suite.Equal(expProj1, projectOut1) suite.Equal(nil, err) - projectOut2, err := store.QueryProjects("", "ARGO2") + projectOut2, err := store.QueryProjects(ctx, "", "ARGO2") suite.Equal(expProj2, projectOut2) suite.Equal(nil, err) - projectOut3, err := store.QueryProjects("", "") + projectOut3, err := store.QueryProjects(ctx, "", "") suite.Equal(expProj3, projectOut3) suite.Equal(nil, err) - projectOut4, err := store.QueryProjects("", "FOO") + projectOut4, err := store.QueryProjects(ctx, "", "FOO") suite.Equal(expProj4, projectOut4) suite.Equal(errors.New("not found"), err) @@ -408,57 +411,57 @@ func (suite *StoreTestSuite) TestMockStore() { qPr3 := QProject{UUID: "argo_uuid3", Name: "ARGO3", CreatedOn: created, ModifiedOn: modified, CreatedBy: "uuid1", Description: "simple project"} expProj5 := []QProject{qPr, qPr2, qPr3} expProj6 := []QProject{qPr3} - store.InsertProject("argo_uuid3", "ARGO3", created, modified, "uuid1", "simple project") - projectOut5, err := store.QueryProjects("", "") + store.InsertProject(ctx, "argo_uuid3", "ARGO3", created, modified, "uuid1", "simple project") + projectOut5, err := store.QueryProjects(ctx, "", "") suite.Equal(expProj5, projectOut5) suite.Equal(nil, err) - projectOut6, err := store.QueryProjects("argo_uuid2", "ARGO3") + projectOut6, err := store.QueryProjects(ctx, "argo_uuid2", "ARGO3") suite.Equal(expProj6, projectOut6) suite.Equal(nil, err) // Test queries by uuid - projectOut7, err := store.QueryProjects("argo_uuid2", "") + projectOut7, err := store.QueryProjects(ctx, "argo_uuid2", "") suite.Equal(expProj2, projectOut7) suite.Equal(nil, err) - projectOut8, err := store.QueryProjects("foo_uuidNone", "") + projectOut8, err := store.QueryProjects(ctx, "foo_uuidNone", "") suite.Equal(expProj4, projectOut8) suite.Equal(errors.New("not found"), err) // Test update project modified = time.Date(2010, time.November, 10, 23, 0, 0, 0, time.UTC) expPr1 := QProject{UUID: "argo_uuid3", Name: "ARGO3", CreatedOn: created, ModifiedOn: modified, CreatedBy: "uuid1", Description: "a modified description"} - store.UpdateProject("argo_uuid3", "", "a modified description", modified) - prUp1, _ := store.QueryProjects("argo_uuid3", "") + store.UpdateProject(ctx, "argo_uuid3", "", "a modified description", modified) + prUp1, _ := store.QueryProjects(ctx, "argo_uuid3", "") suite.Equal(expPr1, prUp1[0]) expPr2 := QProject{UUID: "argo_uuid3", Name: "ARGO_updated3", CreatedOn: created, ModifiedOn: modified, CreatedBy: "uuid1", Description: "a modified description"} - store.UpdateProject("argo_uuid3", "ARGO_updated3", "", modified) - prUp2, _ := store.QueryProjects("argo_uuid3", "") + store.UpdateProject(ctx, "argo_uuid3", "ARGO_updated3", "", modified) + prUp2, _ := store.QueryProjects(ctx, "argo_uuid3", "") suite.Equal(expPr2, prUp2[0]) expPr3 := QProject{UUID: "argo_uuid3", Name: "ARGO_3", CreatedOn: created, ModifiedOn: modified, CreatedBy: "uuid1", Description: "a newly modified description"} - store.UpdateProject("argo_uuid3", "ARGO_3", "a newly modified description", modified) - prUp3, _ := store.QueryProjects("argo_uuid3", "") + store.UpdateProject(ctx, "argo_uuid3", "ARGO_3", "a newly modified description", modified) + prUp3, _ := store.QueryProjects(ctx, "argo_uuid3", "") suite.Equal(expPr3, prUp3[0]) // Test Sub Update Pull - err = store.UpdateSubPull("argo_uuid", "sub4", 4, "2016-10-11T12:00:35:15Z") - qSubUpd, _, _, err := store.QuerySubs("argo_uuid", "", "sub4", "", 0) + err = store.UpdateSubPull(ctx, "argo_uuid", "sub4", 4, "2016-10-11T12:00:35:15Z") + qSubUpd, _, _, err := store.QuerySubs(ctx, "argo_uuid", "", "sub4", "", 0) var nxtOff int64 = 4 suite.Equal(qSubUpd[0].NextOffset, nxtOff) suite.Equal("2016-10-11T12:00:35:15Z", qSubUpd[0].PendingAck) // Test RemoveProjectTopics - store.RemoveProjectTopics("argo_uuid") - resTop, _, _, _ := store.QueryTopics("argo_uuid", "", "", "", 0) + store.RemoveProjectTopics(ctx, "argo_uuid") + resTop, _, _, _ := store.QueryTopics(ctx, "argo_uuid", "", "", "", 0) suite.Equal(0, len(resTop)) - store.RemoveProjectSubs("argo_uuid") - resSub, _, _, _ := store.QuerySubs("argo_uuid", "", "", "", 0) + store.RemoveProjectSubs(ctx, "argo_uuid") + resSub, _, _, _ := store.QuerySubs(ctx, "argo_uuid", "", "", "", 0) suite.Equal(0, len(resSub)) - store.RemoveProjectDailyMessageCounters("argo_uuid") - resMc, _ := store.QueryDailyProjectMsgCount("argo_uuid") + store.RemoveProjectDailyMessageCounters(ctx, "argo_uuid") + resMc, _ := store.QueryDailyProjectMsgCount(ctx, "argo_uuid") suite.Equal(0, len(resMc)) // Test RemoveProject - store.RemoveProject("argo_uuid") - resProj, err := store.QueryProjects("argo_uuid", "") + store.RemoveProject(ctx, "argo_uuid") + resProj, err := store.QueryProjects(ctx, "argo_uuid", "") suite.Equal([]QProject{}, resProj) suite.Equal(errors.New("not found"), err) @@ -467,16 +470,16 @@ func (suite *StoreTestSuite) TestMockStore() { qRoles := []QProjectRoles{QProjectRoles{"argo_uuid", []string{"admin"}}, QProjectRoles{"argo_uuid2", []string{"admin", "viewer"}}} expUsr10 := QUser{UUID: "user_uuid10", Projects: qRoleAdmin1, Name: "newUser1", FirstName: "fname", LastName: "lname", Organization: "org1", Description: "desc1", Token: "A3B94A94V3A", Email: "fake@email.com", ServiceRoles: []string{}, CreatedOn: created, ModifiedOn: modified, CreatedBy: "uuid1"} expUsr11 := QUser{UUID: "user_uuid11", Projects: qRoles, Name: "newUser2", Token: "BX312Z34NLQ", Email: "fake@email.com", ServiceRoles: []string{}, CreatedOn: created, ModifiedOn: modified, CreatedBy: "uuid1"} - store.InsertUser("user_uuid10", qRoleAdmin1, "newUser1", "fname", "lname", "org1", "desc1", "A3B94A94V3A", "fake@email.com", []string{}, created, modified, "uuid1") - store.InsertUser("user_uuid11", qRoles, "newUser2", "", "", "", "", "BX312Z34NLQ", "fake@email.com", []string{}, created, modified, "uuid1") - usr10, _ := store.QueryUsers("argo_uuid", "user_uuid10", "") - usr11, _ := store.QueryUsers("argo_uuid", "", "newUser2") + store.InsertUser(ctx, "user_uuid10", qRoleAdmin1, "newUser1", "fname", "lname", "org1", "desc1", "A3B94A94V3A", "fake@email.com", []string{}, created, modified, "uuid1") + store.InsertUser(ctx, "user_uuid11", qRoles, "newUser2", "", "", "", "", "BX312Z34NLQ", "fake@email.com", []string{}, created, modified, "uuid1") + usr10, _ := store.QueryUsers(ctx, "argo_uuid", "user_uuid10", "") + usr11, _ := store.QueryUsers(ctx, "argo_uuid", "", "newUser2") suite.Equal(expUsr10, usr10[0]) suite.Equal(expUsr11, usr11[0]) - rolesA, usernameA := store.GetUserRoles("argo_uuid", "BX312Z34NLQ") - rolesB, usernameB := store.GetUserRoles("argo_uuid2", "BX312Z34NLQ") + rolesA, usernameA := store.GetUserRoles(ctx, "argo_uuid", "BX312Z34NLQ") + rolesB, usernameB := store.GetUserRoles(ctx, "argo_uuid2", "BX312Z34NLQ") suite.Equal("newUser2", usernameA) suite.Equal("newUser2", usernameB) @@ -485,13 +488,13 @@ func (suite *StoreTestSuite) TestMockStore() { // Test Update User usrUpdated := QUser{UUID: "user_uuid11", Projects: qRoles, Name: "updated_name", Token: "BX312Z34NLQ", Email: "fake@email.com", ServiceRoles: []string{"service_admin"}, CreatedOn: created, ModifiedOn: modified, CreatedBy: "uuid1"} - store.UpdateUser("user_uuid11", "", "", "", "", nil, "updated_name", "", []string{"service_admin"}, modified) - usr11, _ = store.QueryUsers("", "user_uuid11", "") + store.UpdateUser(ctx, "user_uuid11", "", "", "", "", nil, "updated_name", "", []string{"service_admin"}, modified) + usr11, _ = store.QueryUsers(ctx, "", "user_uuid11", "") suite.Equal(usrUpdated, usr11[0]) // test append project to user - errUserPrj := store.AppendToUserProjects("uuid1", "p1_uuid", "r1", "r2") - usr, _ := store.QueryUsers("", "uuid1", "") + errUserPrj := store.AppendToUserProjects(ctx, "uuid1", "p1_uuid", "r1", "r2") + usr, _ := store.QueryUsers(ctx, "", "uuid1", "") suite.Equal([]QProjectRoles{ { ProjectUUID: "argo_uuid", @@ -505,29 +508,29 @@ func (suite *StoreTestSuite) TestMockStore() { suite.Nil(errUserPrj) // Test Remove User - store.RemoveUser("user_uuid11") - usr11, err = store.QueryUsers("", "user_uuid11", "") + store.RemoveUser(ctx, "user_uuid11") + usr11, err = store.QueryUsers(ctx, "", "user_uuid11", "") suite.Equal(errors.New("not found"), err) - usrGet, _ := store.GetUserFromToken("A3B94A94V3A") + usrGet, _ := store.GetUserFromToken(ctx, "A3B94A94V3A") suite.Equal(usr10[0], usrGet) // test paginated query users store2 := NewMockStore("", "") // return all users in one page - qUsers1, ts1, pg1, _ := store2.PaginatedQueryUsers("", 0, "") + qUsers1, ts1, pg1, _ := store2.PaginatedQueryUsers(ctx, "", 0, "") // return a page with the first 2 - qUsers2, ts2, pg2, _ := store2.PaginatedQueryUsers("", 2, "") + qUsers2, ts2, pg2, _ := store2.PaginatedQueryUsers(ctx, "", 2, "") // empty store store3 := NewMockStore("", "") store3.UserList = []QUser{} - qUsers3, ts3, pg3, _ := store3.PaginatedQueryUsers("", 0, "") + qUsers3, ts3, pg3, _ := store3.PaginatedQueryUsers(ctx, "", 0, "") // use page token "5" to grab another 2 results - qUsers4, ts4, pg4, _ := store2.PaginatedQueryUsers("4", 2, "") + qUsers4, ts4, pg4, _ := store2.PaginatedQueryUsers(ctx, "4", 2, "") suite.Equal(store2.UserList, qUsers1) suite.Equal("", pg1) @@ -548,40 +551,40 @@ func (suite *StoreTestSuite) TestMockStore() { suite.Equal(int64(2), ts4) // test update topic latest publish time - e1ulp := store2.UpdateTopicLatestPublish("argo_uuid", "topic1", time.Date(2019, 8, 8, 0, 0, 0, 0, time.UTC)) + e1ulp := store2.UpdateTopicLatestPublish(ctx, "argo_uuid", "topic1", time.Date(2019, 8, 8, 0, 0, 0, 0, time.UTC)) suite.Nil(e1ulp) - tpc, _, _, _ := store2.QueryTopics("argo_uuid", "", "topic1", "", 0) + tpc, _, _, _ := store2.QueryTopics(ctx, "argo_uuid", "", "topic1", "", 0) suite.Equal(time.Date(2019, 8, 8, 0, 0, 0, 0, time.UTC), tpc[0].LatestPublish) // test update topic publishing rate - e1upr := store2.UpdateTopicPublishRate("argo_uuid", "topic1", 8.44) + e1upr := store2.UpdateTopicPublishRate(ctx, "argo_uuid", "topic1", 8.44) suite.Nil(e1upr) - tpc2, _, _, _ := store2.QueryTopics("argo_uuid", "", "topic1", "", 0) + tpc2, _, _, _ := store2.QueryTopics(ctx, "argo_uuid", "", "topic1", "", 0) suite.Equal(8.44, tpc2[0].PublishRate) // test update topic latest publish time - scre1 := store2.UpdateSubLatestConsume("argo_uuid", "sub1", time.Date(2019, 8, 8, 0, 0, 0, 0, time.UTC)) + scre1 := store2.UpdateSubLatestConsume(ctx, "argo_uuid", "sub1", time.Date(2019, 8, 8, 0, 0, 0, 0, time.UTC)) suite.Nil(scre1) - spc, _, _, _ := store2.QuerySubs("argo_uuid", "", "sub1", "", 0) + spc, _, _, _ := store2.QuerySubs(ctx, "argo_uuid", "", "sub1", "", 0) suite.Equal(time.Date(2019, 8, 8, 0, 0, 0, 0, time.UTC), spc[0].LatestConsume) // test update topic publishing rate - scre2 := store2.UpdateSubConsumeRate("argo_uuid", "sub1", 8.44) + scre2 := store2.UpdateSubConsumeRate(ctx, "argo_uuid", "sub1", 8.44) suite.Nil(scre2) - spc2, _, _, _ := store2.QuerySubs("argo_uuid", "", "sub1", "", 0) + spc2, _, _, _ := store2.QuerySubs(ctx, "argo_uuid", "", "sub1", "", 0) suite.Equal(8.44, spc2[0].ConsumeRate) // test QueryTotalMessagesPerProject expectedQpmc := []QProjectMessageCount{ {ProjectUUID: "argo_uuid", NumberOfMessages: 30, AverageDailyMessages: 7}, } - qpmc, qpmcerr1 := store2.QueryTotalMessagesPerProject([]string{"argo_uuid"}, time.Date(2018, 10, 1, 0, 0, 0, 0, time.UTC), time.Date(2018, 10, 4, 0, 0, 0, 0, time.UTC)) + qpmc, qpmcerr1 := store2.QueryTotalMessagesPerProject(ctx, []string{"argo_uuid"}, time.Date(2018, 10, 1, 0, 0, 0, 0, time.UTC), time.Date(2018, 10, 4, 0, 0, 0, 0, time.UTC)) suite.Equal(expectedQpmc, qpmc) suite.Nil(qpmcerr1) // test InsertSchema - eis := store.InsertSchema("argo_uuid", "uuid1", "s1-insert", "json", "raw") - qs1, _ := store.QuerySchemas("argo_uuid", "uuid1", "s1-insert") + eis := store.InsertSchema(ctx, "argo_uuid", "uuid1", "s1-insert", "json", "raw") + qs1, _ := store.QuerySchemas(ctx, "argo_uuid", "uuid1", "s1-insert") suite.Equal(QSchema{ ProjectUUID: "argo_uuid", UUID: "uuid1", @@ -599,32 +602,32 @@ func (suite *StoreTestSuite) TestMockStore() { {UUID: "schema_uuid_2", ProjectUUID: "argo_uuid", Type: "json", Name: "schema-2", RawSchema: s}, {UUID: "schema_uuid_3", ProjectUUID: "argo_uuid", Type: "avro", Name: "schema-3", RawSchema: avros}, } - qqs1, _ := store2.QuerySchemas("argo_uuid", "", "") - qqs2, _ := store2.QuerySchemas("argo_uuid", "schema_uuid_1", "") - qqs3, _ := store2.QuerySchemas("argo_uuid", "schema_uuid_1", "schema-1") + qqs1, _ := store2.QuerySchemas(ctx, "argo_uuid", "", "") + qqs2, _ := store2.QuerySchemas(ctx, "argo_uuid", "schema_uuid_1", "") + qqs3, _ := store2.QuerySchemas(ctx, "argo_uuid", "schema_uuid_1", "schema-1") suite.Equal(expectedSchemas, qqs1) suite.Equal(expectedSchemas[0], qqs2[0]) suite.Equal(expectedSchemas[0], qqs3[0]) // test update schema - store2.UpdateSchema("schema_uuid_1", "new-name", "new-type", "new-raw-schema") + store2.UpdateSchema(ctx, "schema_uuid_1", "new-name", "new-type", "new-raw-schema") eus := QSchema{UUID: "schema_uuid_1", ProjectUUID: "argo_uuid", Type: "new-type", Name: "new-name", RawSchema: "new-raw-schema"} - qus, _ := store2.QuerySchemas("argo_uuid", "schema_uuid_1", "") + qus, _ := store2.QuerySchemas(ctx, "argo_uuid", "schema_uuid_1", "") suite.Equal(eus, qus[0]) //test delete schema store4 := NewMockStore("", "") - ed := store4.DeleteSchema("schema_uuid_1") - expd, _ := store4.QuerySchemas("argo_uuid", "schema_uuid_1", "") + ed := store4.DeleteSchema(ctx, "schema_uuid_1") + expd, _ := store4.QuerySchemas(ctx, "argo_uuid", "schema_uuid_1", "") // check that topic-1 no longer has any schema_uuid associated with it - qtd, _, _, _ := store4.QueryTopics("argo_uuid", "", "topic2", "", 1) + qtd, _, _, _ := store4.QueryTopics(ctx, "argo_uuid", "", "topic2", "", 1) suite.Equal("", qtd[0].SchemaUUID) suite.Equal([]QSchema{}, expd) suite.Nil(ed) // test user registration - store.RegisterUser("ruuid1", "n1", "f1", "l1", "e1", "o1", "d1", "time", "atkn", "pending") + store.RegisterUser(ctx, "ruuid1", "n1", "f1", "l1", "e1", "o1", "d1", "time", "atkn", "pending") expur1 := []QUserRegistration{{ UUID: "ruuid1", Name: "n1", @@ -638,10 +641,10 @@ func (suite *StoreTestSuite) TestMockStore() { Status: "pending", }} - ur1, _ := store.QueryRegistrations("ruuid1", "pending", "atkn", "n1", "e1", "o1") + ur1, _ := store.QueryRegistrations(ctx, "ruuid1", "pending", "atkn", "n1", "e1", "o1") suite.Equal(expur1, ur1) - ur12, _ := store.QueryRegistrations("ruuid1", "", "", "", "", "") + ur12, _ := store.QueryRegistrations(ctx, "ruuid1", "", "", "", "", "") suite.Equal(expur1, ur12) expur2 := []QUserRegistration{{ @@ -658,22 +661,22 @@ func (suite *StoreTestSuite) TestMockStore() { ModifiedBy: "uuid1", ModifiedAt: "2020-05-17T22:26:58Z", }} - store.UpdateRegistration("ur-uuid1", "accepted", "", "uuid1", "2020-05-17T22:26:58Z") - ur2, _ := store.QueryRegistrations("ur-uuid1", "accepted", "", "", "", "") + store.UpdateRegistration(ctx, "ur-uuid1", "accepted", "", "uuid1", "2020-05-17T22:26:58Z") + ur2, _ := store.QueryRegistrations(ctx, "ur-uuid1", "accepted", "", "", "", "") suite.Equal(expur2, ur2) suite.Equal(2, len(store.UserRegistrations)) - _ = store.DeleteRegistration("ruuid1") + _ = store.DeleteRegistration(ctx, "ruuid1") suite.Equal(1, len(store.UserRegistrations)) - dErr := store.DeleteRegistration("unknown") + dErr := store.DeleteRegistration(ctx, "unknown") suite.Equal("not found", dErr.Error()) sdate := time.Date(2008, 11, 19, 8, 0, 0, 0, time.UTC) edate := time.Date(2020, 11, 21, 6, 0, 0, 0, time.UTC) - tc, _ := store3.TopicsCount(sdate, edate, []string{}) - sc, _ := store3.SubscriptionsCount(sdate, edate, []string{}) - uc, _ := store2.UsersCount(sdate, edate, []string{}) + tc, _ := store3.TopicsCount(ctx, sdate, edate, []string{}) + sc, _ := store3.SubscriptionsCount(ctx, sdate, edate, []string{}) + uc, _ := store2.UsersCount(ctx, sdate, edate, []string{}) suite.Equal(map[string]int64{"argo_uuid": 3}, tc) suite.Equal(map[string]int64{"argo_uuid": 3}, sc) diff --git a/subscriptions/subscription.go b/subscriptions/subscription.go index adce8cdb..c1e13de9 100644 --- a/subscriptions/subscription.go +++ b/subscriptions/subscription.go @@ -1,6 +1,7 @@ package subscriptions import ( + "context" "encoding/json" "errors" "strconv" @@ -165,9 +166,9 @@ func IsAuthorizationHeaderTypeSupported(authzType string) bool { } // FindMetric returns the metric of a specific subscription -func FindMetric(projectUUID string, name string, store stores.Store) (SubMetrics, error) { +func FindMetric(ctx context.Context, projectUUID string, name string, store stores.Store) (SubMetrics, error) { result := SubMetrics{MsgNum: 0} - subs, _, _, err := store.QuerySubs(projectUUID, "", name, "", 0) + subs, _, _, err := store.QuerySubs(ctx, projectUUID, "", name, "", 0) // check if sub exists if len(subs) == 0 { @@ -175,7 +176,7 @@ func FindMetric(projectUUID string, name string, store stores.Store) (SubMetrics } for _, item := range subs { - projectName := projects.GetNameByUUID(item.ProjectUUID, store) + projectName := projects.GetNameByUUID(ctx, item.ProjectUUID, store) if projectName == "" { return result, errors.New("invalid project") } @@ -215,7 +216,7 @@ func GetPullOptionsJSON(input []byte) (SubPullOptions, error) { return s, err } -// GetAckDeadlineFromJson retrieves ack deadline from json input +// GetAckDeadlineFromJSON retrieves ack deadline from json input func GetAckDeadlineFromJSON(input []byte) (AckDeadline, error) { s := AckDeadline{} err := json.Unmarshal([]byte(input), &s) @@ -278,7 +279,7 @@ func (sl *PaginatedSubscriptions) ExportJSON() (string, error) { } // VerifyPushEndpoint verifies the ownership of a push endpoint -func VerifyPushEndpoint(sub Subscription, c *http.Client, store stores.Store) error { +func VerifyPushEndpoint(ctx context.Context, sub Subscription, c *http.Client, store stores.Store) error { // extract the push endpoint host if sub.PushCfg.Pend == "" { @@ -309,8 +310,16 @@ func VerifyPushEndpoint(sub Subscription, c *http.Client, store stores.Store) er } if resp.StatusCode != 200 { - log.Errorf("failed to verify push endpoint for subscription %v. Expected status response %v but got %v", - sub.FullName, http.StatusOK, resp.StatusCode) + log.WithFields( + log.Fields{ + "trace_id": ctx.Value("trace_id"), + "type": "backend_log", + "backend_service": "remote_endpoint", + "backend_hosts": u.String(), + "subscription": sub.FullName, + "status": resp.StatusCode, + }, + ).Error("failed to verify push endpoint for subscription") return errors.New("Wrong response status code") } else { // read the response @@ -320,8 +329,18 @@ func VerifyPushEndpoint(sub Subscription, c *http.Client, store stores.Store) er defer resp.Body.Close() if sub.PushCfg.VerificationHash != buf.String() { - log.Errorf("failed to verify push endpoint for subscription %v. Expected verification hash %v but got %v", - sub.FullName, sub.PushCfg.VerificationHash, buf.String()) + log.WithFields( + log.Fields{ + "trace_id": ctx.Value("trace_id"), + "type": "backend_log", + "backend_service": "remote_endpoint", + "backend_hosts": u.String(), + "subscription": sub.FullName, + "status": resp.StatusCode, + "expected_hash": sub.PushCfg.VerificationHash, + "actual_hash": buf.String(), + }, + ).Error("failed to verify hash for push endpoint of subscription") return errors.New("Wrong verification hash") } } @@ -339,7 +358,7 @@ func VerifyPushEndpoint(sub Subscription, c *http.Client, store stores.Store) er MattermostUsername: sub.PushCfg.MattermostUsername, MattermostChannel: sub.PushCfg.MattermostChannel, } - err = ModSubPush(sub.ProjectUUID, sub.Name, cfg, store) + err = ModSubPush(ctx, sub.ProjectUUID, sub.Name, cfg, store) if err != nil { return err } @@ -348,7 +367,7 @@ func VerifyPushEndpoint(sub Subscription, c *http.Client, store stores.Store) er } // Find searches the store for all subscriptions of a given project or a specific one -func Find(projectUUID, userUUID, name, pageToken string, pageSize int64, store stores.Store) (PaginatedSubscriptions, error) { +func Find(ctx context.Context, projectUUID, userUUID, name, pageToken string, pageSize int64, store stores.Store) (PaginatedSubscriptions, error) { var err error var qSubs []stores.QSub @@ -360,15 +379,22 @@ func Find(projectUUID, userUUID, name, pageToken string, pageSize int64, store s // decode the base64 pageToken if pageTokenBytes, err = base64.StdEncoding.DecodeString(pageToken); err != nil { - log.Errorf("Page token %v produced an error while being decoded to base64: %v", pageToken, err.Error()) + log.WithFields( + log.Fields{ + "trace_id": ctx.Value("trace_id"), + "type": "request_log", + "page_token": pageToken, + "error": err.Error(), + }, + ).Error("error while decoding to base64") return result, err } - if qSubs, totalSize, nextPageToken, err = store.QuerySubs(projectUUID, userUUID, name, string(pageTokenBytes), pageSize); err != nil { + if qSubs, totalSize, nextPageToken, err = store.QuerySubs(ctx, projectUUID, userUUID, name, string(pageTokenBytes), pageSize); err != nil { return result, err } - projectName := projects.GetNameByUUID(projectUUID, store) + projectName := projects.GetNameByUUID(ctx, projectUUID, store) if projectName == "" { return result, errors.New("invalid project") @@ -425,15 +451,15 @@ func Find(projectUUID, userUUID, name, pageToken string, pageSize int64, store s } // FindByTopic retrieves all subscriptions associated with the given topic -func FindByTopic(projectUUID string, topicName string, store stores.Store) (NamesList, error) { +func FindByTopic(ctx context.Context, projectUUID string, topicName string, store stores.Store) (NamesList, error) { - subs, err := store.QuerySubsByTopic(projectUUID, topicName) + subs, err := store.QuerySubsByTopic(ctx, projectUUID, topicName) if err != nil { return NewNamesList(), err } subNames := NewNamesList() - projectName := projects.GetNameByUUID(projectUUID, store) + projectName := projects.GetNameByUUID(ctx, projectUUID, store) for _, sub := range subs { subNames.Subscriptions = append(subNames.Subscriptions, @@ -446,9 +472,9 @@ func FindByTopic(projectUUID string, topicName string, store stores.Store) (Name // LoadPushSubs returns all subscriptions defined in store that have a push configuration func LoadPushSubs(store stores.Store) PaginatedSubscriptions { result := PaginatedSubscriptions{Subscriptions: []Subscription{}} - subs := store.QueryPushSubs() + subs := store.QueryPushSubs(context.Background()) for _, item := range subs { - projectName := projects.GetNameByUUID(item.ProjectUUID, store) + projectName := projects.GetNameByUUID(context.Background(), item.ProjectUUID, store) curSub := New(item.ProjectUUID, projectName, item.Name, item.Topic) curSub.Offset = item.Offset curSub.NextOffset = item.NextOffset @@ -462,10 +488,10 @@ func LoadPushSubs(store stores.Store) PaginatedSubscriptions { } // Create creates a new subscription -func Create(projectUUID string, name string, topic string, offset int64, ack int, +func Create(ctx context.Context, projectUUID string, name string, topic string, offset int64, ack int, pushCfg PushConfig, createdOn time.Time, store stores.Store) (Subscription, error) { - if HasSub(projectUUID, name, store) { + if HasSub(ctx, projectUUID, name, store) { return Subscription{}, errors.New("exists") } @@ -493,12 +519,12 @@ func Create(projectUUID string, name string, topic string, offset int64, ack int Base64Decode: pushCfg.Base64Decode, } - err := store.InsertSub(projectUUID, name, topic, offset, ack, qPushCfg, createdOn) + err := store.InsertSub(ctx, projectUUID, name, topic, offset, ack, qPushCfg, createdOn) if err != nil { return Subscription{}, errors.New("backend error") } - results, err := Find(projectUUID, "", name, "", 0, store) + results, err := Find(ctx, projectUUID, "", name, "", 0, store) if len(results.Subscriptions) != 1 { return Subscription{}, errors.New("backend error") } @@ -507,23 +533,31 @@ func Create(projectUUID string, name string, topic string, offset int64, ack int } // ModAck updates the subscription's acknowledgment timeout -func ModAck(projectUUID string, name string, ack int, store stores.Store) error { +func ModAck(ctx context.Context, projectUUID string, name string, ack int, store stores.Store) error { // minimum deadline allowed 0 seconds, maximum: 600 sec (10 minutes) if ack < 0 || ack > 600 { return errors.New("wrong value") } - if HasSub(projectUUID, name, store) == false { + if HasSub(ctx, projectUUID, name, store) == false { return errors.New("not found") } - return store.ModAck(projectUUID, name, ack) + log.WithFields( + log.Fields{ + "trace_id": ctx.Value("trace_id"), + "type": "service_log", + "deadline": ack, + }, + ).Info("modifying ack deadline") + + return store.ModAck(ctx, projectUUID, name, ack) } // ModSubPush updates the subscription push config -func ModSubPush(projectUUID string, name string, pushCfg PushConfig, store stores.Store) error { +func ModSubPush(ctx context.Context, projectUUID string, name string, pushCfg PushConfig, store stores.Store) error { - if HasSub(projectUUID, name, store) == false { + if HasSub(ctx, projectUUID, name, store) == false { return errors.New("not found") } @@ -546,22 +580,22 @@ func ModSubPush(projectUUID string, name string, pushCfg PushConfig, store store MattermostUsername: pushCfg.MattermostUsername, } - return store.ModSubPush(projectUUID, name, qPushCfg) + return store.ModSubPush(ctx, projectUUID, name, qPushCfg) } // RemoveSub removes an existing subscription -func RemoveSub(projectUUID string, name string, store stores.Store) error { +func RemoveSub(ctx context.Context, projectUUID string, name string, store stores.Store) error { - if HasSub(projectUUID, name, store) == false { + if HasSub(ctx, projectUUID, name, store) == false { return errors.New("not found") } - return store.RemoveSub(projectUUID, name) + return store.RemoveSub(ctx, projectUUID, name) } // HasSub returns true if project & subscription combination exist -func HasSub(projectUUID string, name string, store stores.Store) bool { - res, err := Find(projectUUID, "", name, "", 0, store) +func HasSub(ctx context.Context, projectUUID string, name string, store stores.Store) bool { + res, err := Find(ctx, projectUUID, "", name, "", 0, store) if len(res.Subscriptions) > 0 && err == nil { return true } diff --git a/subscriptions/subscription_test.go b/subscriptions/subscription_test.go index f338a88b..c933a0e2 100644 --- a/subscriptions/subscription_test.go +++ b/subscriptions/subscription_test.go @@ -1,6 +1,7 @@ package subscriptions import ( + "context" "errors" "io/ioutil" "testing" @@ -19,6 +20,7 @@ import ( type SubTestSuite struct { suite.Suite cfgStr string + ctx context.Context } type MockPushRoundTripper struct{} @@ -68,6 +70,7 @@ func (m *MockPushRoundTripper) RoundTrip(r *http.Request) (*http.Response, error } func (suite *SubTestSuite) SetupTest() { + suite.ctx = context.Background() suite.cfgStr = `{ "port":8080, "broker_hosts":["localhost:9092"], @@ -91,13 +94,13 @@ func (suite *SubTestSuite) TestFindByTopic() { store := stores.NewMockStore("", "") - nl1, err1 := FindByTopic("argo_uuid", "topic1", store) + nl1, err1 := FindByTopic(suite.ctx, "argo_uuid", "topic1", store) suite.Equal([]string{"/projects/ARGO/subscriptions/sub1"}, nl1.Subscriptions) suite.Nil(err1) // check empty case store.SubList = nil - nl2, err2 := FindByTopic("argo_uuid", "topic1", store) + nl2, err2 := FindByTopic(suite.ctx, "argo_uuid", "topic1", store) suite.Equal([]string{}, nl2.Subscriptions) suite.Nil(err2) } @@ -134,7 +137,7 @@ func (suite *SubTestSuite) TestGetSubByName() { cfgAPI.LoadStrJSON(suite.cfgStr) store := stores.NewMockStore(cfgAPI.StoreHost, cfgAPI.StoreDB) - result, _ := Find("argo_uuid", "", "sub1", "", 0, store) + result, _ := Find(suite.ctx, "argo_uuid", "", "sub1", "", 0, store) expSub := New("argo_uuid", "ARGO", "sub1", "topic1") expSub.PushCfg.RetPol.PolicyType = "" expSub.PushCfg.RetPol.Period = 0 @@ -149,7 +152,7 @@ func (suite *SubTestSuite) TestGetSubMetric() { APIcfg := config.NewAPICfg() APIcfg.LoadStrJSON(suite.cfgStr) store := stores.NewMockStore(APIcfg.StoreHost, APIcfg.StoreDB) - mySubM, _ := FindMetric("argo_uuid", "sub1", store) + mySubM, _ := FindMetric(suite.ctx, "argo_uuid", "sub1", store) expTopic := SubMetrics{ MsgNum: 0, LatestConsume: time.Date(2019, 5, 6, 0, 0, 0, 0, time.UTC), @@ -163,8 +166,8 @@ func (suite *SubTestSuite) TestHasProjectTopic() { cfgAPI.LoadStrJSON(suite.cfgStr) store := stores.NewMockStore(cfgAPI.StoreHost, cfgAPI.StoreDB) - suite.Equal(false, HasSub("argo_uuid", "FOO", store)) - suite.Equal(true, HasSub("argo_uuid", "sub1", store)) + suite.Equal(false, HasSub(suite.ctx, "argo_uuid", "FOO", store)) + suite.Equal(true, HasSub(suite.ctx, "argo_uuid", "sub1", store)) } func (suite *SubTestSuite) TestGetSubsByProject() { @@ -219,35 +222,35 @@ func (suite *SubTestSuite) TestGetSubsByProject() { expSubs1 = append(expSubs1, expSub3) expSubs1 = append(expSubs1, expSub2) expSubs1 = append(expSubs1, expSub1) - result1, _ := Find("argo_uuid", "", "", "", 0, store) + result1, _ := Find(suite.ctx, "argo_uuid", "", "", "", 0, store) // retrieve first two subs expSubs2 := []Subscription{} expSubs2 = append(expSubs2, expSub4) expSubs2 = append(expSubs2, expSub3) - result2, _ := Find("argo_uuid", "", "", "", 2, store) + result2, _ := Find(suite.ctx, "argo_uuid", "", "", "", 2, store) //retrieve the next two subs expSubs3 := []Subscription{} expSubs3 = append(expSubs3, expSub2) expSubs3 = append(expSubs3, expSub1) - result3, _ := Find("argo_uuid", "", "", "MQ==", 2, store) + result3, _ := Find(suite.ctx, "argo_uuid", "", "", "MQ==", 2, store) // provide an invalid page token - _, err := Find("", "", "", "invalid", 0, store) + _, err := Find(suite.ctx, "", "", "", "invalid", 0, store) // retrieve user's subs expSubs4 := []Subscription{} expSubs4 = append(expSubs4, expSub4) expSubs4 = append(expSubs4, expSub3) expSubs4 = append(expSubs4, expSub2) - result4, _ := Find("argo_uuid", "uuid1", "", "", 0, store) + result4, _ := Find(suite.ctx, "argo_uuid", "uuid1", "", "", 0, store) // retrieve user's subs with pagination expSubs5 := []Subscription{} expSubs5 = append(expSubs5, expSub4) expSubs5 = append(expSubs5, expSub3) - result5, _ := Find("argo_uuid", "uuid1", "", "", 2, store) + result5, _ := Find(suite.ctx, "argo_uuid", "uuid1", "", "", 2, store) suite.Equal(expSubs1, result1.Subscriptions) suite.Equal("", result1.NextPageToken) @@ -277,7 +280,7 @@ func (suite *SubTestSuite) TestLoadFromCfg() { cfgAPI.LoadStrJSON(suite.cfgStr) store := stores.NewMockStore(cfgAPI.StoreHost, cfgAPI.StoreDB) - results, _ := Find("argo_uuid", "", "", "", 0, store) + results, _ := Find(suite.ctx, "argo_uuid", "", "", "", 0, store) expSub1 := New("argo_uuid", "ARGO", "sub1", "topic1") expSub1.PushCfg.RetPol.PolicyType = "" expSub1.PushCfg.RetPol.Period = 0 @@ -344,12 +347,12 @@ func (suite *SubTestSuite) TestRemoveSubStore() { store := stores.NewMockStore(APIcfg.StoreHost, APIcfg.StoreDB) - suite.Equal(true, HasSub("argo_uuid", "sub1", store)) + suite.Equal(true, HasSub(suite.ctx, "argo_uuid", "sub1", store)) - suite.Equal("not found", RemoveSub("argo_uuid", "subFoo", store).Error()) - suite.Equal(nil, RemoveSub("argo_uuid", "sub1", store)) + suite.Equal("not found", RemoveSub(suite.ctx, "argo_uuid", "subFoo", store).Error()) + suite.Equal(nil, RemoveSub(suite.ctx, "argo_uuid", "sub1", store)) - suite.Equal(false, HasSub("ARGO", "sub1", store)) + suite.Equal(false, HasSub(suite.ctx, "ARGO", "sub1", store)) } func (suite *SubTestSuite) TestCreateSubStore() { @@ -359,12 +362,12 @@ func (suite *SubTestSuite) TestCreateSubStore() { store := stores.NewMockStore(APIcfg.StoreHost, APIcfg.StoreDB) - sub, err := Create("argo_uuid", "sub1", "topic1", 0, 300, + sub, err := Create(suite.ctx, "argo_uuid", "sub1", "topic1", 0, 300, PushConfig{}, time.Date(2019, 7, 7, 0, 0, 0, 0, time.UTC), store) suite.Equal(Subscription{}, sub) suite.Equal("exists", err.Error()) - sub2, err2 := Create("argo_uuid", "subNew", "topicNew", 0, 0, + sub2, err2 := Create(suite.ctx, "argo_uuid", "subNew", "topicNew", 0, 0, PushConfig{}, time.Date(2019, 7, 7, 0, 0, 0, 0, time.UTC), store) expSub := New("argo_uuid", "ARGO", "subNew", "topicNew") expSub.CreatedOn = "2019-07-07T00:00:00Z" @@ -380,16 +383,16 @@ func (suite *SubTestSuite) TestModAck() { store := stores.NewMockStore(APIcfg.StoreHost, APIcfg.StoreDB) - err := ModAck("argo_uuid", "sub1", 300, store) + err := ModAck(suite.ctx, "argo_uuid", "sub1", 300, store) suite.Equal(nil, err) - err = ModAck("argo_uuid", "sub1", 0, store) + err = ModAck(suite.ctx, "argo_uuid", "sub1", 0, store) suite.Equal(nil, err) - err = ModAck("argo_uuid", "sub1", -300, store) + err = ModAck(suite.ctx, "argo_uuid", "sub1", -300, store) suite.Equal(errors.New("wrong value"), err) - err = ModAck("argo_uuid", "sub1", 601, store) + err = ModAck(suite.ctx, "argo_uuid", "sub1", 601, store) suite.Equal(errors.New("wrong value"), err) } @@ -419,11 +422,11 @@ func (suite *SubTestSuite) TestModSubPush() { MattermostUsername: "", MattermostChannel: "", } - err1 := ModSubPush("argo_uuid", "sub1", cfg, store) + err1 := ModSubPush(suite.ctx, "argo_uuid", "sub1", cfg, store) suite.Nil(err1) - sub1, _ := store.QueryOneSub("argo_uuid", "sub1") + sub1, _ := store.QueryOneSub(suite.ctx, "argo_uuid", "sub1") suite.Equal("example.com", sub1.PushEndpoint) suite.Equal(int64(2), sub1.MaxMessages) suite.Equal("linear", sub1.RetPolicy) @@ -432,7 +435,7 @@ func (suite *SubTestSuite) TestModSubPush() { suite.True(sub1.Verified) // test error case - err2 := ModSubPush("argo_uuid", "unknown", PushConfig{}, store) + err2 := ModSubPush(suite.ctx, "argo_uuid", "unknown", PushConfig{}, store) suite.Equal("not found", err2.Error()) } @@ -498,9 +501,9 @@ func (suite *SubTestSuite) TestVerifyPushEndpoint() { Transport: new(MockPushRoundTripper), } - e1 := VerifyPushEndpoint(s1, c1, str) + e1 := VerifyPushEndpoint(suite.ctx, s1, c1, str) - qs1, _ := str.QueryOneSub("argo_uuid", "push-sub-v1") + qs1, _ := str.QueryOneSub(suite.ctx, "argo_uuid", "push-sub-v1") suite.Nil(e1) suite.True(qs1.Verified) @@ -517,7 +520,7 @@ func (suite *SubTestSuite) TestVerifyPushEndpoint() { Transport: new(MockPushRoundTripper), } - e2 := VerifyPushEndpoint(s2, c2, nil) + e2 := VerifyPushEndpoint(suite.ctx, s2, c2, nil) suite.Equal("Wrong response status code", e2.Error()) @@ -533,7 +536,7 @@ func (suite *SubTestSuite) TestVerifyPushEndpoint() { Transport: new(MockPushRoundTripper), } - e3 := VerifyPushEndpoint(s3, c3, nil) + e3 := VerifyPushEndpoint(suite.ctx, s3, c3, nil) suite.Equal("Wrong verification hash", e3.Error()) } @@ -544,7 +547,7 @@ func (suite *SubTestSuite) TestExportJson() { store := stores.NewMockStore(cfgAPI.StoreHost, cfgAPI.StoreDB) - res, _ := Find("argo_uuid", "", "sub1", "", 0, store) + res, _ := Find(suite.ctx, "argo_uuid", "", "sub1", "", 0, store) outJSON, _ := res.Subscriptions[0].ExportJSON() expJSON := `{ @@ -656,7 +659,7 @@ func (suite *SubTestSuite) TestExportJson() { "nextPageToken": "", "totalSize": 4 }` - results, _ := Find("argo_uuid", "", "", "", 0, store) + results, _ := Find(suite.ctx, "argo_uuid", "", "", "", 0, store) outJSON2, _ := results.ExportJSON() suite.Equal(expJSON2, outJSON2) diff --git a/topics/topic.go b/topics/topic.go index 713bc447..2076c59a 100644 --- a/topics/topic.go +++ b/topics/topic.go @@ -1,6 +1,7 @@ package topics import ( + "context" "encoding/json" "errors" @@ -56,10 +57,10 @@ func New(projectUUID string, projectName string, name string) Topic { return t } -// Find searches and returns a specific topic or all topics of a given project -func FindMetric(projectUUID string, name string, store stores.Store) (TopicMetrics, error) { +// FindMetric searches and returns a specific topic or all topics of a given project +func FindMetric(ctx context.Context, projectUUID string, name string, store stores.Store) (TopicMetrics, error) { result := TopicMetrics{MsgNum: 0} - topics, _, _, err := store.QueryTopics(projectUUID, "", name, "", 0) + topics, _, _, err := store.QueryTopics(ctx, projectUUID, "", name, "", 0) // check if the topic exists if len(topics) == 0 { @@ -67,7 +68,7 @@ func FindMetric(projectUUID string, name string, store stores.Store) (TopicMetri } for _, item := range topics { - projectName := projects.GetNameByUUID(item.ProjectUUID, store) + projectName := projects.GetNameByUUID(ctx, item.ProjectUUID, store) if projectName == "" { return result, errors.New("invalid project") } @@ -81,7 +82,7 @@ func FindMetric(projectUUID string, name string, store stores.Store) (TopicMetri } // Find searches and returns a specific topic or all topics of a given project -func Find(projectUUID, userUUID, name, pageToken string, pageSize int64, store stores.Store) (PaginatedTopics, error) { +func Find(ctx context.Context, projectUUID, userUUID, name, pageToken string, pageSize int64, store stores.Store) (PaginatedTopics, error) { var err error var qTopics []stores.QTopic @@ -95,17 +96,20 @@ func Find(projectUUID, userUUID, name, pageToken string, pageSize int64, store s if pageTokenBytes, err = base64.StdEncoding.DecodeString(pageToken); err != nil { log.WithFields( log.Fields{ - "type": "service_log", + "trace_id": ctx.Value("trace_id"), + "type": "request_log", + "page_token": pageToken, + "error": err.Error(), }, - ).Errorf("Page token %v produced an error while being decoded to base64: %v", pageToken, err.Error()) + ).Error("error while decoding to base64") return result, err } - if qTopics, totalSize, nextPageToken, err = store.QueryTopics(projectUUID, userUUID, name, string(pageTokenBytes), pageSize); err != nil { + if qTopics, totalSize, nextPageToken, err = store.QueryTopics(ctx, projectUUID, userUUID, name, string(pageTokenBytes), pageSize); err != nil { return result, err } - projectName := projects.GetNameByUUID(projectUUID, store) + projectName := projects.GetNameByUUID(ctx, projectUUID, store) if projectName == "" { return result, errors.New("invalid project") @@ -119,7 +123,7 @@ func Find(projectUUID, userUUID, name, pageToken string, pageSize int64, store s curTop.CreatedOn = item.CreatedOn.UTC().Format("2006-01-02T15:04:05Z") if item.SchemaUUID != "" { - sl, err := schemas.Find(projectUUID, item.SchemaUUID, "", store) + sl, err := schemas.Find(ctx, projectUUID, item.SchemaUUID, "", store) if err == nil { if !sl.Empty() { curTop.Schema = schemas.FormatSchemaRef(projectName, sl.Schemas[0].Name) @@ -127,6 +131,7 @@ func Find(projectUUID, userUUID, name, pageToken string, pageSize int64, store s } else { log.WithFields( log.Fields{ + "trace_id": ctx.Value("trace_id"), "type": "service_log", "topic_name": item.Name, "project_uuid": projectUUID, @@ -166,18 +171,18 @@ func (tl *PaginatedTopics) ExportJSON() (string, error) { } // CreateTopic creates a new topic -func CreateTopic(projectUUID string, name string, schemaUUID string, createdOn time.Time, store stores.Store) (Topic, error) { +func CreateTopic(ctx context.Context, projectUUID string, name string, schemaUUID string, createdOn time.Time, store stores.Store) (Topic, error) { - if HasTopic(projectUUID, name, store) { + if HasTopic(ctx, projectUUID, name, store) { return Topic{}, errors.New("exists") } - err := store.InsertTopic(projectUUID, name, schemaUUID, createdOn) + err := store.InsertTopic(ctx, projectUUID, name, schemaUUID, createdOn) if err != nil { return Topic{}, errors.New("backend error") } - results, err := Find(projectUUID, "", name, "", 0, store) + results, err := Find(ctx, projectUUID, "", name, "", 0, store) if len(results.Topics) != 1 { return Topic{}, errors.New("backend error") @@ -187,27 +192,27 @@ func CreateTopic(projectUUID string, name string, schemaUUID string, createdOn t } // AttachSchemaToTopic links the provided schema with the given topic -func AttachSchemaToTopic(projectUUID, name, schemaUUID string, store stores.Store) error { - return store.LinkTopicSchema(projectUUID, name, schemaUUID) +func AttachSchemaToTopic(ctx context.Context, projectUUID, name, schemaUUID string, store stores.Store) error { + return store.LinkTopicSchema(ctx, projectUUID, name, schemaUUID) } // DetachSchemaFromTopic removes the link between the provided schema and the given topic -func DetachSchemaFromTopic(projectUUID, name string, store stores.Store) error { - return store.LinkTopicSchema(projectUUID, name, "") +func DetachSchemaFromTopic(ctx context.Context, projectUUID, name string, store stores.Store) error { + return store.LinkTopicSchema(ctx, projectUUID, name, "") } // RemoveTopic removes an existing topic -func RemoveTopic(projectUUID string, name string, store stores.Store) error { - if HasTopic(projectUUID, name, store) == false { +func RemoveTopic(ctx context.Context, projectUUID string, name string, store stores.Store) error { + if HasTopic(ctx, projectUUID, name, store) == false { return errors.New("not found") } - return store.RemoveTopic(projectUUID, name) + return store.RemoveTopic(ctx, projectUUID, name) } // HasTopic returns true if project & topic combination exist -func HasTopic(projectUUID string, name string, store stores.Store) bool { - res, err := Find(projectUUID, "", name, "", 0, store) +func HasTopic(ctx context.Context, projectUUID string, name string, store stores.Store) bool { + res, err := Find(ctx, projectUUID, "", name, "", 0, store) if len(res.Topics) > 0 && err == nil { return true } diff --git a/topics/topic_test.go b/topics/topic_test.go index cb1f66cb..83ca5e09 100644 --- a/topics/topic_test.go +++ b/topics/topic_test.go @@ -1,6 +1,7 @@ package topics import ( + "context" "io/ioutil" "testing" @@ -16,9 +17,11 @@ import ( type TopicTestSuite struct { suite.Suite cfgStr string + ctx context.Context } func (suite *TopicTestSuite) SetupTest() { + suite.ctx = context.Background() suite.cfgStr = `{ "broker_host":"localhost:9092", "store_host":"localhost", @@ -38,7 +41,7 @@ func (suite *TopicTestSuite) TestGetTopicByName() { APIcfg := config.NewAPICfg() APIcfg.LoadStrJSON(suite.cfgStr) store := stores.NewMockStore(APIcfg.StoreHost, APIcfg.StoreDB) - myTopics, _ := Find("argo_uuid", "", "topic1", "", 0, store) + myTopics, _ := Find(suite.ctx, "argo_uuid", "", "topic1", "", 0, store) expTopic := New("argo_uuid", "ARGO", "topic1") expTopic.PublishRate = 10 expTopic.LatestPublish = time.Date(2019, 5, 6, 0, 0, 0, 0, time.UTC) @@ -57,36 +60,36 @@ func (suite *TopicTestSuite) TestGetPaginatedTopics() { {"argo_uuid", "topic2", "/projects/ARGO/topics/topic2", time.Date(2019, 5, 8, 0, 0, 0, 0, time.UTC), 5.45, "projects/ARGO/schemas/schema-1", "2020-11-21T00:00:00Z"}, {"argo_uuid", "topic1", "/projects/ARGO/topics/topic1", time.Date(2019, 5, 6, 0, 0, 0, 0, time.UTC), 10, "", "2020-11-22T00:00:00Z"}}, NextPageToken: "", TotalSize: 4} - pgTopics1, err1 := Find("argo_uuid", "", "", "", 0, store) + pgTopics1, err1 := Find(suite.ctx, "argo_uuid", "", "", "", 0, store) // retrieve first 2 topics expPt2 := PaginatedTopics{Topics: []Topic{ {"argo_uuid", "topic4", "/projects/ARGO/topics/topic4", time.Date(0, 0, 0, 0, 0, 0, 0, time.UTC), 0, "", "2020-11-19T00:00:00Z"}, {"argo_uuid", "topic3", "/projects/ARGO/topics/topic3", time.Date(2019, 5, 7, 0, 0, 0, 0, time.UTC), 8.99, "projects/ARGO/schemas/schema-3", "2020-11-20T00:00:00Z"}}, NextPageToken: "MQ==", TotalSize: 4} - pgTopics2, err2 := Find("argo_uuid", "", "", "", 2, store) + pgTopics2, err2 := Find(suite.ctx, "argo_uuid", "", "", "", 2, store) // retrieve the next topic expPt3 := PaginatedTopics{Topics: []Topic{ {"argo_uuid", "topic1", "/projects/ARGO/topics/topic1", time.Date(2019, 5, 6, 0, 0, 0, 0, time.UTC), 10, "", "2020-11-22T00:00:00Z"}}, NextPageToken: "", TotalSize: 4} - pgTopics3, err3 := Find("argo_uuid", "", "", "MA==", 1, store) + pgTopics3, err3 := Find(suite.ctx, "argo_uuid", "", "", "MA==", 1, store) // invalid page token - _, err4 := Find("", "", "", "invalid", 0, store) + _, err4 := Find(suite.ctx, "", "", "", "invalid", 0, store) // retrieve topics for a specific user expPt5 := PaginatedTopics{Topics: []Topic{ {"argo_uuid", "topic2", "/projects/ARGO/topics/topic2", time.Date(2019, 5, 8, 0, 0, 0, 0, time.UTC), 5.45, "projects/ARGO/schemas/schema-1", "2020-11-21T00:00:00Z"}, {"argo_uuid", "topic1", "/projects/ARGO/topics/topic1", time.Date(2019, 5, 6, 0, 0, 0, 0, time.UTC), 10, "", "2020-11-22T00:00:00Z"}}, NextPageToken: "", TotalSize: 2} - pgTopics5, err5 := Find("argo_uuid", "uuid1", "", "", 2, store) + pgTopics5, err5 := Find(suite.ctx, "argo_uuid", "uuid1", "", "", 2, store) // retrieve topics for a specific user with pagination expPt6 := PaginatedTopics{Topics: []Topic{ {"argo_uuid", "topic2", "/projects/ARGO/topics/topic2", time.Date(2019, 5, 8, 0, 0, 0, 0, time.UTC), 5.45, "projects/ARGO/schemas/schema-1", "2020-11-21T00:00:00Z"}}, NextPageToken: "MA==", TotalSize: 2} - pgTopics6, err6 := Find("argo_uuid", "uuid1", "", "", 1, store) + pgTopics6, err6 := Find(suite.ctx, "argo_uuid", "uuid1", "", "", 1, store) suite.Equal(expPt1, pgTopics1) suite.Equal(expPt2, pgTopics2) @@ -106,7 +109,7 @@ func (suite *TopicTestSuite) TestGetTopicMetric() { APIcfg := config.NewAPICfg() APIcfg.LoadStrJSON(suite.cfgStr) store := stores.NewMockStore(APIcfg.StoreHost, APIcfg.StoreDB) - myTopics, _ := FindMetric("argo_uuid", "topic1", store) + myTopics, _ := FindMetric(suite.ctx, "argo_uuid", "topic1", store) expTopic := TopicMetrics{MsgNum: 0} expTopic.LatestPublish = time.Date(2019, 5, 6, 0, 0, 0, 0, time.UTC) expTopic.PublishRate = 10 @@ -118,7 +121,7 @@ func (suite *TopicTestSuite) TestGetTopicMetrics() { APIcfg := config.NewAPICfg() APIcfg.LoadStrJSON(suite.cfgStr) store := stores.NewMockStore(APIcfg.StoreHost, APIcfg.StoreDB) - myTopics, _ := FindMetric("argo_uuid", "topic1", store) + myTopics, _ := FindMetric(suite.ctx, "argo_uuid", "topic1", store) expTopic := TopicMetrics{MsgNum: 0} expTopic.PublishRate = 10 expTopic.LatestPublish = time.Date(2019, 5, 6, 0, 0, 0, 0, time.UTC) @@ -131,11 +134,11 @@ func (suite *TopicTestSuite) TestCreateTopicStore() { store := stores.NewMockStore(APIcfg.StoreHost, APIcfg.StoreDB) - tp, err := CreateTopic("argo_uuid", "topic1", "", time.Time{}, store) + tp, err := CreateTopic(suite.ctx, "argo_uuid", "topic1", "", time.Time{}, store) suite.Equal(Topic{}, tp) suite.Equal("exists", err.Error()) - tp2, err2 := CreateTopic("argo_uuid", "topicNew", "schema_uuid_1", time.Date(2019, 5, 7, 0, 0, 0, 0, time.UTC), store) + tp2, err2 := CreateTopic(suite.ctx, "argo_uuid", "topicNew", "schema_uuid_1", time.Date(2019, 5, 7, 0, 0, 0, 0, time.UTC), store) expTopic := New("argo_uuid", "ARGO", "topicNew") expTopic.Schema = "projects/ARGO/schemas/schema-1" expTopic.CreatedOn = "2019-05-07T00:00:00Z" @@ -149,11 +152,11 @@ func (suite *TopicTestSuite) TestRemoveTopicStore() { store := stores.NewMockStore(APIcfg.StoreHost, APIcfg.StoreDB) - suite.Equal(true, HasTopic("argo_uuid", "topic1", store)) + suite.Equal(true, HasTopic(suite.ctx, "argo_uuid", "topic1", store)) - suite.Equal("not found", RemoveTopic("argo_uuid", "topicFoo", store).Error()) - suite.Equal(nil, RemoveTopic("argo_uuid", "topic1", store)) - suite.Equal(false, HasTopic("argo_uuid", "topic1", store)) + suite.Equal("not found", RemoveTopic(suite.ctx, "argo_uuid", "topicFoo", store).Error()) + suite.Equal(nil, RemoveTopic(suite.ctx, "argo_uuid", "topic1", store)) + suite.Equal(false, HasTopic(suite.ctx, "argo_uuid", "topic1", store)) } func (suite *TopicTestSuite) TestHasProjectTopic() { @@ -162,8 +165,8 @@ func (suite *TopicTestSuite) TestHasProjectTopic() { store := stores.NewMockStore(APIcfg.StoreHost, APIcfg.StoreDB) - suite.Equal(false, HasTopic("argo_uuid", "FOO", store)) - suite.Equal(true, HasTopic("argo_uuid", "topic1", store)) + suite.Equal(false, HasTopic(suite.ctx, "argo_uuid", "FOO", store)) + suite.Equal(true, HasTopic(suite.ctx, "argo_uuid", "topic1", store)) } func (suite *TopicTestSuite) TestExportJson() { @@ -172,7 +175,7 @@ func (suite *TopicTestSuite) TestExportJson() { store := stores.NewMockStore(APIcfg.StoreHost, APIcfg.StoreDB) - topics, _ := Find("argo_uuid", "", "topic1", "", 0, store) + topics, _ := Find(suite.ctx, "argo_uuid", "", "topic1", "", 0, store) outJSON, _ := topics.Topics[0].ExportJSON() expJSON := `{ "name": "/projects/ARGO/topics/topic1", @@ -204,7 +207,7 @@ func (suite *TopicTestSuite) TestExportJson() { "nextPageToken": "", "totalSize": 4 }` - topics2, _ := Find("argo_uuid", "", "", "", 0, store) + topics2, _ := Find(suite.ctx, "argo_uuid", "", "", "", 0, store) outJSON2, _ := topics2.ExportJSON() suite.Equal(expJSON2, outJSON2) From b21026b04937334482218199ee28bf63dcee36b9 Mon Sep 17 00:00:00 2001 From: Themis Zamani Date: Tue, 4 Jul 2023 11:23:23 +0300 Subject: [PATCH 2/9] Update codemeta.json --- codemeta.json | 105 +++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 86 insertions(+), 19 deletions(-) diff --git a/codemeta.json b/codemeta.json index 7b1329ae..8e239b9d 100644 --- a/codemeta.json +++ b/codemeta.json @@ -1,21 +1,88 @@ { - "@context": "https://doi.org/10.5063/SCHEMA/CODEMETA-2.0", - "@type": "SoftwareSourceCode", - "name": "Argo Messaging Service", - "description": "The ARGO Messaging Service is a Publish/Subscribe Service, which implements the Google PubSub protocol. Instead of focusing on a single Messaging API specification for handling the logic of publishing/subscribing to the broker network the API focuses on creating nodes of Publishers and Subscribers as a Service. It provides an HTTP API that enables Users/Systems to implement message oriented services using the Publish/Subscribe Model over plain HTTP. In the Publish/Subscribe paradigm, Publishers are users/systems that can send messages to named-channels called Topics. Subscribers are users/systems that create Subscriptions to specific topics and receive messages.", - "programmingLanguage": ["Golang", "Python"], - "license": "Apache", - "version": "1.5.0", - "url": "https://github.com/ARGOeu/argo-messaging", - "author": { - "@type": "Organization", - "name": "GRNET", - "url": "https://grnet.gr/" - }, - "keywords": ["messaging", "pub/sub", "golang"], - "repositoryCode": { - "@type": "GitRepository", - "codeRepository": "https://github.com/ARGOeu/argo-messaging.git" - } + "@context": "https://doi.org/10.5063/schema/codemeta-2.0", + "@type": "SoftwareSourceCode", + "license": "https://spdx.org/licenses/Apache-2.0.html", + "codeRepository": "https://github.com/ARGOeu/argo-messaging.git", + "dateCreated": "2015-12-20", + "datePublished": "2016-03-24", + "dateModified": "2023-05-23", + "downloadUrl": "https://github.com/ARGOeu/argo-messaging/archive/refs/heads/master.zip", + "issueTracker": "https://github.com/ARGOeu/argo-messaging/issues", + "name": "Argo Messaging Service", + "version": "1.5.0", + "description": "The ARGO Messaging Service is a Publish/Subscribe Service, which implements the Google PubSub protocol. Instead of focusing on a single Messaging API specification for handling the logic of publishing/subscribing to the broker network the API focuses on creating nodes of Publishers and Subscribers as a Service. It provides an HTTP API that enables Users/Systems to implement message oriented services using the Publish/Subscribe Model over plain HTTP. In the Publish/Subscribe paradigm, Publishers are users/systems that can send messages to named-channels called Topics. Subscribers are users/systems that create Subscriptions to specific topics and receive messages.", + "applicationCategory": "messaging service", + "releaseNotes": "Features\nAM-311 Upgrade to go1.19\nBug/Fixes\nUpdate docusaurus dependencies\nAM-306 Static analysis assessment fixes\nUpdate Postman CI/CD collection to use x-api-key header\nDocumentation\n\nUpdate README.md\nCreate CITATION.cff\nCreate CONTRIBUTING.md\nCreate CODE_OF_CONDUCT.md\nProcess for adding a new mattermost integration", + "developmentStatus": "active", + "isPartOf": "https://eosc-portal.eu", + "funder": { + "@type": "Organization", + "name": "GRNET S.A. – National Infrastructures for Research and Technology" + }, + "keywords": [ + "messaging", + "pub/sub", + "golang", + "kafka" + ], + "programmingLanguage": [ + "Golang", + "Python", + "Java" + ], + "author": [ + { + "@type": "Person", + "@id": "https://orcid.org/0000-0003-1148-1738", + "givenName": "Themis", + "familyName": "Zamani", + "email": "themis@admin.grnet.gr", + "affiliation": { + "@type": "Organization", + "name": "GRNET S.A" + } + }, + { + "@type": "Person", + "givenName": "Konstantinos", + "familyName": "Kagkelidis", + "email": "kaggis@admin.grnet.gr", + "affiliation": { + "@type": "Organization", + "name": "GRNET S.A " + } + }, + { + "@type": "Person", + "givenName": "Agelos ", + "familyName": "Tsalapatis", + "email": "agelostsal@admin.grnet.gr", + "affiliation": { + "@type": "Organization", + "name": "GRNET S.A" + } + }, + { + "@type": "Person", + "givenName": "Kostas", + "familyName": "Koumantaros", + "email": "kkoum@admin.grnet.gr", + "affiliation": { + "@type": "Organization", + "name": "GRNET S.A" + } + } + ], + "contributor": [ + { + "@type": "Person", + "givenName": "Konstantinos", + "familyName": "Evangelou", + "email": "kevangel@admin.grnet.gr", + "affiliation": { + "@type": "Organization", + "name": "GRNET S.A" + } + } + ] } - From 454c61ba4e03db2e8c0b324f5ca87b2d0ccfb513 Mon Sep 17 00:00:00 2001 From: agelostsal Date: Sun, 1 Oct 2023 00:19:04 +0300 Subject: [PATCH 3/9] AM-38 Introduce MongoDB integration tests in AMS --- README.md | 14 + go.mod | 82 +- go.sum | 632 ++++++++++-- stores/mock.go | 5 + stores/mongo.go | 14 + stores/mongo_store_integration_test.go | 1242 ++++++++++++++++++++++++ stores/store.go | 1 + stores/store_test.go | 10 +- 8 files changed, 1910 insertions(+), 90 deletions(-) create mode 100644 stores/mongo_store_integration_test.go diff --git a/README.md b/README.md index f7e956ca..0d429bb6 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,20 @@ resides in two possible locations: - `auth_option` - (`key`|`header`|`both`), where should the service look for the access token. - `proxy_hostname` - The FQDN of any proxy or load balancer that might serve request in place of the AMS +#### Run the tests + +Inside the project's root directory issue the command: + +```bash +go test ./... +``` + +For the db store integration test suite, +you can issue the command: +```bash +go test ./... -tags integration +``` + #### Build & Run the service In order to build the service, inside the AMS repo issue the command: diff --git a/go.mod b/go.mod index 4ed982a6..8b7a2bfb 100644 --- a/go.mod +++ b/go.mod @@ -4,51 +4,83 @@ go 1.19 require ( github.com/Shopify/sarama v1.22.1 - github.com/golang/protobuf v1.4.2 + github.com/golang/protobuf v1.5.3 github.com/gorilla/context v0.0.0-20150820051245-1c83b3eabd45 github.com/gorilla/handlers v0.0.0-20160816184729-a5775781a543 - github.com/gorilla/mux v0.0.0-20150908165839-49c024275504 + github.com/gorilla/mux v1.8.0 github.com/linkedin/goavro v2.1.0+incompatible github.com/samuel/go-zookeeper v0.0.0-20160616024954-e64db453f351 - github.com/sirupsen/logrus v1.0.6-0.20180625052543-e3292c4c4d7f - github.com/spf13/pflag v1.0.2 - github.com/spf13/viper v1.2.2-0.20181107110859-ae103d7e593e - github.com/stretchr/testify v1.3.0 + github.com/sirupsen/logrus v1.9.0 + github.com/spf13/pflag v1.0.5 + github.com/spf13/viper v1.16.0 + github.com/stretchr/testify v1.8.4 + github.com/testcontainers/testcontainers-go v0.24.1 github.com/twinj/uuid v0.0.0-20150629100731-70cac2bcd273 github.com/xeipuuv/gojsonschema v1.2.0 - google.golang.org/grpc v1.17.0 + google.golang.org/grpc v1.57.0 gopkg.in/mgo.v2 v2.0.0-20160818020120-3f83fa500528 ) require ( - github.com/BurntSushi/toml v0.3.1 // indirect + dario.cat/mergo v1.0.0 // indirect + github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/DataDog/zstd v1.4.0 // indirect + github.com/Microsoft/go-winio v0.6.1 // indirect + github.com/Microsoft/hcsshim v0.11.0 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect + github.com/containerd/containerd v1.7.6 // indirect + github.com/cpuguy83/dockercfg v0.3.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/docker/distribution v2.8.2+incompatible // indirect + github.com/docker/docker v24.0.6+incompatible // indirect + github.com/docker/go-connections v0.4.0 // indirect + github.com/docker/go-units v0.5.0 // indirect github.com/eapache/go-resiliency v1.1.0 // indirect github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect github.com/eapache/queue v1.1.0 // indirect - github.com/fsnotify/fsnotify v1.4.9 // indirect + github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect + github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/snappy v0.0.1 // indirect + github.com/google/uuid v1.3.1 // indirect github.com/hashicorp/hcl v1.0.0 // indirect - github.com/magiconair/properties v1.8.0 // indirect - github.com/mitchellh/mapstructure v1.0.0 // indirect - github.com/pelletier/go-toml v1.2.0 // indirect + github.com/klauspost/compress v1.16.0 // indirect + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect + github.com/magiconair/properties v1.8.7 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/moby/patternmatcher v0.5.0 // indirect + github.com/moby/sys/sequential v0.5.0 // indirect + github.com/moby/term v0.5.0 // indirect + github.com/morikuni/aec v1.0.0 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.0-rc4 // indirect + github.com/opencontainers/runc v1.1.5 // indirect + github.com/pelletier/go-toml/v2 v2.0.8 // indirect github.com/pierrec/lz4 v0.0.0-20190327172049-315a67e90e41 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a // indirect - github.com/spf13/afero v1.1.2 // indirect - github.com/spf13/cast v1.3.0 // indirect - github.com/spf13/jwalterweatherman v1.0.0 // indirect + github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect + github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 // indirect + github.com/shirou/gopsutil/v3 v3.23.7 // indirect + github.com/shoenig/go-m1cpu v0.1.6 // indirect + github.com/spf13/afero v1.9.5 // indirect + github.com/spf13/cast v1.5.1 // indirect + github.com/spf13/jwalterweatherman v1.1.0 // indirect + github.com/subosito/gotenv v1.4.2 // indirect + github.com/tklauser/go-sysconf v0.3.11 // indirect + github.com/tklauser/numcpus v0.6.0 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect - golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5 // indirect - golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7 // indirect - golang.org/x/sys v0.5.0 // indirect - golang.org/x/text v0.3.2 // indirect - google.golang.org/genproto v0.0.0-20181219182458-5a97ab628bfb // indirect - google.golang.org/protobuf v1.23.0 // indirect - gopkg.in/airbrake/gobrake.v2 v2.0.9 // indirect - gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 // indirect + github.com/yusufpapurcu/wmi v1.2.3 // indirect + golang.org/x/exp v0.0.0-20230510235704-dd950f8aeaea // indirect + golang.org/x/mod v0.9.0 // indirect + golang.org/x/net v0.10.0 // indirect + golang.org/x/sys v0.11.0 // indirect + golang.org/x/text v0.9.0 // indirect + golang.org/x/tools v0.7.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect + google.golang.org/protobuf v1.30.0 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/linkedin/goavro.v1 v1.0.5 // indirect - gopkg.in/yaml.v2 v2.3.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index d8bf7a3f..769c1cb0 100644 --- a/go.sum +++ b/go.sum @@ -1,89 +1,302 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= +dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= +dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DataDog/zstd v1.3.6-0.20190409195224-796139022798/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/DataDog/zstd v1.4.0 h1:vhoV+DUHnRZdKW1i5UMjAk2G4JY8wN4ayRfYDNdEhwo= github.com/DataDog/zstd v1.4.0/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= +github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= +github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= +github.com/Microsoft/hcsshim v0.11.0 h1:7EFNIY4igHEXUdj1zXgAyU3fLc7QfOKHbkldRVTBdiM= +github.com/Microsoft/hcsshim v0.11.0/go.mod h1:OEthFdQv/AD2RAdzR6Mm1N1KPCztGKDurW1Z8b8VGMM= github.com/Shopify/sarama v1.22.1 h1:exyEsKLGyCsDiqpV5Lr4slFi8ev2KiM3cP1KZ6vnCQ0= github.com/Shopify/sarama v1.22.1/go.mod h1:FRzlvRpMFO/639zY1SDxUxkqH97Y0ndM5CbGj6oG3As= github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= +github.com/containerd/containerd v1.7.6 h1:oNAVsnhPoy4BTPQivLgTzI9Oleml9l/+eYIDYXRCYo8= +github.com/containerd/containerd v1.7.6/go.mod h1:SY6lrkkuJT40BVNO37tlYTSnKJnP5AXBc0fhx0q+TJ4= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/cpuguy83/dockercfg v0.3.1 h1:/FpZ+JaygUR/lZP2NlFI2DVfrOEMAIKP5wWEJdoYe9E= +github.com/cpuguy83/dockercfg v0.3.1/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= +github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= +github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/docker v24.0.6+incompatible h1:hceabKCtUgDqPu+qm0NgsaXf28Ljf4/pWFL7xjWWDgE= +github.com/docker/docker v24.0.6+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= +github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= +github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/eapache/go-resiliency v1.1.0 h1:1NtRmCAqadE2FN4ZcN6g90TP3uk8cg9rn9eNK2197aU= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= -github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= +github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/gorilla/context v0.0.0-20150820051245-1c83b3eabd45 h1:f69IpTg7NxbCoH51W7oD+gPkojLLvi6ybJ6Iya76wB8= github.com/gorilla/context v0.0.0-20150820051245-1c83b3eabd45/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/handlers v0.0.0-20160816184729-a5775781a543 h1:dAB4uWBz7LFnjYiZpwFhSJ2fqI23B3mE3XJmz22Zc+g= github.com/gorilla/handlers v0.0.0-20160816184729-a5775781a543/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= -github.com/gorilla/mux v0.0.0-20150908165839-49c024275504 h1:+8ZTptweg7pCIymRVkFGnZgZ3ttDt++xgkxyeAwxMPY= -github.com/gorilla/mux v0.0.0-20150908165839-49c024275504/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.16.0 h1:iULayQNOReoYUe+1qtKOqw9CwJv3aNQu8ivo7lw1HU4= +github.com/klauspost/compress v1.16.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/linkedin/goavro v2.1.0+incompatible h1:DV2aUlj2xZiuxQyvag8Dy7zjY69ENjS66bWkSfdpddY= github.com/linkedin/goavro v2.1.0+incompatible/go.mod h1:bBCwI2eGYpUI/4820s67MElg9tdeLbINjLjiM2xZFYM= -github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/mitchellh/mapstructure v1.0.0 h1:vVpGvMXJPqSDh2VYHF7gsfQj8Ncx+Xw5Y1KHeTRY+7I= -github.com/mitchellh/mapstructure v1.0.0/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/moby/patternmatcher v0.5.0 h1:YCZgJOeULcxLw1Q+sVR636pmS7sPEn1Qo2iAN6M7DBo= +github.com/moby/patternmatcher v0.5.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= +github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU= +github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc= +github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo= +github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= +github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.0-rc4 h1:oOxKUJWnFC4YGHCCMNql1x4YaDfYBTS5Y4x/Cgeo1E0= +github.com/opencontainers/image-spec v1.1.0-rc4/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= +github.com/opencontainers/runc v1.1.5 h1:L44KXEpKmfWDcS02aeGm8QNTFXTo2D+8MYGDIJ/GDEs= +github.com/opencontainers/runc v1.1.5/go.mod h1:1J5XiS+vdZ3wCyZybsuxXZWGrgSr8fFJHLXuG2PsnNg= +github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= +github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= +github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= github.com/pierrec/lz4 v0.0.0-20190327172049-315a67e90e41 h1:GeinFsrjWz97fAxVUEd748aV0cYL+I6k44gFJTCVvpU= github.com/pierrec/lz4 v0.0.0-20190327172049-315a67e90e41/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= +github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a h1:9ZKAASQSHhDYGoxY8uLVpewe1GDZ2vu2Tr/vTdVAkFQ= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 h1:MkV+77GLUNo5oJ0jf870itWm3D0Sjh7+Za9gazKc5LQ= +github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/samuel/go-zookeeper v0.0.0-20160616024954-e64db453f351 h1:n98voimum5+t4kn3wZVoJei1TzxX7iibyOZ6FQ5HHb0= github.com/samuel/go-zookeeper v0.0.0-20160616024954-e64db453f351/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= -github.com/sirupsen/logrus v1.0.6-0.20180625052543-e3292c4c4d7f h1:4FzWyuvP8LUKPDTSWuOiqcOO1Orq9OfFytAsx+tDV7c= -github.com/sirupsen/logrus v1.0.6-0.20180625052543-e3292c4c4d7f/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= -github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg= -github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v1.0.2 h1:Fy0orTDgHdbnzHcsOgfCN4LtHf0ec3wwtiwJqwvf3Gc= -github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/viper v1.2.2-0.20181107110859-ae103d7e593e h1:noY5oq1EsXDRpuGImnFLiKBViKVAz6M83DAF3O38fWc= -github.com/spf13/viper v1.2.2-0.20181107110859-ae103d7e593e/go.mod h1:P4AexN0a+C9tGAnUFNwDMYYZv3pjFuvmeiMyKRaNVlI= +github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= +github.com/shirou/gopsutil/v3 v3.23.7 h1:C+fHO8hfIppoJ1WdsVm1RoI0RwXoNdfTK7yWXV0wVj4= +github.com/shirou/gopsutil/v3 v3.23.7/go.mod h1:c4gnmoRC0hQuaLqvxnx1//VXQ0Ms/X9UnJF8pddY5z4= +github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= +github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= +github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= +github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/spf13/afero v1.9.5 h1:stMpOSZFs//0Lv29HduCmli3GUfpFoF3Y1Q/aXj/wVM= +github.com/spf13/afero v1.9.5/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ= +github.com/spf13/cast v1.5.1 h1:R+kOtfhWQE6TVQzY+4D7wJLBgkdVasCEFxSUBYBYIlA= +github.com/spf13/cast v1.5.1/go.mod h1:b9PdjNptOpzXr7Rq1q9gJML/2cdGQAo69NKzQ10KN48= +github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= +github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.16.0 h1:rGGH0XDZhdUOryiDWjmIvUSWpbNqisK8Wk0Vyefw8hc= +github.com/spf13/viper v1.16.0/go.mod h1:yg78JgCJcbrQOvV9YLXgkLaZqUidkY9K+Dd1FofRzQg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= +github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= +github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= +github.com/testcontainers/testcontainers-go v0.24.1 h1:gJdZuQIVWnMJTo+CmQMEP7/CAagNk/0jbcUPn3OWvD8= +github.com/testcontainers/testcontainers-go v0.24.1/go.mod h1:MGBiAkCm86yXQoCiipmQCqZLVdk1uFqtMqaU1Or0MRk= +github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM= +github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI= +github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms= +github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4= github.com/twinj/uuid v0.0.0-20150629100731-70cac2bcd273 h1:YqFyfcgqxQqjpRr0SEG0Z555J/3kPqDL/xmRyeAaX/0= github.com/twinj/uuid v0.0.0-20150629100731-70cac2bcd273/go.mod h1:mMgcE1RHFUFqe5AfiwlINXisXfDGro23fWdPUfOMjRY= +github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= +github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= @@ -93,60 +306,359 @@ github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHo github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= +github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5 h1:bselrhR0Or1vomJZC8ZIjWtbDmn9OYFLX5Ik9alpJpE= golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= -golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20230510235704-dd950f8aeaea h1:vLCWI/yYrdEHyN2JzIzPO3aaQJHQdp89IZBA/+azVC4= +golang.org/x/exp v0.0.0-20230510235704-dd950f8aeaea/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs= +golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7 h1:AeiKBIuRw3UomYXSbLy0Mc2dDLfdtbT/IVn4keq83P0= -golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180906133057-8cf3aee42992/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200519105757-fe76b779f299 h1:DYfZAGf2WMFjMxbgTjaC+2HC7NkNAQs+6Q8b9WEB/F4= -golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= +golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20181219182458-5a97ab628bfb h1:dQshZyyJ5W/Xk8myF4GKBak1pZW6EywJuQ8+44EQhGA= -google.golang.org/genproto v0.0.0-20181219182458-5a97ab628bfb/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg= -google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= -google.golang.org/grpc v1.17.0 h1:TRJYBgMclJvGYn2rIMjj+h9KtMt5r1Ij7ODVRIZkwhk= -google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= +google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -gopkg.in/airbrake/gobrake.v2 v2.0.9 h1:7z2uVWwn7oVeeugY1DtlPAy5H+KYgB1KeKTnqjNatLo= -gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 h1:OAj3g0cR6Dx/R07QgQe8wkA9RNjB2u4i700xBkIT4e0= -gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/linkedin/goavro.v1 v1.0.5 h1:BJa69CDh0awSsLUmZ9+BowBdokpduDZSM9Zk8oKHfN4= gopkg.in/linkedin/goavro.v1 v1.0.5/go.mod h1:Aw5GdAbizjOEl0kAMHV9iHmA8reZzW/OKuJAl4Hb9F0= gopkg.in/mgo.v2 v2.0.0-20160818020120-3f83fa500528 h1:/saqWwm73dLmuzbNhe92F0QsZ/KiFND+esHco2v1hiY= gopkg.in/mgo.v2 v2.0.0-20160818020120-3f83fa500528/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.5.0 h1:Ljk6PdHdOhAb5aDMWXjDLMMhph+BpztA4v1QdqEW2eY= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/stores/mock.go b/stores/mock.go index 7f26bd31..d25869c4 100644 --- a/stores/mock.go +++ b/stores/mock.go @@ -1008,6 +1008,11 @@ func (mk *MockStore) Initialize() { mk.UserRegistrations = append(mk.UserRegistrations, ur1) } +func (mk *MockStore) InsertResourceRoles(ctx context.Context, resource string, roles []string) error { + mk.RoleList = append(mk.RoleList, QRole{Name: resource, Roles: roles}) + return nil +} + func (mk *MockStore) QueryTotalMessagesPerProject(ctx context.Context, projectUUIDs []string, startDate time.Time, endDate time.Time) ([]QProjectMessageCount, error) { projectCount := make(map[string]int64) diff --git a/stores/mongo.go b/stores/mongo.go index 228f22e8..b56d590d 100644 --- a/stores/mongo.go +++ b/stores/mongo.go @@ -1099,6 +1099,20 @@ func (mong *MongoStore) HasResourceRoles(ctx context.Context, resource string, r } +func (mong *MongoStore) InsertResourceRoles(ctx context.Context, resource string, roles []string) error { + db := mong.Session.DB(mong.Database) + c := db.C("roles") + role := QRole{ + Name: resource, + Roles: roles, + } + err := c.Insert(role) + if err != nil { + mong.logErrorAndCrash(ctx, "InsertResourceRoles", err) + } + return nil +} + // GetAllRoles returns a list of all available roles func (mong *MongoStore) GetAllRoles(ctx context.Context) []string { diff --git a/stores/mongo_store_integration_test.go b/stores/mongo_store_integration_test.go new file mode 100644 index 00000000..f050977b --- /dev/null +++ b/stores/mongo_store_integration_test.go @@ -0,0 +1,1242 @@ +//go:build integration + +package stores + +import ( + "context" + "errors" + "fmt" + "github.com/stretchr/testify/suite" + "github.com/testcontainers/testcontainers-go" + "gopkg.in/mgo.v2/bson" + "testing" + "time" +) + +// mongodbContainer represents the mongodb container type used in the module +type mongodbContainer struct { + testcontainers.Container +} + +// startContainer creates an instance of the mongodb container type +func startContainer(ctx context.Context) (*mongodbContainer, error) { + + req := testcontainers.ContainerRequest{ + Name: "mongodb-4.2-ams", + Image: "mongo:4.2", + ExposedPorts: []string{"27017/tcp"}, + } + container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: req, + Started: true, + }) + if err != nil { + return nil, err + } + + return &mongodbContainer{Container: container}, nil +} + +type MongoStoreIntegrationTestSuite struct { + suite.Suite + store Store + ctx context.Context + UserRegistrations []QUserRegistration + SubList []QSub + TopicList []QTopic + DailyTopicMsgCount []QDailyTopicMsgCount + ProjectList []QProject + UserList []QUser + RoleList []QRole + SchemaList []QSchema + Session bool + TopicsACL map[string]QAcl + SubsACL map[string]QAcl + OpMetrics map[string]QopMetric +} + +type QListReverser[T any] struct{} + +func (suite *QListReverser[T]) reverse(s []T) []T { + a := make([]T, len(s)) + copy(a, s) + + for i := len(a)/2 - 1; i >= 0; i-- { + opp := len(a) - 1 - i + a[i], a[opp] = a[opp], a[i] + } + + return a +} + +func (suite *MongoStoreIntegrationTestSuite) assertSchemasEqual(expected []QSchema, actual []QSchema) { + suite.True(len(actual) == len(expected)) + for idx, schema := range expected { + suite.Equal(schema.UUID, actual[idx].UUID, schema.Name) + suite.Equal(schema.Name, actual[idx].Name, schema.Name) + suite.Equal(schema.ProjectUUID, actual[idx].ProjectUUID, schema.Name) + suite.Equal(schema.Type, actual[idx].Type, schema.Name) + suite.Equal(schema.RawSchema, actual[idx].RawSchema, schema.Name) + } +} + +func (suite *MongoStoreIntegrationTestSuite) assertUsersEqual(expected []QUser, actual []QUser) { + suite.True(len(actual) == len(expected)) + for idx, user := range expected { + suite.Equal(user.UUID, actual[idx].UUID, user.Name) + suite.Equal(user.Name, actual[idx].Name, user.Name) + suite.Equal(user.Projects, actual[idx].Projects, user.Name) + suite.Equal(user.LastName, actual[idx].LastName, user.Name) + suite.Equal(user.FirstName, actual[idx].FirstName, user.Name) + suite.Equal(user.Organization, actual[idx].Organization, user.Name) + suite.Equal(user.Description, actual[idx].Description, user.Name) + suite.Equal(user.Email, actual[idx].Email, user.Name) + suite.Equal(user.ServiceRoles, actual[idx].ServiceRoles, user.Name) + suite.Equal(user.CreatedBy, actual[idx].CreatedBy, user.Name) + suite.Equal(user.CreatedOn, actual[idx].CreatedOn, user.Name) + suite.Equal(user.ModifiedOn, actual[idx].ModifiedOn, user.Name) + suite.Equal(user.Token, actual[idx].Token, user.Name) + suite.True((actual[idx].ID.(bson.ObjectId)).Valid(), user.Name) + } +} + +func (suite *MongoStoreIntegrationTestSuite) assertTopicsEqual(expected []QTopic, actual []QTopic) { + suite.True(len(actual) == len(expected)) + for idx, topic := range expected { + suite.Equal(topic.Name, actual[idx].Name, topic.Name) + suite.Equal(topic.ProjectUUID, actual[idx].ProjectUUID, topic.Name) + suite.Equal(topic.SchemaUUID, actual[idx].SchemaUUID, topic.Name) + suite.Equal(topic.PublishRate, actual[idx].PublishRate, topic.Name) + suite.Equal(topic.LatestPublish, actual[idx].LatestPublish, topic.Name) + suite.Equal(topic.CreatedOn, actual[idx].CreatedOn, topic.Name) + suite.Equal(topic.MsgNum, actual[idx].MsgNum, topic.Name) + suite.Equal(topic.TotalBytes, actual[idx].TotalBytes, topic.Name) + suite.Equal(topic.ACL, actual[idx].ACL, topic.Name) + suite.True((actual[idx].ID.(bson.ObjectId)).Valid(), topic.Name) + } +} + +func (suite *MongoStoreIntegrationTestSuite) assertSubsEqual(expected []QSub, actual []QSub) { + suite.True(len(actual) == len(expected)) + for idx, sub := range expected { + suite.Equal(sub.Name, actual[idx].Name, sub.Name) + suite.Equal(sub.ProjectUUID, actual[idx].ProjectUUID, sub.Name) + suite.Equal(sub.MattermostChannel, actual[idx].MattermostChannel, sub.Name) + suite.Equal(sub.MattermostUrl, actual[idx].MattermostUrl, sub.Name) + suite.Equal(sub.MattermostUsername, actual[idx].MattermostUsername, sub.Name) + suite.Equal(sub.ConsumeRate, actual[idx].ConsumeRate, sub.Name) + suite.Equal(sub.LatestConsume, actual[idx].LatestConsume, sub.Name) + suite.Equal(sub.CreatedOn, actual[idx].CreatedOn, sub.Name) + suite.Equal(sub.MsgNum, actual[idx].MsgNum, sub.Name) + suite.Equal(sub.TotalBytes, actual[idx].TotalBytes, sub.Name) + suite.Equal(sub.ACL, actual[idx].ACL, sub.Name) + suite.True((actual[idx].ID.(bson.ObjectId)).Valid(), sub.Name) + suite.Equal(sub.PushType, actual[idx].PushType, sub.Name) + suite.Equal(sub.PushEndpoint, actual[idx].PushEndpoint, sub.Name) + suite.Equal(sub.AuthorizationType, actual[idx].AuthorizationType, sub.Name) + suite.Equal(sub.RetPolicy, actual[idx].RetPolicy, sub.Name) + suite.Equal(sub.RetPeriod, actual[idx].RetPeriod, sub.Name) + suite.Equal(sub.Base64Decode, actual[idx].Base64Decode, sub.Name) + suite.Equal(sub.AuthorizationHeader, actual[idx].AuthorizationHeader, sub.Name) + suite.Equal(sub.MaxMessages, actual[idx].MaxMessages, sub.Name) + suite.Equal(sub.VerificationHash, actual[idx].VerificationHash, sub.Name) + suite.Equal(sub.Verified, actual[idx].Verified, sub.Name) + } +} + +func (suite *MongoStoreIntegrationTestSuite) initDB() { + + created := time.Date(2009, time.November, 10, 23, 0, 0, 0, time.Local) + modified := time.Date(2009, time.November, 10, 23, 0, 0, 0, time.Local) + + // populate Projects + qPr := QProject{ + UUID: "argo_uuid", + Name: "ARGO", + CreatedOn: created, + ModifiedOn: modified, + CreatedBy: "uuid1", + Description: "simple project", + } + qPr2 := QProject{ + UUID: "argo_uuid2", + Name: "ARGO2", + CreatedOn: created, + ModifiedOn: modified, + CreatedBy: "uuid1", + Description: "simple project", + } + suite.ProjectList = append(suite.ProjectList, qPr, qPr2) + for _, project := range suite.ProjectList { + err := suite.store.InsertProject(suite.ctx, project.UUID, project.Name, + project.CreatedOn, project.ModifiedOn, project.CreatedBy, + project.Description) + if err != nil { + panic("could not insert project") + } + } + + qTopicACL01 := QAcl{[]string{"uuid1", "uuid2"}} + qTopicACL02 := QAcl{[]string{"uuid1", "uuid2", "uuid4"}} + qTopicACL03 := QAcl{[]string{"uuid3"}} + + qSubACL01 := QAcl{[]string{"uuid1", "uuid2"}} + qSubACL02 := QAcl{[]string{"uuid1", "uuid3"}} + qSubACL03 := QAcl{[]string{"uuid4", "uuid2", "uuid1"}} + qSubACL04 := QAcl{[]string{"uuid2", "uuid4", "uuid7"}} + + suite.TopicsACL = make(map[string]QAcl) + suite.SubsACL = make(map[string]QAcl) + suite.TopicsACL["topic1"] = qTopicACL01 + suite.TopicsACL["topic2"] = qTopicACL02 + suite.TopicsACL["topic3"] = qTopicACL03 + suite.SubsACL["sub1"] = qSubACL01 + suite.SubsACL["sub2"] = qSubACL02 + suite.SubsACL["sub3"] = qSubACL03 + suite.SubsACL["sub4"] = qSubACL04 + + // populate topics + qtop4 := QTopic{ + ProjectUUID: "argo_uuid", + Name: "topic4", + MsgNum: 0, + TotalBytes: 0, + LatestPublish: time.Date(1970, time.January, 1, 2, 0, 0, 0, time.Local), + PublishRate: 0, + SchemaUUID: "", + CreatedOn: time.Date(2020, 11, 19, 0, 0, 0, 0, time.Local), + ACL: []string{}, + } + qtop3 := QTopic{ + ProjectUUID: "argo_uuid", + Name: "topic3", + MsgNum: 0, + TotalBytes: 0, + LatestPublish: time.Date(2019, 5, 7, 0, 0, 0, 0, time.Local), + PublishRate: 8.99, + SchemaUUID: "schema_uuid_3", + CreatedOn: time.Date(2020, 11, 20, 0, 0, 0, 0, time.Local), + ACL: qTopicACL03.ACL, + } + qtop2 := QTopic{ + ProjectUUID: "argo_uuid", + Name: "topic2", + MsgNum: 0, + TotalBytes: 0, + LatestPublish: time.Date(2019, 5, 8, 0, 0, 0, 0, time.Local), + PublishRate: 5.45, + SchemaUUID: "schema_uuid_1", + CreatedOn: time.Date(2020, 11, 21, 0, 0, 0, 0, time.Local), + ACL: qTopicACL02.ACL, + } + qtop1 := QTopic{ + ProjectUUID: "argo_uuid", + Name: "topic1", + MsgNum: 0, + TotalBytes: 0, + LatestPublish: time.Date(2019, 5, 6, 0, 0, 0, 0, time.Local), + PublishRate: 10, + SchemaUUID: "", + CreatedOn: time.Date(2020, 11, 22, 0, 0, 0, 0, time.Local), + ACL: qTopicACL01.ACL, + } + suite.TopicList = append(suite.TopicList, qtop1) + suite.TopicList = append(suite.TopicList, qtop2) + suite.TopicList = append(suite.TopicList, qtop3) + suite.TopicList = append(suite.TopicList, qtop4) + for _, qTopic := range suite.TopicList { + err := suite.store.InsertTopic(suite.ctx, qTopic.ProjectUUID, qTopic.Name, qTopic.SchemaUUID, qTopic.CreatedOn) + if err != nil { + panic("could not insert topics") + } + err = suite.store.ModACL(suite.ctx, qTopic.ProjectUUID, "topics", qTopic.Name, qTopic.ACL) + if err != nil { + panic("could not mod topics acl") + } + if qTopic.SchemaUUID != "" { + err = suite.store.LinkTopicSchema(suite.ctx, qTopic.ProjectUUID, qTopic.Name, qTopic.SchemaUUID) + if err != nil { + panic("could not link topic schema") + } + } + + if qTopic.PublishRate != 0 { + err = suite.store.UpdateTopicPublishRate(suite.ctx, qTopic.ProjectUUID, qTopic.Name, qTopic.PublishRate) + if err != nil { + panic("could not update topic publishing rate") + } + } + + if !qTopic.LatestPublish.IsZero() { + err = suite.store.UpdateTopicLatestPublish(suite.ctx, qTopic.ProjectUUID, qTopic.Name, qTopic.LatestPublish) + if err != nil { + panic("could not update topic latest publish") + } + } + } + + // populate Subscriptions + qsub1 := QSub{ + ID: 0, + ProjectUUID: "argo_uuid", + Name: "sub1", + Topic: "topic1", + Ack: 10, + LatestConsume: time.Date(2019, 5, 6, 0, 0, 0, 0, time.Local), + ConsumeRate: 10, + CreatedOn: time.Date(2020, 11, 19, 0, 0, 0, 0, time.Local), + ACL: qSubACL01.ACL, + } + + qsub2 := QSub{ + ID: 1, + ProjectUUID: "argo_uuid", + Name: "sub2", + Topic: "topic2", + Ack: 10, + LatestConsume: time.Date(2019, 5, 7, 0, 0, 0, 0, time.Local), + ConsumeRate: 8.99, + CreatedOn: time.Date(2020, 11, 20, 0, 0, 0, 0, time.Local), + ACL: qSubACL02.ACL, + } + + qsub3 := QSub{ + ID: 2, + ProjectUUID: "argo_uuid", + Name: "sub3", + Topic: "topic3", + Ack: 10, + LatestConsume: time.Date(2019, 5, 8, 0, 0, 0, 0, time.Local), + ConsumeRate: 5.45, + CreatedOn: time.Date(2020, 11, 21, 0, 0, 0, 0, time.Local), + ACL: qSubACL03.ACL, + } + + qsub4 := QSub{ + ID: 3, + ProjectUUID: "argo_uuid", + Name: "sub4", + Topic: "topic4", + PushType: "http_endpoint", + PushEndpoint: "endpoint.foo", + MaxMessages: 1, + AuthorizationType: "autogen", + AuthorizationHeader: "auth-header-1", + Ack: 10, + RetPolicy: "linear", + RetPeriod: 300, + VerificationHash: "push-id-1", + Verified: true, + Base64Decode: true, + CreatedOn: time.Date(2020, 11, 22, 0, 0, 0, 0, time.Local), + ACL: qSubACL04.ACL, + } + + suite.SubList = append(suite.SubList, qsub1) + suite.SubList = append(suite.SubList, qsub2) + suite.SubList = append(suite.SubList, qsub3) + suite.SubList = append(suite.SubList, qsub4) + for _, qSub := range suite.SubList { + pushCfg := QPushConfig{ + Type: qSub.PushType, + PushEndpoint: qSub.PushEndpoint, + MaxMessages: qSub.MaxMessages, + AuthorizationType: qSub.AuthorizationType, + AuthorizationHeader: qSub.AuthorizationHeader, + RetPolicy: qSub.RetPolicy, + RetPeriod: qSub.RetPeriod, + VerificationHash: qSub.VerificationHash, + Verified: qSub.Verified, + MattermostUrl: qSub.MattermostUrl, + MattermostUsername: qSub.MattermostUsername, + MattermostChannel: qSub.MattermostChannel, + Base64Decode: qSub.Base64Decode, + } + err := suite.store.InsertSub(suite.ctx, qSub.ProjectUUID, qSub.Name, qSub.Topic, + qSub.Offset, qSub.Ack, pushCfg, qSub.CreatedOn) + if err != nil { + panic("could not insert subs") + } + err = suite.store.ModACL(suite.ctx, qSub.ProjectUUID, "subscriptions", qSub.Name, qSub.ACL) + if err != nil { + panic("could not mod subs acl") + } + + if qSub.ConsumeRate != 0 { + err = suite.store.UpdateSubConsumeRate(suite.ctx, qSub.ProjectUUID, qSub.Name, qSub.ConsumeRate) + if err != nil { + panic("could not update sub consume rate") + } + } + + if !qSub.LatestConsume.IsZero() { + err = suite.store.UpdateSubLatestConsume(suite.ctx, qSub.ProjectUUID, qSub.Name, qSub.LatestConsume) + if err != nil { + panic("could not update sub latest consume") + } + } + + } + + // populate schemas + //{ + // "type": "object", + // "properties": { + // "name": { "type": "string" }, + // "email": { "type": "string" }, + // "address": { "type": "string" }, + // "telephone": { "type": "string" } + // }, + // "required": ["name", "email"] + //} + //} + // the above schema base64 encoded + s := "eyJwcm9wZXJ0aWVzIjp7ImFkZHJlc3MiOnsidHlwZSI6InN0cmluZyJ9LCJlbWFpbCI6eyJ0eXBlIjoic3RyaW5nIn0sIm5hbWUiOnsidHlwZSI6InN0cmluZyJ9LCJ0ZWxlcGhvbmUiOnsidHlwZSI6InN0cmluZyJ9fSwicmVxdWlyZWQiOlsibmFtZSIsImVtYWlsIl0sInR5cGUiOiJvYmplY3QifQ==" + qSchema1 := QSchema{ + UUID: "schema_uuid_1", + ProjectUUID: "argo_uuid", + Type: "json", + Name: "schema-1", + RawSchema: s, + } + qSchema2 := QSchema{ + UUID: "schema_uuid_2", + ProjectUUID: "argo_uuid", + Type: "json", + Name: "schema-2", + RawSchema: s, + } + // { + // "namespace": "user.avro", + // "type": "record", + // "name": "User", + // "fields": [ + // {"name": "username", "type":"string"}, + // {"name": "phone", "type": "int"} + // ] + // } + avros := "eyJmaWVsZHMiOlt7Im5hbWUiOiJ1c2VybmFtZSIsInR5cGUiOiJzdHJpbmcifSx7Im5hbWUiOiJwaG9uZSIsInR5cGUiOiJpbnQifV0sIm5hbWUiOiJVc2VyIiwibmFtZXNwYWNlIjoidXNlci5hdnJvIiwidHlwZSI6InJlY29yZCJ9" + qSchema3 := QSchema{ + UUID: "schema_uuid_3", + ProjectUUID: "argo_uuid", + Type: "avro", + Name: "schema-3", + RawSchema: avros, + } + suite.SchemaList = append(suite.SchemaList, qSchema1, qSchema2, qSchema3) + for _, qSchema := range suite.SchemaList { + err := suite.store.InsertSchema(suite.ctx, qSchema.ProjectUUID, qSchema.UUID, qSchema.Name, qSchema.Type, qSchema.RawSchema) + if err != nil { + panic("could not insert schema") + } + } + + // populate daily msg count for topics + dc1 := QDailyTopicMsgCount{ + Date: time.Date(2018, 10, 1, 0, 0, 0, 0, time.Local), + ProjectUUID: "argo_uuid", + TopicName: "topic1", + NumberOfMessages: 40, + } + dc2 := QDailyTopicMsgCount{ + Date: time.Date(2018, 10, 2, 0, 0, 0, 0, time.Local), + ProjectUUID: "argo_uuid", + TopicName: "topic1", + NumberOfMessages: 30, + } + dc3 := QDailyTopicMsgCount{ + Date: time.Date(2018, 10, 1, 0, 0, 0, 0, time.Local), + ProjectUUID: "argo_uuid", + TopicName: "topic2", + NumberOfMessages: 70, + } + dc4 := QDailyTopicMsgCount{ + Date: time.Date(2018, 10, 1, 0, 0, 0, 0, time.Local), + ProjectUUID: "argo_uuid", + TopicName: "topic3", + } + + suite.DailyTopicMsgCount = append(suite.DailyTopicMsgCount, dc2, dc1, dc3, dc4) + for _, dc := range suite.DailyTopicMsgCount { + err := suite.store.IncrementDailyTopicMsgCount(suite.ctx, dc.ProjectUUID, dc.TopicName, dc.NumberOfMessages, dc.Date) + if err != nil { + panic(" could not increment daily message count") + } + } + + // populate Users + qRole := []QProjectRoles{{ + ProjectUUID: "argo_uuid", + Roles: []string{"consumer", "publisher"}, + }} + qRoleB := []QProjectRoles{{ + ProjectUUID: "argo_uuid2", + Roles: []string{"consumer", "publisher"}, + }} + qUsr := QUser{ + ID: 0, + UUID: "uuid0", + Projects: qRole, + Name: "Test", + Token: "S3CR3T", + Email: "Test@test.com", + ServiceRoles: []string{}, + CreatedOn: created, + ModifiedOn: modified, + } + + suite.UserList = append(suite.UserList, qUsr) + + qRoleConsumerPub := []QProjectRoles{{ + "argo_uuid", + []string{"publisher", "consumer"}, + }} + + suite.UserList = append(suite.UserList, QUser{ + ID: 1, + UUID: "uuid1", + Projects: qRole, + Name: "UserA", + FirstName: "FirstA", + LastName: "LastA", + Organization: "OrgA", + Description: "DescA", + Token: "S3CR3T1", + Email: "foo-email", + ServiceRoles: []string{}, + CreatedOn: created, + ModifiedOn: modified, + }) + suite.UserList = append(suite.UserList, QUser{ + ID: 2, + UUID: "uuid2", + Projects: qRole, + Name: "UserB", + Token: "S3CR3T2", + Email: "foo-email", + ServiceRoles: []string{}, + CreatedOn: created, + ModifiedOn: modified, + CreatedBy: "uuid1", + }) + suite.UserList = append(suite.UserList, QUser{ + ID: 3, + UUID: "uuid3", + Projects: qRoleConsumerPub, + Name: "UserX", + Token: "S3CR3T3", + Email: "foo-email", + ServiceRoles: []string{}, + CreatedOn: created, + ModifiedOn: modified, + CreatedBy: "uuid1", + }) + suite.UserList = append(suite.UserList, QUser{ + ID: 4, + UUID: "uuid4", + Projects: qRoleConsumerPub, + Name: "UserZ", + Token: "S3CR3T4", + Email: "foo-email", + ServiceRoles: []string{}, + CreatedOn: created, + ModifiedOn: modified, + CreatedBy: "uuid1", + }) + suite.UserList = append(suite.UserList, QUser{ + ID: 5, + UUID: "same_uuid", + Projects: qRoleConsumerPub, + Name: "UserSame1", + Token: "S3CR3T41", + Email: "foo-email", + ServiceRoles: []string{}, + CreatedOn: created, + ModifiedOn: modified, + CreatedBy: "uuid1", + }) + suite.UserList = append(suite.UserList, QUser{ + ID: 6, + UUID: "same_uuid", + Projects: qRoleConsumerPub, + Name: "UserSame2", + Token: "S3CR3T42", + Email: "foo-email", + ServiceRoles: []string{}, + CreatedOn: created, + ModifiedOn: modified, + CreatedBy: "uuid1", + }) + suite.UserList = append(suite.UserList, QUser{ + ID: 7, + UUID: "uuid7", + Projects: []QProjectRoles{}, + Name: "push_worker_0", + Token: "push_token", + Email: "foo-email", + ServiceRoles: []string{"push_worker"}, + CreatedOn: created, + ModifiedOn: modified, + }) + suite.UserList = append(suite.UserList, QUser{ + ID: 8, + UUID: "uuid8", + Projects: qRoleB, + Name: "UserZ", + Token: "S3CR3T1", + Email: "foo-email", + ServiceRoles: []string{}, + CreatedOn: created, + ModifiedOn: modified, + }) + + for _, qUser := range suite.UserList { + err := suite.store.InsertUser(suite.ctx, qUser.UUID, qUser.Projects, qUser.Name, qUser.FirstName, qUser.LastName, + qUser.Organization, qUser.Description, qUser.Token, qUser.Email, qUser.ServiceRoles, + qUser.CreatedOn, qUser.ModifiedOn, qUser.CreatedBy) + if err != nil { + panic("could not insert user") + } + } + + qRole1 := QRole{"topics:list_all", []string{"admin", "reader", "publisher"}} + qRole2 := QRole{"topics:publish", []string{"admin", "publisher"}} + suite.RoleList = append(suite.RoleList, qRole1, qRole2) + for _, role := range suite.RoleList { + err := suite.store.InsertResourceRoles(suite.ctx, role.Name, role.Roles) + if err != nil { + panic("could not insert roles") + } + } + + // Populate user registrations + ur1 := QUserRegistration{ + UUID: "ur-uuid1", + Name: "urname", + FirstName: "urfname", + LastName: "urlname", + Organization: "urorg", + Description: "urdesc", + Email: "uremail", + ActivationToken: "uratkn-1", + Status: "pending", + RegisteredAt: "2019-05-12T22:26:58Z", + ModifiedBy: "uuid1", + ModifiedAt: "2020-05-15T22:26:58Z", + } + + suite.UserRegistrations = append(suite.UserRegistrations, ur1) + for _, ur := range suite.UserRegistrations { + _ = suite.store.RegisterUser(suite.ctx, ur.UUID, ur.Name, ur.FirstName, ur.LastName, ur.Email, + ur.Organization, ur.Description, ur.RegisteredAt, ur.ActivationToken, ur.Status) + } + +} + +func (suite *MongoStoreIntegrationTestSuite) TestQueryTopics() { + + qTopicListReverser := QListReverser[QTopic]{} + + // retrieve all topics + tpList, ts1, pg1, _ := suite.store.QueryTopics(suite.ctx, "argo_uuid", "", "", "", 0) + suite.assertTopicsEqual(qTopicListReverser.reverse(suite.TopicList), tpList) + suite.Equal(int64(4), ts1) + suite.Equal("", pg1) + + // retrieve first 2 + eTopList1st2 := []QTopic{suite.TopicList[3], suite.TopicList[2]} + tpList2, ts2, pg2, _ := suite.store.QueryTopics(suite.ctx, "argo_uuid", "", "", "", 2) + suite.assertTopicsEqual(eTopList1st2, tpList2) + suite.Equal(int64(4), ts2) + suite.True(bson.IsObjectIdHex(pg2)) + + // retrieve the next one + eTopList3 := []QTopic{suite.TopicList[1]} + tpList3, ts3, pg3, _ := suite.store.QueryTopics(suite.ctx, "argo_uuid", "", "", pg2, 1) + suite.assertTopicsEqual(eTopList3, tpList3) + suite.Equal(int64(4), ts3) + suite.True(bson.IsObjectIdHex(pg3)) + + // retrieve a single topic + eTopList4 := []QTopic{suite.TopicList[0]} + tpList4, ts4, pg4, _ := suite.store.QueryTopics(suite.ctx, "argo_uuid", "", "topic1", "", 0) + suite.assertTopicsEqual(eTopList4, tpList4) + suite.Equal(int64(0), ts4) + suite.Equal("", pg4) + + // retrieve user's topics + eTopList5 := []QTopic{suite.TopicList[1], suite.TopicList[0]} + tpList5, ts5, pg5, _ := suite.store.QueryTopics(suite.ctx, "argo_uuid", "uuid1", "", "", 0) + suite.assertTopicsEqual(eTopList5, tpList5) + suite.Equal(int64(2), ts5) + suite.Equal("", pg5) + + // retrieve use's topic with pagination + eTopList6 := []QTopic{suite.TopicList[1]} + tpList6, ts6, pg6, _ := suite.store.QueryTopics(suite.ctx, "argo_uuid", "uuid1", "", "", 1) + suite.assertTopicsEqual(eTopList6, tpList6) + suite.Equal(int64(2), ts6) + suite.True(bson.IsObjectIdHex(pg6)) +} + +func (suite *MongoStoreIntegrationTestSuite) TestQuerySubs() { + + qSubcListReverser := QListReverser[QSub]{} + + subList, ts1, pg1, err1 := suite.store.QuerySubs(suite.ctx, "argo_uuid", "", "", "", 0) + suite.assertSubsEqual(qSubcListReverser.reverse(suite.SubList), subList) + suite.Equal(int64(4), ts1) + suite.Equal("", pg1) + + // retrieve first 2 subs + eSubListFirstPage := []QSub{ + suite.SubList[3], + suite.SubList[2], + } + subList2, ts2, pg2, err2 := suite.store.QuerySubs(suite.ctx, "argo_uuid", "", "", "", 2) + suite.assertSubsEqual(eSubListFirstPage, subList2) + suite.Equal(int64(4), ts2) + suite.True(bson.IsObjectIdHex(pg2)) + + // retrieve next 2 subs + eSubListNextPage := []QSub{ + suite.SubList[1], + suite.SubList[0], + } + + subList3, ts3, pg3, err3 := suite.store.QuerySubs(suite.ctx, "argo_uuid", "", "", pg2, 2) + suite.assertSubsEqual(eSubListNextPage, subList3) + suite.Equal(int64(4), ts3) + suite.Equal("", pg3) + + // retrieve user's subs + eSubList4 := []QSub{ + suite.SubList[2], + suite.SubList[1], + suite.SubList[0], + } + + subList4, ts4, pg4, err4 := suite.store.QuerySubs(suite.ctx, "argo_uuid", "uuid1", "", "", 0) + + suite.Equal(int64(3), ts4) + suite.Equal("", pg4) + suite.assertSubsEqual(eSubList4, subList4) + + // retrieve user's subs + eSubList5 := []QSub{ + suite.SubList[2], + suite.SubList[1], + } + subList5, ts5, pg5, err5 := suite.store.QuerySubs(suite.ctx, "argo_uuid", "uuid1", "", "", 2) + + suite.Equal(int64(3), ts5) + suite.True(bson.IsObjectIdHex(pg5)) + suite.assertSubsEqual(eSubList5, subList5) + + suite.Nil(err1) + suite.Nil(err2) + suite.Nil(err3) + suite.Nil(err4) + suite.Nil(err5) + + // test retrieve subs by topic + subListByTopic, errSublistByTopic := suite.store.QuerySubsByTopic(suite.ctx, "argo_uuid", "topic1") + suite.assertSubsEqual([]QSub{suite.SubList[0]}, subListByTopic) + suite.Nil(errSublistByTopic) + + sb, err := suite.store.QueryOneSub(suite.ctx, "argo_uuid", "sub1") + suite.assertSubsEqual([]QSub{suite.SubList[0]}, []QSub{sb}) + suite.Nil(err) +} + +func (suite *MongoStoreIntegrationTestSuite) TestDailyTopicMsgCount() { + + // check query all + qdsAll, _ := suite.store.QueryDailyTopicMsgCount(suite.ctx, "", "", time.Time{}) + suite.Equal(suite.DailyTopicMsgCount, qdsAll) + + // test daily count + _ = suite.store.IncrementDailyTopicMsgCount(suite.ctx, "argo_uuid", "topic1", 40, time.Date(2018, 10, 1, 0, 0, 0, 0, time.Local)) + qds, _ := suite.store.QueryDailyTopicMsgCount(suite.ctx, "argo_uuid", "topic1", time.Date(2018, 10, 1, 0, 0, 0, 0, time.Local)) + suite.Equal(int64(80), qds[0].NumberOfMessages) + + // check if it was inserted since it wasn't present + _ = suite.store.IncrementDailyTopicMsgCount(suite.ctx, "argo_uuid", "some_other_topic", 70, time.Date(2018, 10, 1, 0, 0, 0, 0, time.Local)) + qds2, _ := suite.store.QueryDailyTopicMsgCount(suite.ctx, "argo_uuid", "some_other_topic", time.Date(2018, 10, 1, 0, 0, 0, 0, time.Local)) + suite.Equal(int64(70), qds2[0].NumberOfMessages) +} + +func (suite *MongoStoreIntegrationTestSuite) TestHasResourceRoles() { + suite.True(suite.store.HasResourceRoles(suite.ctx, "topics:list_all", []string{"admin"})) + suite.True(suite.store.HasResourceRoles(suite.ctx, "topics:list_all", []string{"admin", "reader"})) + suite.True(suite.store.HasResourceRoles(suite.ctx, "topics:list_all", []string{"admin", "foo"})) + suite.False(suite.store.HasResourceRoles(suite.ctx, "topics:list_all", []string{"foo"})) + suite.False(suite.store.HasResourceRoles(suite.ctx, "topics:publish", []string{"reader"})) + suite.True(suite.store.HasResourceRoles(suite.ctx, "topics:list_all", []string{"admin"})) + suite.True(suite.store.HasResourceRoles(suite.ctx, "topics:list_all", []string{"publisher"})) + suite.True(suite.store.HasResourceRoles(suite.ctx, "topics:publish", []string{"publisher"})) + +} + +func (suite *MongoStoreIntegrationTestSuite) TestHasProject() { + suite.True(suite.store.HasProject(suite.ctx, "ARGO")) + suite.False(suite.store.HasProject(suite.ctx, "FOO")) +} + +func (suite *MongoStoreIntegrationTestSuite) TestGetUserRoles() { + roles01, _ := suite.store.GetUserRoles(suite.ctx, "argo_uuid", "S3CR3T") + roles02, _ := suite.store.GetUserRoles(suite.ctx, "argo_uuid", "SecretKey") + suite.Equal([]string{"consumer", "publisher"}, roles01) + suite.Equal([]string{}, roles02) +} + +func (suite *MongoStoreIntegrationTestSuite) TestRemoveSub() { + _ = suite.store.InsertSub(suite.ctx, "argo_uuid", "subFresh", "topicFresh", 0, 10, QPushConfig{}, time.Date(2020, 12, 19, 0, 0, 0, 0, time.Local)) + err := suite.store.RemoveSub(suite.ctx, "argo_uuid", "subFresh") + suite.Equal(nil, err) + subList, _, _, _ := suite.store.QuerySubs(suite.ctx, "argo_uuid", "", "", "", 0) + suite.Equal(4, len(subList)) + err = suite.store.RemoveSub(suite.ctx, "argo_uuid", "subFresh") + suite.Equal("not found", err.Error()) +} + +func (suite *MongoStoreIntegrationTestSuite) TestRemoveTopic() { + _ = suite.store.InsertTopic(suite.ctx, "argo_uuid", "topicFresh", "", time.Date(2020, 9, 11, 0, 0, 0, 0, time.Local)) + err := suite.store.RemoveTopic(suite.ctx, "argo_uuid", "topicFresh") + suite.Equal(nil, err) + qTopicListReverser := QListReverser[QTopic]{} + tpList, _, _, _ := suite.store.QueryTopics(suite.ctx, "argo_uuid", "", "", "", 0) + suite.assertTopicsEqual(qTopicListReverser.reverse(suite.TopicList), tpList) + err = suite.store.RemoveTopic(suite.ctx, "argo_uuid", "topicFresh") + suite.Equal("not found", err.Error()) +} + +func (suite *MongoStoreIntegrationTestSuite) TestModAck() { + _ = suite.store.ModAck(suite.ctx, "argo_uuid", "sub1", 66) + subAck, _ := suite.store.QueryOneSub(suite.ctx, "argo_uuid", "sub1") + suite.Equal(66, subAck.Ack) +} + +func (suite *MongoStoreIntegrationTestSuite) TestModPushSub() { + _ = suite.store.InsertSub(suite.ctx, "argo_uuid", "subFresh", "topicFresh", 0, 10, QPushConfig{}, time.Date(2020, 12, 19, 0, 0, 0, 0, time.Local)) + qCfg := QPushConfig{ + Type: "http_endpoint", + PushEndpoint: "example.com", + MaxMessages: 3, + AuthorizationType: "autogen", + AuthorizationHeader: "auth-h-1", + RetPolicy: "linear", + RetPeriod: 400, + VerificationHash: "hash-1", + Verified: true, + MattermostUrl: "m-url", + MattermostUsername: "m-u", + MattermostChannel: "m-c", + Base64Decode: true, + } + e1 := suite.store.ModSubPush(suite.ctx, "argo_uuid", "subFresh", qCfg) + sub1, _ := suite.store.QueryOneSub(suite.ctx, "argo_uuid", "subFresh") + suite.Nil(e1) + suite.Equal("example.com", sub1.PushEndpoint) + suite.Equal(int64(3), sub1.MaxMessages) + suite.Equal("linear", sub1.RetPolicy) + suite.Equal(400, sub1.RetPeriod) + suite.Equal("hash-1", sub1.VerificationHash) + suite.Equal("autogen", sub1.AuthorizationType) + suite.Equal("auth-h-1", sub1.AuthorizationHeader) + suite.Equal("m-url", sub1.MattermostUrl) + suite.Equal("m-c", sub1.MattermostChannel) + suite.Equal("m-u", sub1.MattermostUsername) + suite.True(sub1.Verified) + suite.True(sub1.Base64Decode) + + e2 := suite.store.ModSubPush(suite.ctx, "argo_uuid", "unknown", QPushConfig{}) + suite.Equal("not found", e2.Error()) + + _ = suite.store.RemoveSub(suite.ctx, "argo_uuid", "subFresh") +} + +func (suite *MongoStoreIntegrationTestSuite) TestExistsInACL() { + existsE1 := suite.store.ExistsInACL(suite.ctx, "argo_uuid", "topics", "topic1", "uuid1") + suite.Nil(existsE1) + + existsE2 := suite.store.ExistsInACL(suite.ctx, "argo_uuid", "topics", "topic1", "unknown") + suite.Equal("not found", existsE2.Error()) +} + +func (suite *MongoStoreIntegrationTestSuite) TestQueryACL() { + ExpectedACL01 := QAcl{[]string{"uuid1", "uuid2"}} + QAcl01, _ := suite.store.QueryACL(suite.ctx, "argo_uuid", "topics", "topic1") + suite.Equal(ExpectedACL01, QAcl01) + + ExpectedACL02 := QAcl{[]string{"uuid1", "uuid2", "uuid4"}} + QAcl02, _ := suite.store.QueryACL(suite.ctx, "argo_uuid", "topics", "topic2") + suite.Equal(ExpectedACL02, QAcl02) + + ExpectedACL03 := QAcl{[]string{"uuid3"}} + QAcl03, _ := suite.store.QueryACL(suite.ctx, "argo_uuid", "topics", "topic3") + suite.Equal(ExpectedACL03, QAcl03) + + ExpectedACL04 := QAcl{[]string{"uuid1", "uuid2"}} + QAcl04, _ := suite.store.QueryACL(suite.ctx, "argo_uuid", "subscriptions", "sub1") + suite.Equal(ExpectedACL04, QAcl04) + + ExpectedACL05 := QAcl{[]string{"uuid1", "uuid3"}} + QAcl05, _ := suite.store.QueryACL(suite.ctx, "argo_uuid", "subscriptions", "sub2") + suite.Equal(ExpectedACL05, QAcl05) + + ExpectedACL06 := QAcl{[]string{"uuid4", "uuid2", "uuid1"}} + QAcl06, _ := suite.store.QueryACL(suite.ctx, "argo_uuid", "subscriptions", "sub3") + suite.Equal(ExpectedACL06, QAcl06) + + ExpectedACL07 := QAcl{[]string{"uuid2", "uuid4", "uuid7"}} + QAcl07, _ := suite.store.QueryACL(suite.ctx, "argo_uuid", "subscriptions", "sub4") + suite.Equal(ExpectedACL07, QAcl07) + + QAcl08, err08 := suite.store.QueryACL(suite.ctx, "argo_uuid", "subscr", "sub4ss") + suite.Equal(QAcl{}, QAcl08) + suite.Equal(errors.New("wrong resource type"), err08) + + QAcl09, err09 := suite.store.QueryACL(suite.ctx, "argo_uuid", "subscriptions", "sub4ss") + suite.Equal(QAcl{}, QAcl09) + suite.Equal(errors.New("not found"), err09) +} + +func (suite *MongoStoreIntegrationTestSuite) TestUpdateUser() { + created := time.Date(2009, time.November, 10, 23, 0, 0, 0, time.Local) + modified := time.Date(2009, time.November, 10, 23, 0, 0, 0, time.Local) + qRoles := []QProjectRoles{QProjectRoles{"argo_uuid", []string{"admin"}}, QProjectRoles{"argo_uuid2", []string{"admin", "viewer"}}} + _ = suite.store.InsertUser(suite.ctx, "user_uuid11", qRoles, "newUser2", "", "", "", "", "BX312Z34NLQ", "fake@email.com", []string{}, created, modified, "uuid1") + usrUpdated := QUser{UUID: "user_uuid11", Projects: qRoles, Name: "updated_name", Token: "BX312Z34NLQ", Email: "fake@email.com", ServiceRoles: []string{"service_admin"}, CreatedOn: created, ModifiedOn: modified, CreatedBy: "uuid1"} + _ = suite.store.UpdateUser(suite.ctx, "user_uuid11", "", "", "", "", nil, "updated_name", "", []string{"service_admin"}, modified) + usr11, _ := suite.store.QueryUsers(suite.ctx, "", "user_uuid11", "") + suite.assertUsersEqual([]QUser{usrUpdated}, usr11) + // test append project to user + errUserPrj := suite.store.AppendToUserProjects(suite.ctx, "user_uuid11", "p1_uuid", "r1", "r2") + usr, _ := suite.store.QueryUsers(suite.ctx, "", "user_uuid11", "") + suite.Equal([]QProjectRoles{ + { + ProjectUUID: "argo_uuid", + Roles: []string{"admin"}, + }, + { + ProjectUUID: "argo_uuid2", + Roles: []string{"admin", "viewer"}, + }, + { + ProjectUUID: "p1_uuid", + Roles: []string{"r1", "r2"}, + }, + }, usr[0].Projects) + suite.Nil(errUserPrj) + + // Test Remove User + _ = suite.store.RemoveUser(suite.ctx, "user_uuid11") + qu, _ := suite.store.QueryUsers(suite.ctx, "", "user_uuid11", "") + suite.Empty(qu) +} + +func (suite *MongoStoreIntegrationTestSuite) TestGetUserFromToken() { + usrGet, _ := suite.store.GetUserFromToken(suite.ctx, "S3CR3T") + suite.assertUsersEqual([]QUser{suite.UserList[0]}, []QUser{usrGet}) +} + +func (suite *MongoStoreIntegrationTestSuite) TestPaginatedQueryUsers() { + + reverser := QListReverser[QUser]{} + + // return all users in one page + qUsers1, ts1, pg1, _ := suite.store.PaginatedQueryUsers(suite.ctx, "", 0, "") + + // return a page with the first 2 + qUsers2, ts2, pg2, _ := suite.store.PaginatedQueryUsers(suite.ctx, "", 2, "") + + // empty store + qUsers3, ts3, pg3, _ := suite.store.PaginatedQueryUsers(suite.ctx, "", 0, "unkknown") + + // use page token to grab another 2 results + qUsers4, ts4, pg4, _ := suite.store.PaginatedQueryUsers(suite.ctx, pg2, 2, "") + + suite.assertUsersEqual(reverser.reverse(suite.UserList), qUsers1) + suite.Equal("", pg1) + suite.Equal(int64(9), ts1) + + suite.assertUsersEqual([]QUser{suite.UserList[8], suite.UserList[7]}, qUsers2) + suite.True(bson.IsObjectIdHex(pg2)) + suite.Equal(int64(9), ts2) + + suite.Equal(0, len(qUsers3)) + suite.Equal("", pg3) + suite.Equal(int64(0), ts3) + + suite.assertUsersEqual([]QUser{suite.UserList[6], suite.UserList[5]}, qUsers4) + suite.True(bson.IsObjectIdHex(pg4)) + suite.Equal(int64(9), ts4) +} + +func (suite *MongoStoreIntegrationTestSuite) TestACLModificationActions() { + + _ = suite.store.InsertSub(suite.ctx, "argo_uuid", "subFresh", "topicFresh", 0, 10, QPushConfig{}, time.Date(2020, 12, 19, 0, 0, 0, 0, time.Local)) + _ = suite.store.InsertTopic(suite.ctx, "argo_uuid", "topicFresh", "", time.Date(2020, 9, 11, 0, 0, 0, 0, time.Local)) + + // test mod acl + ExpectedACL01 := QAcl{[]string{"uuid1", "uuid2"}} + eModACL1 := suite.store.ModACL(suite.ctx, "argo_uuid", "topics", "topicFresh", ExpectedACL01.ACL) + suite.Nil(eModACL1) + QAcl01, _ := suite.store.QueryACL(suite.ctx, "argo_uuid", "topics", "topicFresh") + suite.Equal(ExpectedACL01, QAcl01) + + eModACL2 := suite.store.ModACL(suite.ctx, "argo_uuid", "subscriptions", "subFresh", ExpectedACL01.ACL) + suite.Nil(eModACL2) + QAcl01sub, _ := suite.store.QueryACL(suite.ctx, "argo_uuid", "topics", "topicFresh") + suite.Equal(ExpectedACL01, QAcl01sub) + + eModACL3 := suite.store.ModACL(suite.ctx, "argo_uuid", "mistype", "sub1", []string{"u1", "u2"}) + suite.Equal("wrong resource type", eModACL3.Error()) + + // test append acl + ExpectedACL02 := QAcl{ACL: append(ExpectedACL01.ACL, "u3", "u4")} + eAppACL1 := suite.store.AppendToACL(suite.ctx, "argo_uuid", "topics", "topicFresh", []string{"u3", "u4", "u4"}) + suite.Nil(eAppACL1) + QAcl02, _ := suite.store.QueryACL(suite.ctx, "argo_uuid", "topics", "topicFresh") + suite.Equal(ExpectedACL02, QAcl02) + + eAppACL2 := suite.store.AppendToACL(suite.ctx, "argo_uuid", "subscriptions", "subFresh", []string{"u3", "u4", "u4"}) + suite.Nil(eAppACL2) + QAcl02sub, _ := suite.store.QueryACL(suite.ctx, "argo_uuid", "topics", "topicFresh") + suite.Equal(ExpectedACL02, QAcl02sub) + + eAppACL3 := suite.store.AppendToACL(suite.ctx, "argo_uuid", "mistype", "sub1", []string{"u3", "u4", "u4"}) + suite.Equal("wrong resource type", eAppACL3.Error()) + + // test remove acl + ExpectedACL03 := QAcl{ACL: append(ExpectedACL01.ACL, "u3")} + eRemACL1 := suite.store.RemoveFromACL(suite.ctx, "argo_uuid", "topics", "topicFresh", []string{"u1", "u4", "u5"}) + suite.Nil(eRemACL1) + QAcl03, _ := suite.store.QueryACL(suite.ctx, "argo_uuid", "topics", "topicFresh") + suite.Equal(ExpectedACL03, QAcl03) + + eRemACL2 := suite.store.RemoveFromACL(suite.ctx, "argo_uuid", "subscriptions", "subFresh", []string{"u1", "u4", "u5"}) + suite.Nil(eRemACL2) + QAcl03sub, _ := suite.store.QueryACL(suite.ctx, "argo_uuid", "topics", "topicFresh") + suite.Equal(ExpectedACL03, QAcl03sub) + + eRemACL3 := suite.store.RemoveFromACL(suite.ctx, "argo_uuid", "mistype", "sub1", []string{"u3", "u4", "u4"}) + suite.Equal("wrong resource type", eRemACL3.Error()) + + _ = suite.store.RemoveSub(suite.ctx, "argo_uuid", "subFresh") + _ = suite.store.RemoveTopic(suite.ctx, "argo_uuid", "topicFresh") +} + +func (suite *MongoStoreIntegrationTestSuite) TestHasUsers() { + allFound, notFound := suite.store.HasUsers(suite.ctx, "argo_uuid", []string{"UserA", "UserB", "FooUser"}) + suite.Equal(false, allFound) + suite.Equal([]string{"FooUser"}, notFound) + + allFound, notFound = suite.store.HasUsers(suite.ctx, "argo_uuid", []string{"UserA", "UserB"}) + suite.Equal(true, allFound) + suite.Equal([]string(nil), notFound) +} + +func (suite *MongoStoreIntegrationTestSuite) TestCRUDProjects() { + + expProj1 := []QProject{suite.ProjectList[0]} + expProj2 := []QProject{suite.ProjectList[1]} + expProj3 := []QProject{suite.ProjectList[0], suite.ProjectList[1]} + var expProj4 []QProject + + projectOut1, err := suite.store.QueryProjects(suite.ctx, "", "ARGO") + suite.Equal(expProj1, projectOut1) + suite.Equal(nil, err) + projectOut2, err := suite.store.QueryProjects(suite.ctx, "", "ARGO2") + suite.Equal(expProj2, projectOut2) + suite.Equal(nil, err) + projectOut3, err := suite.store.QueryProjects(suite.ctx, "", "") + suite.Equal(expProj3, projectOut3) + suite.Equal(nil, err) + + projectOut4, err := suite.store.QueryProjects(suite.ctx, "", "FOO") + + suite.Equal(expProj4, projectOut4) + suite.Equal(errors.New("not found"), err) + + // Test queries by uuid + projectOut7, err := suite.store.QueryProjects(suite.ctx, "argo_uuid2", "") + suite.Equal(expProj2, projectOut7) + suite.Equal(nil, err) + projectOut8, err := suite.store.QueryProjects(suite.ctx, "foo_uuidNone", "") + suite.Equal(expProj4, projectOut8) + suite.Equal(errors.New("not found"), err) + + created := time.Date(2009, time.November, 10, 23, 0, 0, 0, time.Local) + modified := time.Date(2009, time.November, 10, 23, 0, 0, 0, time.Local) + _ = suite.store.InsertProject(suite.ctx, "argo_uuid3", "ARGO3", created, modified, "uuid1", "simple project") + modified = time.Date(2010, time.November, 10, 23, 0, 0, 0, time.Local) + expPr1 := QProject{UUID: "argo_uuid3", Name: "ARGO3", CreatedOn: created, ModifiedOn: modified, CreatedBy: "uuid1", Description: "a modified description"} + _ = suite.store.UpdateProject(suite.ctx, "argo_uuid3", "", "a modified description", modified) + prUp1, _ := suite.store.QueryProjects(suite.ctx, "argo_uuid3", "") + suite.Equal(expPr1, prUp1[0]) + expPr2 := QProject{UUID: "argo_uuid3", Name: "ARGO_updated3", CreatedOn: created, ModifiedOn: modified, CreatedBy: "uuid1", Description: "a modified description"} + _ = suite.store.UpdateProject(suite.ctx, "argo_uuid3", "ARGO_updated3", "", modified) + prUp2, _ := suite.store.QueryProjects(suite.ctx, "argo_uuid3", "") + suite.Equal(expPr2, prUp2[0]) + expPr3 := QProject{UUID: "argo_uuid3", Name: "ARGO_3", CreatedOn: created, ModifiedOn: modified, CreatedBy: "uuid1", Description: "a newly modified description"} + _ = suite.store.UpdateProject(suite.ctx, "argo_uuid3", "ARGO_3", "a newly modified description", modified) + prUp3, _ := suite.store.QueryProjects(suite.ctx, "argo_uuid3", "") + suite.Equal(expPr3, prUp3[0]) + // Test RemoveProject + _ = suite.store.RemoveProject(suite.ctx, "argo_uuid3") + _, err = suite.store.QueryProjects(suite.ctx, "argo_uuid3", "") + suite.Equal(errors.New("not found"), err) +} + +func (suite *MongoStoreIntegrationTestSuite) TestQueryTotalMessagesPerProject() { + expectedQpmc := []QProjectMessageCount{ + {ProjectUUID: "argo_uuid", NumberOfMessages: 30, AverageDailyMessages: 7.5}, + } + qpmc, qpmcerr1 := suite.store.QueryTotalMessagesPerProject(suite.ctx, []string{"argo_uuid"}, time.Date(2018, 10, 1, 0, 0, 0, 0, time.UTC), time.Date(2018, 10, 4, 0, 0, 0, 0, time.UTC)) + suite.Equal(expectedQpmc, qpmc) + suite.Nil(qpmcerr1) +} + +func (suite *MongoStoreIntegrationTestSuite) TestCRUDSchemas() { + + // test QuerySchemas + expectedSchemas := []QSchema{ + suite.SchemaList[0], + suite.SchemaList[1], + suite.SchemaList[2], + } + qqs1, _ := suite.store.QuerySchemas(suite.ctx, "argo_uuid", "", "") + qqs2, _ := suite.store.QuerySchemas(suite.ctx, "argo_uuid", "schema_uuid_1", "") + qqs3, _ := suite.store.QuerySchemas(suite.ctx, "argo_uuid", "schema_uuid_1", "schema-1") + suite.assertSchemasEqual(expectedSchemas, qqs1) + suite.assertSchemasEqual([]QSchema{expectedSchemas[0]}, []QSchema{qqs2[0]}) + suite.assertSchemasEqual([]QSchema{expectedSchemas[0]}, []QSchema{qqs3[0]}) + + // test InsertSchema + eis := suite.store.InsertSchema(suite.ctx, "argo_uuid", "uuid1", "s1-insert", "json", "raw") + qs1, _ := suite.store.QuerySchemas(suite.ctx, "argo_uuid", "uuid1", "s1-insert") + suite.Equal(QSchema{ + ProjectUUID: "argo_uuid", + UUID: "uuid1", + Name: "s1-insert", + Type: "json", + RawSchema: "raw", + }, qs1[0]) + suite.Nil(eis) + + // test update schema + _ = suite.store.InsertTopic(suite.ctx, "argo_uuid", "topicFresh", "", time.Date(2020, 9, 11, 0, 0, 0, 0, time.Local)) + _ = suite.store.UpdateSchema(suite.ctx, "uuid1", "new-name", "new-type", "new-raw-schema") + eus := QSchema{UUID: "uuid1", ProjectUUID: "argo_uuid", Type: "new-type", Name: "new-name", RawSchema: "new-raw-schema"} + qus, _ := suite.store.QuerySchemas(suite.ctx, "argo_uuid", "uuid1", "") + suite.Equal(eus, qus[0]) + _ = suite.store.LinkTopicSchema(suite.ctx, "argo_uuid", "topicFresh", "uuid1") + t, _, _, _ := suite.store.QueryTopics(suite.ctx, "argo_uuid", "", "topicFresh", "", 0) + suite.Equal("uuid1", t[0].SchemaUUID) + ed := suite.store.DeleteSchema(suite.ctx, "uuid1") + expd, _ := suite.store.QuerySchemas(suite.ctx, "argo_uuid", "uuid1", "") + // check that topic no longer has any schema_uuid associated with it + qtd, _, _, _ := suite.store.QueryTopics(suite.ctx, "argo_uuid", "", "topicFresh", "", 1) + suite.Equal("", qtd[0].SchemaUUID) + suite.Empty(expd) + suite.Nil(ed) + _ = suite.store.RemoveTopic(suite.ctx, "argo_uuid", "topicFresh") +} + +func (suite *MongoStoreIntegrationTestSuite) TestCRUDRegistrations() { + + _ = suite.store.RegisterUser(suite.ctx, "ruuid1", "n1", "f1", "l1", "e1", "o1", "d1", "time", "atkn", "pending") + expur1 := []QUserRegistration{{ + UUID: "ruuid1", + Name: "n1", + FirstName: "f1", + LastName: "l1", + Email: "e1", + Organization: "o1", + Description: "d1", + RegisteredAt: "time", + ActivationToken: "atkn", + Status: "pending", + }} + + ur1, _ := suite.store.QueryRegistrations(suite.ctx, "ruuid1", "pending", "atkn", "n1", "e1", "o1") + suite.Equal(expur1, ur1) + + ur12, _ := suite.store.QueryRegistrations(suite.ctx, "ruuid1", "", "", "", "", "") + suite.Equal(expur1, ur12) + + expur2 := []QUserRegistration{{ + UUID: "ur-uuid1", + Name: "urname", + FirstName: "urfname", + LastName: "urlname", + Organization: "urorg", + Description: "urdesc", + Email: "uremail", + ActivationToken: "", + Status: "accepted", + RegisteredAt: "2019-05-12T22:26:58Z", + ModifiedBy: "uuid1", + ModifiedAt: "2020-05-17T22:26:58Z", + }} + _ = suite.store.UpdateRegistration(suite.ctx, "ur-uuid1", "accepted", "", "uuid1", "2020-05-17T22:26:58Z") + ur2, _ := suite.store.QueryRegistrations(suite.ctx, "ur-uuid1", "accepted", "", "", "", "") + suite.Equal(expur2, ur2) + + ur3, _ := suite.store.QueryRegistrations(suite.ctx, "", "", "", "", "", "") + + suite.Equal(2, len(ur3)) + suite.Equal([]QUserRegistration{expur2[0], expur1[0]}, ur3) + _ = suite.store.DeleteRegistration(suite.ctx, "ruuid1") + ur4, _ := suite.store.QueryRegistrations(suite.ctx, "", "", "", "", "", "") + suite.Equal(1, len(ur4)) + + dErr := suite.store.DeleteRegistration(suite.ctx, "unknown") + suite.Equal("not found", dErr.Error()) + +} + +func (suite *MongoStoreIntegrationTestSuite) TestResourcesCounters() { + sdate := time.Date(2008, 11, 19, 8, 0, 0, 0, time.UTC) + edate := time.Date(2020, 11, 21, 6, 0, 0, 0, time.UTC) + tc, _ := suite.store.TopicsCount(suite.ctx, sdate, edate, []string{}) + sc, _ := suite.store.SubscriptionsCount(suite.ctx, sdate, edate, []string{}) + uc, _ := suite.store.UsersCount(suite.ctx, sdate, edate, []string{}) + + suite.Equal(map[string]int64{"argo_uuid": 3}, tc) + suite.Equal(map[string]int64{"argo_uuid": 3}, sc) + suite.Equal(map[string]int64{"argo_uuid": 7, "argo_uuid2": 1}, uc) +} + +func (suite *MongoStoreIntegrationTestSuite) SetupSuite() { + suite.ctx = context.Background() + suite.store.Initialize() + suite.initDB() +} + +func (suite *MongoStoreIntegrationTestSuite) TearDownSuite() { + suite.store.Close() +} + +func TestMongoStoreIntegrationTestSuite(t *testing.T) { + + container, err := startContainer(context.Background()) + if err != nil { + panic("Could not start container for mongodb integration tests. " + err.Error()) + } + + p, _ := container.MappedPort(context.Background(), "27017/tcp") + + mongoDBUri := fmt.Sprintf("mongodb://localhost:%s", p.Port()) + + mongoStore := &MongoStore{ + Server: mongoDBUri, + Database: "argo_ams", + } + suite.Run(t, &MongoStoreIntegrationTestSuite{ + store: mongoStore, + ctx: context.Background(), + }) +} diff --git a/stores/store.go b/stores/store.go index a97b48d1..5d115d4b 100644 --- a/stores/store.go +++ b/stores/store.go @@ -54,6 +54,7 @@ type Store interface { QueryOneSub(ctx context.Context, projectUUID string, name string) (QSub, error) QueryPushSubs(ctx context.Context) []QSub HasResourceRoles(ctx context.Context, resource string, roles []string) bool + InsertResourceRoles(ctx context.Context, resource string, roles []string) error GetOpMetrics(ctx context.Context) []QopMetric GetUserRoles(ctx context.Context, projectUUID string, token string) ([]string, string) GetUserFromToken(ctx context.Context, token string) (QUser, error) diff --git a/stores/store_test.go b/stores/store_test.go index f82d6dcb..8e96b22f 100644 --- a/stores/store_test.go +++ b/stores/store_test.go @@ -562,13 +562,13 @@ func (suite *StoreTestSuite) TestMockStore() { tpc2, _, _, _ := store2.QueryTopics(ctx, "argo_uuid", "", "topic1", "", 0) suite.Equal(8.44, tpc2[0].PublishRate) - // test update topic latest publish time + // test update sub latest consume time scre1 := store2.UpdateSubLatestConsume(ctx, "argo_uuid", "sub1", time.Date(2019, 8, 8, 0, 0, 0, 0, time.UTC)) suite.Nil(scre1) spc, _, _, _ := store2.QuerySubs(ctx, "argo_uuid", "", "sub1", "", 0) suite.Equal(time.Date(2019, 8, 8, 0, 0, 0, 0, time.UTC), spc[0].LatestConsume) - // test update topic publishing rate + // test update sub consume rate scre2 := store2.UpdateSubConsumeRate(ctx, "argo_uuid", "sub1", 8.44) suite.Nil(scre2) spc2, _, _, _ := store2.QuerySubs(ctx, "argo_uuid", "", "sub1", "", 0) @@ -610,7 +610,7 @@ func (suite *StoreTestSuite) TestMockStore() { suite.Equal(expectedSchemas[0], qqs3[0]) // test update schema - store2.UpdateSchema(ctx, "schema_uuid_1", "new-name", "new-type", "new-raw-schema") + _ = store2.UpdateSchema(ctx, "schema_uuid_1", "new-name", "new-type", "new-raw-schema") eus := QSchema{UUID: "schema_uuid_1", ProjectUUID: "argo_uuid", Type: "new-type", Name: "new-name", RawSchema: "new-raw-schema"} qus, _ := store2.QuerySchemas(ctx, "argo_uuid", "schema_uuid_1", "") suite.Equal(eus, qus[0]) @@ -627,7 +627,7 @@ func (suite *StoreTestSuite) TestMockStore() { suite.Nil(ed) // test user registration - store.RegisterUser(ctx, "ruuid1", "n1", "f1", "l1", "e1", "o1", "d1", "time", "atkn", "pending") + _ = store.RegisterUser(ctx, "ruuid1", "n1", "f1", "l1", "e1", "o1", "d1", "time", "atkn", "pending") expur1 := []QUserRegistration{{ UUID: "ruuid1", Name: "n1", @@ -661,7 +661,7 @@ func (suite *StoreTestSuite) TestMockStore() { ModifiedBy: "uuid1", ModifiedAt: "2020-05-17T22:26:58Z", }} - store.UpdateRegistration(ctx, "ur-uuid1", "accepted", "", "uuid1", "2020-05-17T22:26:58Z") + _ = store.UpdateRegistration(ctx, "ur-uuid1", "accepted", "", "uuid1", "2020-05-17T22:26:58Z") ur2, _ := store.QueryRegistrations(ctx, "ur-uuid1", "accepted", "", "", "", "") suite.Equal(expur2, ur2) From 2371790a1f943a8756d4fa4ada961ba33817e0d3 Mon Sep 17 00:00:00 2001 From: agelostsal Date: Mon, 9 Oct 2023 12:05:49 +0300 Subject: [PATCH 4/9] AM-338 ams: Use the official mongodb client --- go.mod | 8 + go.sum | 28 + handlers/handlers.go | 4 +- main.go | 9 +- stores/mongo.go | 3 - stores/mongo_official_driver.go | 2089 ++++++++++++++++++++++++ stores/mongo_store_integration_test.go | 285 +++- stores/store.go | 102 +- 8 files changed, 2418 insertions(+), 110 deletions(-) create mode 100644 stores/mongo_official_driver.go diff --git a/go.mod b/go.mod index 8b7a2bfb..2fcf344f 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( github.com/testcontainers/testcontainers-go v0.24.1 github.com/twinj/uuid v0.0.0-20150629100731-70cac2bcd273 github.com/xeipuuv/gojsonschema v1.2.0 + go.mongodb.org/mongo-driver v1.12.1 google.golang.org/grpc v1.57.0 gopkg.in/mgo.v2 v2.0.0-20160818020120-3f83fa500528 ) @@ -51,6 +52,7 @@ require ( github.com/moby/patternmatcher v0.5.0 // indirect github.com/moby/sys/sequential v0.5.0 // indirect github.com/moby/term v0.5.0 // indirect + github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect github.com/morikuni/aec v1.0.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0-rc4 // indirect @@ -69,12 +71,18 @@ require ( github.com/subosito/gotenv v1.4.2 // indirect github.com/tklauser/go-sysconf v0.3.11 // indirect github.com/tklauser/numcpus v0.6.0 // indirect + github.com/xdg-go/pbkdf2 v1.0.0 // indirect + github.com/xdg-go/scram v1.1.2 // indirect + github.com/xdg-go/stringprep v1.0.4 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect + github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect github.com/yusufpapurcu/wmi v1.2.3 // indirect + golang.org/x/crypto v0.9.0 // indirect golang.org/x/exp v0.0.0-20230510235704-dd950f8aeaea // indirect golang.org/x/mod v0.9.0 // indirect golang.org/x/net v0.10.0 // indirect + golang.org/x/sync v0.3.0 // indirect golang.org/x/sys v0.11.0 // indirect golang.org/x/text v0.9.0 // indirect golang.org/x/tools v0.7.0 // indirect diff --git a/go.sum b/go.sum index 769c1cb0..8c42242f 100644 --- a/go.sum +++ b/go.sum @@ -193,6 +193,7 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.16.0 h1:iULayQNOReoYUe+1qtKOqw9CwJv3aNQu8ivo7lw1HU4= github.com/klauspost/compress v1.16.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= @@ -217,6 +218,8 @@ github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5 github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo= github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= @@ -297,6 +300,12 @@ github.com/twinj/uuid v0.0.0-20150629100731-70cac2bcd273/go.mod h1:mMgcE1RHFUFqe github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= +github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= +github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= +github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= +github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= @@ -306,12 +315,17 @@ github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHo github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +go.mongodb.org/mongo-driver v1.12.1 h1:nLkghSU8fQNaK7oUmDhQFsnrtcoNy7Z6LVFKsEecqgE= +go.mongodb.org/mongo-driver v1.12.1/go.mod h1:/rGBTebI3XYboVmgz+Wv3Bcbl3aD0QF9zl6kDDw18rQ= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -325,7 +339,11 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= +golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -361,6 +379,7 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs= golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -395,6 +414,7 @@ golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -416,7 +436,9 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -463,13 +485,16 @@ golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -478,6 +503,8 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -533,6 +560,7 @@ golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/handlers/handlers.go b/handlers/handlers.go index ef33e101..06a20451 100644 --- a/handlers/handlers.go +++ b/handlers/handlers.go @@ -90,12 +90,10 @@ func WrapMockAuthConfig(hfn http.HandlerFunc, cfg *config.APICfg, brk brokers.Br func WrapConfig(hfn http.HandlerFunc, cfg *config.APICfg, brk brokers.Broker, str stores.Store, mgr *oldPush.Manager, c push.Client) http.HandlerFunc { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - nStr := str.Clone() - defer nStr.Close() traceId := uuid.NewV4().String() gorillaContext.Set(r, "trace_id", traceId) gorillaContext.Set(r, "brk", brk) - gorillaContext.Set(r, "str", nStr) + gorillaContext.Set(r, "str", str) gorillaContext.Set(r, "mgr", mgr) gorillaContext.Set(r, "apsc", c) gorillaContext.Set(r, "authOption", cfg.AuthOption()) diff --git a/main.go b/main.go index 228261b0..1b6f5c7a 100644 --- a/main.go +++ b/main.go @@ -34,12 +34,11 @@ func main() { cfg := config.NewAPICfg("LOAD") // create the store - store := stores.NewMongoStore(cfg.StoreHost, cfg.StoreDB) + store := stores.NewMongoStoreWithOfficialDriver(cfg.StoreHost, cfg.StoreDB) store.Initialize() // create and initialize broker based on configuration broker := brokers.NewKafkaBroker(cfg.GetBrokerInfo()) - defer broker.CloseConnections() mgr := &oldPush.Manager{} @@ -56,7 +55,11 @@ func main() { ).Error(err.Error()) } - defer pushClient.Close() + defer func() { + store.Close() + broker.CloseConnections() + pushClient.Close() + }() // create and initialize API routing object API := NewRouting(cfg, broker, store, mgr, pushClient, defaultRoutes) diff --git a/stores/mongo.go b/stores/mongo.go index b56d590d..25000dd4 100644 --- a/stores/mongo.go +++ b/stores/mongo.go @@ -1491,8 +1491,6 @@ func (mong *MongoStore) QueryDailyProjectMsgCount(ctx context.Context, projectUU }, } - fmt.Println(query) - fmt.Println("2") if err = c.Pipe(query).All(&qdp); err != nil { mong.logErrorAndCrash(ctx, "QueryDailyProjectMsgCount", err) } @@ -1861,7 +1859,6 @@ func (mong *MongoStore) DeleteSchema(ctx context.Context, schemaUUID string) err if err != nil { mong.logErrorAndCrash(ctx, "DeleteSchema", err) - } topics := db.C("topics") diff --git a/stores/mongo_official_driver.go b/stores/mongo_official_driver.go new file mode 100644 index 00000000..9e871cb3 --- /dev/null +++ b/stores/mongo_official_driver.go @@ -0,0 +1,2089 @@ +package stores + +import ( + "context" + "errors" + "fmt" + log "github.com/sirupsen/logrus" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "go.mongodb.org/mongo-driver/mongo/readpref" + "time" +) + +const TopicsCollection string = "topics" +const SubscriptionsCollection string = "subscriptions" +const DailyTopicMsgCountCollection string = "daily_topic_msg_count" +const UsersCollection string = "users" +const ProjectsCollection string = "projects" +const UserRegistrationsCollection string = "user_registrations" +const SchemasCollection string = "schemas" +const OpMetricsCollection string = "op_metrics" +const RolesCollection string = "roles" + +type DocNotFound struct{} + +func (DocNotFound) Error() string { + return "not found" +} + +type findQueryProcessor[T any] struct { + collection *mongo.Collection +} + +func (p findQueryProcessor[T]) execute(ctx context.Context, query bson.M, opts ...*options.FindOptions) ([]T, error) { + var results []T + + cursor, err := p.collection.Find(ctx, query, opts...) + if err != nil { + return results, err + } + + for cursor.Next(ctx) { + var result T + err := cursor.Decode(&result) + if err != nil { + return results, err + } + results = append(results, result) + } + + if err := cursor.Err(); err != nil { + return results, err + } + + return results, nil + +} + +type MongoStoreWithOfficialDriver struct { + Server string + Database string + database *mongo.Database + client *mongo.Client + + topicsCollection *mongo.Collection + topicsDailyMsgCountCollection *mongo.Collection + subscriptionsCollection *mongo.Collection + usersCollection *mongo.Collection + projectsCollection *mongo.Collection + userRegistrationsCollection *mongo.Collection + schemasCollection *mongo.Collection + rolesCollection *mongo.Collection + opMetricsCollection *mongo.Collection + + topicsFindQueryProcessor findQueryProcessor[QTopic] + subsFindQueryProcessor findQueryProcessor[QSub] + usersFindQueryProcessor findQueryProcessor[QUser] + projectsFindQueryProcessor findQueryProcessor[QProject] + userRegistrationsFindQueryProcessor findQueryProcessor[QUserRegistration] + schemasFindQueryProcessor findQueryProcessor[QSchema] +} + +func NewMongoStoreWithOfficialDriver(server, database string) *MongoStoreWithOfficialDriver { + return &MongoStoreWithOfficialDriver{ + Server: server, + Database: database, + } +} + +func (store *MongoStoreWithOfficialDriver) Initialize() { + + mongoDBUri := fmt.Sprintf("mongodb://%s", store.Server) + + for { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + log.WithFields( + log.Fields{ + "type": "backend_log", + "backend_service": "mongo", + "backend_hosts": store.Server, + }, + ).Info("Trying to connect to Mongo") + client, err := mongo.Connect(ctx, options.Client().ApplyURI(mongoDBUri)) + if err != nil { + log.WithFields( + log.Fields{ + "type": "backend_log", + "backend_service": "mongo", + "backend_hosts": store.Server, + }, + ).Error(err.Error()) + continue + } + store.client = client + cancel() + break + } + + log.WithFields( + log.Fields{ + "type": "backend_log", + "backend_service": "mongo", + "backend_hosts": store.Server, + }, + ).Info("Connection to Mongo established successfully") + + for { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + log.WithFields( + log.Fields{ + "type": "backend_log", + "backend_service": "mongo", + "backend_hosts": store.Server, + }, + ).Info("Trying to ping Mongo") + err := store.client.Ping(ctx, readpref.Primary()) + if err != nil { + log.WithFields( + log.Fields{ + "type": "backend_log", + "backend_service": "mongo", + "backend_hosts": store.Server, + }, + ).Error(err.Error()) + continue + } + cancel() + break + } + + log.WithFields( + log.Fields{ + "type": "backend_log", + "backend_service": "mongo", + "backend_hosts": store.Server, + }, + ).Info("Mongo Deployment is up and running") + store.database = store.client.Database(store.Database) + + store.topicsCollection = store.database.Collection(TopicsCollection) + store.topicsFindQueryProcessor = findQueryProcessor[QTopic]{ + collection: store.topicsCollection, + } + + store.subscriptionsCollection = store.database.Collection(SubscriptionsCollection) + store.subsFindQueryProcessor = findQueryProcessor[QSub]{ + collection: store.subscriptionsCollection, + } + + store.usersCollection = store.database.Collection(UsersCollection) + store.usersFindQueryProcessor = findQueryProcessor[QUser]{ + collection: store.usersCollection, + } + + store.projectsCollection = store.database.Collection(ProjectsCollection) + store.projectsFindQueryProcessor = findQueryProcessor[QProject]{ + collection: store.projectsCollection, + } + + store.userRegistrationsCollection = store.database.Collection(UserRegistrationsCollection) + store.userRegistrationsFindQueryProcessor = findQueryProcessor[QUserRegistration]{ + collection: store.userRegistrationsCollection, + } + + store.schemasCollection = store.database.Collection(SchemasCollection) + store.schemasFindQueryProcessor = findQueryProcessor[QSchema]{ + collection: store.schemasCollection, + } + + store.topicsDailyMsgCountCollection = store.database.Collection(DailyTopicMsgCountCollection) + store.rolesCollection = store.database.Collection(RolesCollection) + store.opMetricsCollection = store.database.Collection(OpMetricsCollection) +} + +func (store *MongoStoreWithOfficialDriver) Close() { + if store.client != nil { + if err := store.client.Disconnect(context.Background()); err != nil { + log.Fatalf("Could not disconnect mongo client, %s", err.Error()) + } + } +} + +func (store *MongoStoreWithOfficialDriver) Clone() Store { + return store +} + +func (store *MongoStoreWithOfficialDriver) logErrorAndCrash(ctx context.Context, funcName string, err error) { + log.WithFields( + log.Fields{ + "trace_id": ctx.Value("trace_id"), + "type": "backend_log", + "function": funcName, + "backend_service": "mongo", + "backend_hosts": store.Server, + }, + ).Fatal(err.Error()) +} + +func (store *MongoStoreWithOfficialDriver) deleteOne( + ctx context.Context, + collection *mongo.Collection, + query bson.M) error { + dr, err := collection.DeleteOne(ctx, query) + if err != nil { + return err + } + if dr.DeletedCount == 0 { + return DocNotFound{} + } + return nil +} + +func (store *MongoStoreWithOfficialDriver) upsert(ctx context.Context, doc interface{}, + change interface{}, collection *mongo.Collection) error { + updateOptions := options.Update().SetUpsert(true) + _, err := collection.UpdateOne(ctx, doc, change, updateOptions) + return err +} + +// getDocCountForCollectionPerProject returns the document count for a collection in a given time period +// collection should support field created_on. The results are projected on a per-project counter. +func (store *MongoStoreWithOfficialDriver) getDocCountForCollectionPerProject(ctx context.Context, startDate, + endDate time.Time, projectUUIDs []string, collection *mongo.Collection) (map[string]int64, error) { + + var resourceCounts []QProjectResourceCount + + condQuery := []bson.M{ + { + "created_on": bson.M{ + "$gte": startDate, + }, + }, + { + "created_on": bson.M{ + "$lte": endDate, + }, + }, + } + + if len(projectUUIDs) > 0 { + condQuery = append(condQuery, bson.M{ + "project_uuid": bson.M{ + "$in": projectUUIDs, + }, + }, + ) + } + + query := []bson.M{ + { + "$match": bson.M{ + "$and": condQuery, + }, + }, + { + "$group": bson.M{ + "_id": bson.M{ + "project_uuid": "$project_uuid", + }, + "resource_count": bson.M{ + "$sum": 1, + }, + }, + }, + { + "$project": bson.M{ + "_id": 0, + "project_uuid": "$_id.project_uuid", + "resource_count": 1, + }, + }, + } + + res := map[string]int64{} + + cursor, err := collection.Aggregate(ctx, query) + if err != nil { + return nil, err + } + + err = cursor.All(ctx, &resourceCounts) + if err != nil { + return nil, err + } + + for _, t := range resourceCounts { + res[t.ProjectUUID] = t.Count + } + + return res, nil +} + +// ##### OP METRICS QUERIES ##### + +// InsertOpMetric inserts an operational metric +func (store *MongoStoreWithOfficialDriver) InsertOpMetric(ctx context.Context, hostname string, cpu float64, mem float64) error { + doc := bson.M{"hostname": hostname, "cpu": cpu, "mem": mem} + change := bson.M{"$set": bson.M{"hostname": hostname, "cpu": cpu, "mem": mem}} + err := store.upsert(ctx, doc, change, store.opMetricsCollection) + if err != nil { + store.logErrorAndCrash(ctx, "InsertOpMetric", err) + return err + } + return nil +} + +// GetOpMetrics returns the operational metrics from datastore +func (store *MongoStoreWithOfficialDriver) GetOpMetrics(ctx context.Context) []QopMetric { + var results []QopMetric + cursor, err := store.opMetricsCollection.Find(ctx, bson.M{}) + if err != nil { + store.logErrorAndCrash(ctx, "GetOpMetrics", err) + return results + } + err = cursor.All(ctx, &results) + if err != nil { + store.logErrorAndCrash(ctx, "GetOpMetrics", err) + return results + } + return results +} + +// ##### ROLES QUERIES ##### + +func (store *MongoStoreWithOfficialDriver) HasResourceRoles(ctx context.Context, resource string, roles []string) bool { + query := bson.M{ + "resource": resource, + "roles": bson.M{ + "$in": roles, + }, + } + var results []QRole + cursor, err := store.rolesCollection.Find(ctx, query) + if err != nil { + store.logErrorAndCrash(ctx, "HasResourceRoles", err) + return false + } + err = cursor.All(ctx, &results) + if err != nil { + store.logErrorAndCrash(ctx, "HasResourceRoles", err) + return false + } + return len(results) > 0 +} + +func (store *MongoStoreWithOfficialDriver) InsertResourceRoles(ctx context.Context, resource string, roles []string) error { + role := QRole{ + Name: resource, + Roles: roles, + } + _, err := store.rolesCollection.InsertOne(ctx, role) + if err != nil { + store.logErrorAndCrash(ctx, "InsertResourceRoles", err) + return err + } + return nil +} + +// GetAllRoles returns a list of all available roles +func (store *MongoStoreWithOfficialDriver) GetAllRoles(ctx context.Context) []string { + cursor, err := store.rolesCollection.Distinct(ctx, "roles", bson.M{}) + if err != nil { + store.logErrorAndCrash(ctx, "GetAllRoles", err) + return []string{} + } + var roles []string + for _, i := range cursor { + roles = append(roles, i.(string)) + } + return roles + +} + +// ##### ACL QUERIES ###### + +// QueryACL queries topic or subscription for a list of authorized users +func (store *MongoStoreWithOfficialDriver) QueryACL(ctx context.Context, projectUUID string, resource string, name string) (QAcl, error) { + + var results []QAcl + var c *mongo.Collection + + if resource == "topics" { + c = store.topicsCollection + } else if resource == "subscriptions" { + c = store.subscriptionsCollection + } else { + return QAcl{}, errors.New("wrong resource type") + } + + query := bson.M{"project_uuid": projectUUID, "name": name} + + cursor, err := c.Find(ctx, query) + if err != nil { + store.logErrorAndCrash(ctx, "QueryACL", err) + return QAcl{}, err + } + err = cursor.All(ctx, &results) + if err != nil { + store.logErrorAndCrash(ctx, "QueryACL", err) + return QAcl{}, err + } + if len(results) > 0 { + return results[0], nil + } + + return QAcl{}, DocNotFound{} +} + +// ExistsInACL checks if a user is part of a topic's or sub's acl +func (store *MongoStoreWithOfficialDriver) ExistsInACL(ctx context.Context, projectUUID string, resource string, resourceName string, userUUID string) error { + + var c *mongo.Collection + if resource == "topics" { + c = store.topicsCollection + } else if resource == "subscriptions" { + c = store.subscriptionsCollection + } else { + return errors.New("wrong resource type") + } + + query := bson.M{ + "project_uuid": projectUUID, + "name": resourceName, + "acl": bson.M{ + "$in": []string{userUUID}, + }, + } + + err := c.FindOne(ctx, query).Err() + if err != nil { + if err == mongo.ErrNoDocuments { + return DocNotFound{} + } else { + store.logErrorAndCrash(ctx, "ExistsInAcl", err) + return err + } + } + return nil +} + +// ModACL modifies the push configuration +func (store *MongoStoreWithOfficialDriver) ModACL(ctx context.Context, projectUUID string, + resource string, name string, acl []string) error { + var c *mongo.Collection + if resource == "topics" { + c = store.topicsCollection + } else if resource == "subscriptions" { + c = store.subscriptionsCollection + } else { + return errors.New("wrong resource type") + } + doc := bson.M{"project_uuid": projectUUID, "name": name} + change := bson.M{"$set": bson.M{"acl": acl}} + _, err := c.UpdateOne(ctx, doc, change) + if err != nil { + store.logErrorAndCrash(ctx, "ModACL", err) + return err + } + + return nil +} + +// AppendToACL adds additional users to an existing ACL +func (store *MongoStoreWithOfficialDriver) AppendToACL(ctx context.Context, projectUUID string, resource string, name string, acl []string) error { + + var c *mongo.Collection + if resource == "topics" { + c = store.topicsCollection + } else if resource == "subscriptions" { + c = store.subscriptionsCollection + } else { + return errors.New("wrong resource type") + } + doc := bson.M{ + "project_uuid": projectUUID, + "name": name, + } + change := bson.M{ + "$addToSet": bson.M{ + "acl": bson.M{ + "$each": acl, + }, + }} + _, err := c.UpdateOne(ctx, doc, change) + if err != nil { + store.logErrorAndCrash(ctx, "AppendToACL", err) + return err + } + + return nil +} + +// RemoveFromACL removes users for a given ACL +func (store *MongoStoreWithOfficialDriver) RemoveFromACL(ctx context.Context, projectUUID string, resource string, name string, acl []string) error { + var c *mongo.Collection + if resource == "topics" { + c = store.topicsCollection + } else if resource == "subscriptions" { + c = store.subscriptionsCollection + } else { + return errors.New("wrong resource type") + } + + doc := bson.M{ + "project_uuid": projectUUID, + "name": name, + } + change := bson.M{ + "$pullAll": bson.M{ + "acl": acl, + }, + } + _, err := c.UpdateOne(ctx, doc, change) + if err != nil { + store.logErrorAndCrash(ctx, "RemoveFromACL", err) + return err + } + return nil +} + +// ##### SCHEMA QUERIES ##### + +func (store *MongoStoreWithOfficialDriver) InsertSchema(ctx context.Context, projectUUID, schemaUUID, name, + schemaType, rawSchemaString string) error { + schema := QSchema{ + ProjectUUID: projectUUID, + UUID: schemaUUID, + Name: name, + Type: schemaType, + RawSchema: rawSchemaString, + } + _, err := store.schemasCollection.InsertOne(ctx, schema) + if err != nil { + store.logErrorAndCrash(ctx, "InsertSchema", err) + return err + } + return nil +} + +func (store *MongoStoreWithOfficialDriver) QuerySchemas(ctx context.Context, projectUUID, schemaUUID, name string) ([]QSchema, error) { + + query := bson.M{"project_uuid": projectUUID} + + if name != "" { + query["name"] = name + } + + if schemaUUID != "" { + query["uuid"] = schemaUUID + } + + results, err := store.schemasFindQueryProcessor.execute(ctx, query) + if err != nil { + store.logErrorAndCrash(ctx, "QuerySchemas", err) + return nil, err + } + + return results, nil + +} + +func (store *MongoStoreWithOfficialDriver) UpdateSchema(ctx context.Context, schemaUUID, name, schemaType, rawSchemaString string) error { + + doc := bson.M{"uuid": schemaUUID} + + updates := bson.M{} + + if name != "" { + updates["name"] = name + } + + if schemaType != "" { + updates["type"] = schemaType + } + + if rawSchemaString != "" { + updates["raw_schema"] = rawSchemaString + } + + change := bson.M{"$set": updates} + _, err := store.schemasCollection.UpdateOne(ctx, doc, change) + if err != nil { + store.logErrorAndCrash(ctx, "QuerySchemas", err) + return err + } + return nil +} + +// DeleteSchema removes the schema from the store +// It also clears all the respective topics from the schema_uuid of the deleted schema +func (store *MongoStoreWithOfficialDriver) DeleteSchema(ctx context.Context, schemaUUID string) error { + schemaQuery := bson.M{"uuid": schemaUUID} + _, err := store.schemasCollection.DeleteOne(ctx, schemaQuery) + if err != nil { + store.logErrorAndCrash(ctx, "DeleteSchema", err) + return err + } + doc := bson.M{"schema_uuid": schemaUUID} + change := bson.M{ + "$set": bson.M{ + "schema_uuid": "", + }, + } + _, err = store.topicsCollection.UpdateMany(ctx, doc, change) + if err != nil { + store.logErrorAndCrash(ctx, "DeleteSchema-2", err) + return err + } + + return nil +} + +// ##### PROJECT QUERIES ##### + +// QueryProjects queries the database for a specific project or a list of all projects +func (store *MongoStoreWithOfficialDriver) QueryProjects(ctx context.Context, uuid string, name string) ([]QProject, error) { + + query := bson.M{} + + if name != "" { + + query = bson.M{"name": name} + + } else if uuid != "" { + query = bson.M{"uuid": uuid} + } + + results, err := store.projectsFindQueryProcessor.execute(ctx, query) + if err != nil { + store.logErrorAndCrash(ctx, "QueryProjects", err) + return nil, err + } + + if len(results) > 0 { + return results, nil + } + + return results, errors.New("not found") +} + +// UpdateProject updates project information +func (store *MongoStoreWithOfficialDriver) UpdateProject(ctx context.Context, projectUUID string, name string, + description string, modifiedOn time.Time) error { + + doc := bson.M{"uuid": projectUUID} + results, err := store.QueryProjects(ctx, projectUUID, "") + if err != nil { + store.logErrorAndCrash(ctx, "UpdateProject", err) + return err + } + + curPr := results[0] + curPr.ModifiedOn = modifiedOn // modifiedOn should always be updated + + if name != "" { + // Check if name is going to change and if that name already exists + if name != curPr.Name { + if sameRes, _ := store.QueryProjects(ctx, "", name); len(sameRes) > 0 { + return errors.New("invalid project name change, name already exists") + } + } + curPr.Name = name + } + + if description != "" { + curPr.Description = description + } + + change := bson.M{"$set": curPr} + _, err = store.projectsCollection.UpdateOne(ctx, doc, change) + if err != nil { + store.logErrorAndCrash(ctx, "UpdateProject", err) + return err + } + return nil +} + +// RemoveProject removes a project from the store +func (store *MongoStoreWithOfficialDriver) RemoveProject(ctx context.Context, uuid string) error { + project := bson.M{"uuid": uuid} + _, err := store.projectsCollection.DeleteOne(ctx, project) + if err != nil { + store.logErrorAndCrash(ctx, "RemoveProject", err) + return err + } + return nil +} + +// RemoveProjectTopics removes all topics related to a project UUID +func (store *MongoStoreWithOfficialDriver) RemoveProjectTopics(ctx context.Context, projectUUID string) error { + topics := bson.M{"project_uuid": projectUUID} + _, err := store.topicsCollection.DeleteMany(ctx, topics) + if err != nil { + store.logErrorAndCrash(ctx, "RemoveProjectTopics", err) + return err + } + return nil +} + +// RemoveProjectSubs removes all subscriptions related to a project UUID +func (store *MongoStoreWithOfficialDriver) RemoveProjectSubs(ctx context.Context, projectUUID string) error { + subs := bson.M{"project_uuid": projectUUID} + _, err := store.subscriptionsCollection.DeleteMany(ctx, subs) + if err != nil { + store.logErrorAndCrash(ctx, "RemoveProjectSubs", err) + return err + } + return nil +} + +// RemoveProjectDailyMessageCounters removes all message counts related to a project UUID +func (store *MongoStoreWithOfficialDriver) RemoveProjectDailyMessageCounters(ctx context.Context, projectUUID string) error { + counts := bson.M{"project_uuid": projectUUID} + _, err := store.topicsDailyMsgCountCollection.DeleteMany(ctx, counts) + if err != nil { + store.logErrorAndCrash(ctx, "RemoveProjectDailyMessageCounters", err) + return err + } + return nil +} + +// QueryDailyProjectMsgCount queries the total messages per day for a given project +func (store *MongoStoreWithOfficialDriver) QueryDailyProjectMsgCount(ctx context.Context, projectUUID string) ([]QDailyProjectMsgCount, error) { + + query := []bson.M{ + { + "$match": bson.M{ + "project_uuid": projectUUID, + }, + }, + { + "$group": bson.M{ + "_id": bson.M{ + "date": "$date", + }, + "msg_count": bson.M{ + "$sum": "$msg_count", + }, + }, + }, + { + "$sort": bson.M{ + "_id": -1, + }, + }, + { + "$limit": 30, + }, + { + "$project": bson.M{ + "_id": 0, + "date": "$_id.date", + "msg_count": 1, + }, + }, + } + var res []QDailyProjectMsgCount + cursor, err := store.topicsDailyMsgCountCollection.Aggregate(ctx, query) + if err != nil { + return res, err + } + + err = cursor.All(ctx, &res) + if err != nil { + store.logErrorAndCrash(ctx, "QueryDailyProjectMsgCount", err) + return res, err + } + return res, nil +} + +func (store *MongoStoreWithOfficialDriver) QueryTotalMessagesPerProject(ctx context.Context, projectUUIDs []string, startDate time.Time, endDate time.Time) ([]QProjectMessageCount, error) { + + if endDate.Before(startDate) { + startDate, endDate = endDate, startDate + } + + days := 1 + if !endDate.Equal(startDate) { + days = int(endDate.Sub(startDate).Hours() / 24) + // add an extra day to compensate for the fact that we need the starting day included as well + // e.g. Aug 1 to Aug 31 should be calculated as 31 days and not as 30 + days += 1 + } + + condQuery := []bson.M{ + { + "date": bson.M{ + "$gte": startDate, + }, + }, + { + "date": bson.M{ + "$lte": endDate, + }, + }, + } + + if len(projectUUIDs) > 0 { + condQuery = append(condQuery, bson.M{ + "project_uuid": bson.M{ + "$in": projectUUIDs, + }, + }, + ) + } + + query := []bson.M{ + { + "$match": bson.M{ + "$and": condQuery, + }, + }, + { + "$group": bson.M{ + "_id": bson.M{ + "project_uuid": "$project_uuid", + }, + "msg_count": bson.M{ + "$sum": "$msg_count", + }, + }, + }, + { + "$project": bson.M{ + "_id": 0, + "project_uuid": "$_id.project_uuid", + "msg_count": 1, + "avg_daily_msg": bson.M{ + "$divide": []interface{}{"$msg_count", days}, + }, + }, + }, + } + + var res []QProjectMessageCount + cursor, err := store.topicsDailyMsgCountCollection.Aggregate(ctx, query) + if err != nil { + return res, err + } + + err = cursor.All(ctx, &res) + if err != nil { + store.logErrorAndCrash(ctx, "QueryTotalMessagesPerProject", err) + return res, err + } + return res, nil +} + +func (store *MongoStoreWithOfficialDriver) InsertProject(ctx context.Context, uuid string, name string, + createdOn time.Time, modifiedOn time.Time, createdBy string, description string) error { + project := QProject{ + UUID: uuid, + Name: name, + CreatedOn: createdOn, + ModifiedOn: modifiedOn, + CreatedBy: createdBy, + Description: description, + } + _, err := store.projectsCollection.InsertOne(ctx, project) + if err != nil { + store.logErrorAndCrash(ctx, "InsertProject", err) + return err + } + return nil +} + +func (store *MongoStoreWithOfficialDriver) HasProject(ctx context.Context, name string) bool { + query := bson.M{"name": name} + results, err := store.projectsFindQueryProcessor.execute(ctx, query) + if err != nil { + store.logErrorAndCrash(ctx, "HasProject", err) + return false + } + + if len(results) > 0 { + return true + } + + return false +} + +// ##### USER REGISTRATIONS QUERIES ##### + +// RegisterUser inserts a new user registration to the database +func (store *MongoStoreWithOfficialDriver) RegisterUser(ctx context.Context, uuid, name, firstName, lastName, email, + org, desc, registeredAt, atkn, status string) error { + ur := QUserRegistration{ + UUID: uuid, + Name: name, + FirstName: firstName, + LastName: lastName, + Email: email, + Organization: org, + Description: desc, + RegisteredAt: registeredAt, + ActivationToken: atkn, + Status: status, + } + _, err := store.userRegistrationsCollection.InsertOne(ctx, ur) + if err != nil { + store.logErrorAndCrash(ctx, "RegisterUSer", err) + return err + } + return nil +} + +// DeleteRegistration removes the respective registration from the +func (store *MongoStoreWithOfficialDriver) DeleteRegistration(ctx context.Context, uuid string) error { + query := bson.M{"uuid": uuid} + err := store.deleteOne(ctx, store.userRegistrationsCollection, query) + if err != nil { + if (err == DocNotFound{}) { + return err + } else { + store.logErrorAndCrash(ctx, "RemoveSub", err) + return err + } + } + return nil +} + +func (store *MongoStoreWithOfficialDriver) QueryRegistrations(ctx context.Context, regUUID, status, activationToken, name, email, org string) ([]QUserRegistration, error) { + query := bson.M{} + + if regUUID != "" { + query["uuid"] = regUUID + } + + if status != "" { + query["status"] = status + } + + if activationToken != "" { + query["activation_token"] = activationToken + } + + if name != "" { + query["name"] = name + } + + if email != "" { + query["email"] = email + } + + if org != "" { + query["organization"] = org + } + + res, err := store.userRegistrationsFindQueryProcessor.execute(ctx, query) + if err != nil { + store.logErrorAndCrash(ctx, "QueryRegistrations", err) + return nil, err + } + + return res, nil +} + +func (store *MongoStoreWithOfficialDriver) UpdateRegistration(ctx context.Context, regUUID, status, declineComment, modifiedBy, modifiedAt string) error { + + doc := bson.M{"uuid": regUUID} + change := bson.M{ + "$set": bson.M{ + "status": status, + "decline_comment": declineComment, + "modified_by": modifiedBy, + "modified_at": modifiedAt, + "activation_token": "", + }, + } + _, err := store.userRegistrationsCollection.UpdateOne(ctx, doc, change) + if err != nil { + store.logErrorAndCrash(ctx, "UpdateRegistrations", err) + return err + } + + return nil +} + +// ###### USER QUERIES ###### + +// HasUsers accepts a user array of usernames and returns the not found +func (store *MongoStoreWithOfficialDriver) HasUsers(ctx context.Context, projectUUID string, users []string) (bool, []string) { + var results []QUser + var notFound []string + + query := bson.M{ + "projects": bson.M{ + "$elemMatch": bson.M{ + "project_uuid": projectUUID, + }, + }, + "name": bson.M{"$in": users}, + } + + results, err := store.usersFindQueryProcessor.execute(ctx, query) + if err != nil { + store.logErrorAndCrash(ctx, "HasUsers", err) + return false, []string{err.Error()} + } + + // for each given username + for _, username := range users { + found := false + // loop through all found users + for _, user := range results { + if username == user.Name { + found = true + } + } + // if not found add it to the notFound + if !found { + notFound = append(notFound, username) + } + + } + + return len(notFound) == 0, notFound +} + +// PaginatedQueryUsers returns a page of users +func (store *MongoStoreWithOfficialDriver) PaginatedQueryUsers(ctx context.Context, pageToken string, pageSize int64, + projectUUID string) ([]QUser, int64, string, error) { + + var qUsers []QUser + var totalSize int64 + var limit int64 + var nextPageToken string + var err error + var query bson.M + + // if the page size is other than zero(where zero means, no limit), try to grab one more document to check if there + // will be a next page after the current one + if pageSize > 0 { + limit = pageSize + 1 + } + + // if projectUUID is empty string return all users, if projectUUID has a non empty value + // query users that belong to that project + if projectUUID != "" { + query = bson.M{ + "projects": bson.M{ + "$elemMatch": bson.M{ + "project_uuid": projectUUID, + }, + }, + } + } + + // check the total of the users selected by the query not taking into account pagination + totalSize, err = store.usersCollection.CountDocuments(ctx, query) + if err != nil { + store.logErrorAndCrash(ctx, "PaginatedQueryUsers", err) + } + + // now take into account if pagination is enabled and change the query accordingly + // first check if an pageToken is provided and whether is a valid bson ID + if pageToken != "" { + bsonID, err := primitive.ObjectIDFromHex(pageToken) + if err != nil { + err = fmt.Errorf("page token %s is not a valid bson ObjectId. %s", pageToken, err.Error()) + log.WithFields( + log.Fields{ + "type": "backend_log", + "trace_id": ctx.Value("trace_id"), + "backend_service": "mongo", + "page_token": pageToken, + "err": err.Error(), + }, + ).Error("Page token is not a valid bson ObjectId") + return qUsers, totalSize, nextPageToken, err + } + + // now that the paginated query is constructed from start take into account again + // if projectUUID is provided to query only the users of a given project + if projectUUID != "" { + query = bson.M{ + "projects": bson.M{ + "$elemMatch": bson.M{ + "project_uuid": projectUUID, + }, + }, + "_id": bson.M{ + "$lte": bsonID, + }, + } + + } else { + + query = bson.M{ + "_id": bson.M{ + "$lte": bsonID, + }, + } + } + + } + findOptions := options.Find().SetLimit(limit).SetSort(bson.M{"_id": -1}) + qUsers, err = store.usersFindQueryProcessor.execute(ctx, query, findOptions) + if err != nil { + store.logErrorAndCrash(ctx, "PaginatedQueryUsers-2", err) + } + + // if the amount of users that were found was equal to the limit, its a sign that there are users to populate the next page + // so pick the last element's pageToken to use as the starting point for the next page + // and eliminate the extra element from the current response + if pageSize > 0 && len(qUsers) > 0 && len(qUsers) == int(limit) { + + nextPageToken = qUsers[limit-1].ID.(primitive.ObjectID).Hex() + qUsers = qUsers[:len(qUsers)-1] + } + + return qUsers, totalSize, nextPageToken, err + +} + +// QueryUsers queries user(s) information belonging to a project +func (store *MongoStoreWithOfficialDriver) QueryUsers(ctx context.Context, projectUUID string, + uuid string, name string) ([]QUser, error) { + + // By default, return all users + query := bson.M{} + // If project UUID is given return users that belong to the project + if projectUUID != "" { + query = bson.M{"projects.project_uuid": projectUUID} + if uuid != "" { + query = bson.M{"projects.project_uuid": projectUUID, "uuid": uuid} + } else if name != "" { + query = bson.M{"projects.project_uuid": projectUUID, "name": name} + } + } else { + if uuid != "" { + query = bson.M{"uuid": uuid} + } else if name != "" { + query = bson.M{"name": name} + + } + } + + results, err := store.usersFindQueryProcessor.execute(ctx, query) + if err != nil { + store.logErrorAndCrash(ctx, "QueryUsers", err) + } + + return results, err +} + +// UpdateUser updates user information +func (store *MongoStoreWithOfficialDriver) UpdateUser(ctx context.Context, uuid, fname, lname, org, + desc string, projects []QProjectRoles, name string, email string, serviceRoles []string, modifiedOn time.Time) error { + + doc := bson.M{"uuid": uuid} + results, err := store.QueryUsers(ctx, "", uuid, "") + if err != nil { + return err + } + + curUsr := results[0] + + if name != "" { + // Check if name is going to change and if that name already exists + if name != curUsr.Name { + if sameRes, _ := store.QueryUsers(ctx, "", "", name); len(sameRes) > 0 { + return errors.New("invalid user name change, name already exists") + } + } + curUsr.Name = name + } + + if email != "" { + curUsr.Email = email + } + + if fname != "" { + curUsr.FirstName = fname + } + + if lname != "" { + curUsr.LastName = lname + } + + if org != "" { + curUsr.Organization = org + } + + if desc != "" { + curUsr.Description = desc + } + + if projects != nil { + curUsr.Projects = projects + } + + if serviceRoles != nil { + curUsr.ServiceRoles = serviceRoles + } + + curUsr.ModifiedOn = modifiedOn + + change := bson.M{"$set": curUsr} + + _, err = store.usersCollection.UpdateOne(ctx, doc, change) + if err != nil { + store.logErrorAndCrash(ctx, "UpdateUser", err) + } + return err + +} + +// AppendToUserProjects appends a new unique project to the user's projects +func (store *MongoStoreWithOfficialDriver) AppendToUserProjects(ctx context.Context, userUUID string, projectUUID string, pRoles ...string) error { + doc := bson.M{"uuid": userUUID} + change := bson.M{ + "$addToSet": bson.M{ + "projects": QProjectRoles{ + ProjectUUID: projectUUID, + Roles: pRoles, + }, + }, + } + _, err := store.usersCollection.UpdateOne(ctx, doc, change) + if err != nil { + store.logErrorAndCrash(ctx, "AppendToUserProjects", err) + } + return err +} + +// UpdateUserToken updates user's token +func (store *MongoStoreWithOfficialDriver) UpdateUserToken(ctx context.Context, uuid string, token string) error { + doc := bson.M{"uuid": uuid} + change := bson.M{"$set": bson.M{"token": token}} + _, err := store.usersCollection.UpdateOne(ctx, doc, change) + if err != nil { + store.logErrorAndCrash(ctx, "UpdateUserToken", err) + } + return err +} + +func (store *MongoStoreWithOfficialDriver) RemoveUser(ctx context.Context, uuid string) error { + user := bson.M{"uuid": uuid} + _, err := store.usersCollection.DeleteOne(ctx, user) + if err != nil { + store.logErrorAndCrash(ctx, "RemoveUser", err) + } + return err +} + +// InsertUser inserts a new user to the store +func (store *MongoStoreWithOfficialDriver) InsertUser(ctx context.Context, uuid string, projects []QProjectRoles, + name string, firstName string, lastName string, org string, desc string, token string, email string, serviceRoles []string, createdOn time.Time, modifiedOn time.Time, createdBy string) error { + user := QUser{ + UUID: uuid, + Name: name, + Email: email, + Token: token, + FirstName: firstName, + LastName: lastName, + Organization: org, + Description: desc, + Projects: projects, + ServiceRoles: serviceRoles, + CreatedOn: createdOn, + ModifiedOn: modifiedOn, + CreatedBy: createdBy, + } + _, err := store.usersCollection.InsertOne(ctx, user) + if err != nil { + store.logErrorAndCrash(ctx, "InsertUser", err) + } + return err +} + +// GetUserFromToken returns user information from a specific token +func (store *MongoStoreWithOfficialDriver) GetUserFromToken(ctx context.Context, token string) (QUser, error) { + + query := bson.M{"token": token} + results, err := store.usersFindQueryProcessor.execute(ctx, query) + + if err != nil { + store.logErrorAndCrash(ctx, "GetUserFromToken", err) + return QUser{}, err + } + + if len(results) == 0 { + return QUser{}, DocNotFound{} + } + + if len(results) > 1 { + log.WithFields( + log.Fields{ + "type": "backend_log", + "trace_id": ctx.Value("trace_id"), + "token": token, + "backend_service": "mongo", + "backend_hosts": store.Server, + }, + ).Warning("Multiple users with the same token") + } + + // Search the found user for project roles + return results[0], err +} + +// UsersCount returns the amount of users created in the given time period per project +func (store *MongoStoreWithOfficialDriver) UsersCount(ctx context.Context, startDate, endDate time.Time, projectUUIDs []string) (map[string]int64, error) { + + var resourceCounts []QProjectResourceCount + + condQuery := []bson.M{ + { + "created_on": bson.M{ + "$gte": startDate, + }, + }, + { + "created_on": bson.M{ + "$lte": endDate, + }, + }, + } + + if len(projectUUIDs) > 0 { + condQuery = append(condQuery, bson.M{ + "project_uuid": bson.M{ + "$in": projectUUIDs, + }, + }, + ) + } + + query := []bson.M{ + { + "$unwind": "$projects", + }, + { + "$match": bson.M{ + "$and": condQuery, + }, + }, + { + "$group": bson.M{ + "_id": bson.M{ + "project_uuid": "$projects.project_uuid", + }, + "resource_count": bson.M{ + "$sum": 1, + }, + }, + }, + { + "$project": bson.M{ + "_id": 0, + "project_uuid": "$_id.project_uuid", + "resource_count": 1, + }, + }, + } + res := map[string]int64{} + + cursor, err := store.usersCollection.Aggregate(ctx, query) + if err != nil { + return res, err + } + + err = cursor.All(ctx, &resourceCounts) + if err != nil { + store.logErrorAndCrash(ctx, "UsersCount", err) + return res, err + } + + for _, t := range resourceCounts { + res[t.ProjectUUID] = t.Count + } + + return res, nil + +} + +func (store *MongoStoreWithOfficialDriver) GetUserRoles(ctx context.Context, projectUUID string, token string) ([]string, string) { + + query := bson.M{"token": token} + results, err := store.usersFindQueryProcessor.execute(ctx, query) + + if err != nil { + store.logErrorAndCrash(ctx, "GetUserRoles", err) + return []string{}, err.Error() + } + + if len(results) == 0 { + return []string{}, "" + } + + if len(results) > 1 { + log.WithFields( + log.Fields{ + "type": "backend_log", + "trace_id": ctx.Value("trace_id"), + "token": token, + "backend_service": "mongo", + "backend_hosts": store.Server, + }, + ).Warning("Multiple users with the same token") + } + + // Search the found user for project roles + return results[0].getProjectRoles(projectUUID), results[0].Name + +} + +// ##### SUBSCRIPTION QUERIES ##### + +// QuerySubsByTopic returns subscriptions of a specific topic +func (store *MongoStoreWithOfficialDriver) QuerySubsByTopic(ctx context.Context, projectUUID, topic string) ([]QSub, error) { + + // By default, return all subs of a given project + query := bson.M{"project_uuid": projectUUID} + + // If topic is given return only the specific topic + if topic != "" { + query = bson.M{"project_uuid": projectUUID, "topic": topic} + } + + results, err := store.subsFindQueryProcessor.execute(ctx, query) + if err != nil { + store.logErrorAndCrash(ctx, "QuerySubsByTopic", err) + } + + return results, err +} + +// QuerySubsByACL returns subscriptions that a specific username has access to +func (store *MongoStoreWithOfficialDriver) QuerySubsByACL(ctx context.Context, projectUUID, user string) ([]QSub, error) { + // By default, return all subs of a given project + query := bson.M{"project_uuid": projectUUID} + + // If name is given return only the specific topic + if user != "" { + query = bson.M{"project_uuid": projectUUID, "acl": user} + } + results, err := store.subsFindQueryProcessor.execute(ctx, query) + if err != nil { + store.logErrorAndCrash(ctx, "QuerySubsByACL", err) + } + return results, err +} + +// QuerySubs Query Subscription info from store +func (store *MongoStoreWithOfficialDriver) QuerySubs(ctx context.Context, projectUUID string, userUUID string, name string, pageToken string, pageSize int64) ([]QSub, int64, string, error) { + var err error + var totalSize int64 + var limit int64 + var nextPageToken string + var qSubs []QSub + + // By default, return all topics of a given project + query := bson.M{"project_uuid": projectUUID} + + // find all the topics for a specific user + if userUUID != "" { + query["acl"] = bson.M{"$in": []string{userUUID}} + } + + // if the page size is other than zero(where zero means, no limit), try to grab one more document to check if there + // will be a next page after the current one + if pageSize > 0 { + + limit = pageSize + 1 + + } + + // first check if an pageToken is provided and whether is a valid bson ID + if pageToken != "" { + bsonID, err := primitive.ObjectIDFromHex(pageToken) + if err != nil { + err = fmt.Errorf("page token %s is not a valid bson ObjectId. %s", pageToken, err.Error()) + log.WithFields( + log.Fields{ + "type": "backend_log", + "trace_id": ctx.Value("trace_id"), + "backend_service": "mongo", + "page_token": pageToken, + "err": err.Error(), + }, + ).Error("Page token is not a valid bson ObjectId") + return qSubs, totalSize, nextPageToken, err + } + + query["_id"] = bson.M{"$lte": bsonID} + + } else if name != "" { + + query["name"] = name + } + + limitFindOptions := options.Find().SetLimit(limit).SetSort(bson.M{"_id": -1}) + qSubs, err = store.subsFindQueryProcessor.execute(ctx, query, limitFindOptions) + + if name == "" { + + countQuery := bson.M{"project_uuid": projectUUID} + if userUUID != "" { + countQuery["acl"] = bson.M{"$in": []string{userUUID}} + } + + totalSize, err = store.subscriptionsCollection.CountDocuments(ctx, countQuery) + + if err != nil { + store.logErrorAndCrash(ctx, "QuerySubs", err) + + } + + // if the amount of topics that were found was equal to the limit, its a sign that there are topics to populate the next page + // so pick the last element's pageToken to use as the starting point for the next page + // and eliminate the extra element from the current response + if len(qSubs) > 0 && len(qSubs) == int(limit) { + + nextPageToken = qSubs[limit-1].ID.(primitive.ObjectID).Hex() + qSubs = qSubs[:len(qSubs)-1] + } + } + + return qSubs, totalSize, nextPageToken, err + +} + +// UpdateSubLatestConsume updates the subscription's latest consume time +func (store *MongoStoreWithOfficialDriver) UpdateSubLatestConsume(ctx context.Context, projectUUID string, name string, date time.Time) error { + doc := bson.M{ + "project_uuid": projectUUID, + "name": name, + } + + change := bson.M{ + "$set": bson.M{ + "latest_consume": date, + }, + } + _, err := store.subscriptionsCollection.UpdateOne(ctx, doc, change) + if err != nil { + store.logErrorAndCrash(ctx, "UpdateSubLatestConsume", err) + } + return err +} + +// UpdateSubConsumeRate updates the subscription's consume rate +func (store *MongoStoreWithOfficialDriver) UpdateSubConsumeRate(ctx context.Context, projectUUID string, name string, rate float64) error { + doc := bson.M{ + "project_uuid": projectUUID, + "name": name, + } + + change := bson.M{ + "$set": bson.M{ + "consume_rate": rate, + }, + } + _, err := store.subscriptionsCollection.UpdateOne(ctx, doc, change) + if err != nil { + store.logErrorAndCrash(ctx, "UpdateSubConsumeRate", err) + } + return err +} + +// RemoveSub removes a subscription from the store +func (store *MongoStoreWithOfficialDriver) RemoveSub(ctx context.Context, projectUUID string, name string) error { + sub := bson.M{"project_uuid": projectUUID, "name": name} + err := store.deleteOne(ctx, store.subscriptionsCollection, sub) + if err != nil { + if (err == DocNotFound{}) { + return err + } else { + store.logErrorAndCrash(ctx, "RemoveSub", err) + return err + } + } + return nil +} + +// IncrementSubBytes increases the total number of bytes consumed from a subscription +func (store *MongoStoreWithOfficialDriver) IncrementSubBytes(ctx context.Context, projectUUID string, name string, totalBytes int64) error { + doc := bson.M{"project_uuid": projectUUID, "name": name} + change := bson.M{"$inc": bson.M{"total_bytes": totalBytes}} + _, err := store.subscriptionsCollection.UpdateOne(ctx, doc, change) + if err != nil { + store.logErrorAndCrash(ctx, "IncrementSubBytes", err) + } + return err +} + +// IncrementSubMsgNum increments the number of messages pulled in a subscription +func (store *MongoStoreWithOfficialDriver) IncrementSubMsgNum(ctx context.Context, projectUUID string, name string, num int64) error { + doc := bson.M{"project_uuid": projectUUID, "name": name} + change := bson.M{"$inc": bson.M{"msg_num": num}} + _, err := store.subscriptionsCollection.UpdateOne(ctx, doc, change) + if err != nil { + store.logErrorAndCrash(ctx, "IncrementSubMsgNum", err) + } + return err +} + +func (store *MongoStoreWithOfficialDriver) InsertSub(ctx context.Context, projectUUID string, name string, topic string, + offset int64, ack int, pushCfg QPushConfig, createdOn time.Time) error { + + sub := QSub{ + ProjectUUID: projectUUID, + Name: name, + Topic: topic, + Offset: offset, + NextOffset: 0, + PendingAck: "", + Ack: ack, + PushType: pushCfg.Type, + MaxMessages: pushCfg.MaxMessages, + AuthorizationType: pushCfg.AuthorizationType, + AuthorizationHeader: pushCfg.AuthorizationHeader, + PushEndpoint: pushCfg.PushEndpoint, + RetPolicy: pushCfg.RetPolicy, + RetPeriod: pushCfg.RetPeriod, + VerificationHash: pushCfg.VerificationHash, + Verified: pushCfg.Verified, + MattermostUrl: pushCfg.MattermostUrl, + MattermostChannel: pushCfg.MattermostChannel, + MattermostUsername: pushCfg.MattermostUsername, + Base64Decode: pushCfg.Base64Decode, + MsgNum: 0, + TotalBytes: 0, + CreatedOn: createdOn, + ACL: []string{}, + } + _, err := store.subscriptionsCollection.InsertOne(ctx, sub) + if err != nil { + store.logErrorAndCrash(ctx, "InsertSub", err) + } + return err +} + +// QueryOneSub queries and returns specific sub of project +func (store *MongoStoreWithOfficialDriver) QueryOneSub(ctx context.Context, projectUUID string, name string) (QSub, error) { + query := bson.M{"project_uuid": projectUUID, "name": name} + results, err := store.subsFindQueryProcessor.execute(ctx, query) + if err != nil { + store.logErrorAndCrash(ctx, "QueryOneSub", err) + + } + + if len(results) > 0 { + return results[0], nil + } + + return QSub{}, errors.New("empty") +} + +// QueryPushSubs retrieves subscriptions that have a push_endpoint defined +func (store *MongoStoreWithOfficialDriver) QueryPushSubs(ctx context.Context) []QSub { + query := bson.M{"push_endpoint": bson.M{"$ne": ""}} + results, err := store.subsFindQueryProcessor.execute(ctx, query) + if err != nil { + store.logErrorAndCrash(ctx, "QueryPushSubs", err) + + } + return results +} + +// SubscriptionsCount returns the amount of subscriptions created in the given time period per project +func (store *MongoStoreWithOfficialDriver) SubscriptionsCount(ctx context.Context, startDate, endDate time.Time, + projectUUIDs []string) (map[string]int64, error) { + + res, err := store.getDocCountForCollectionPerProject(ctx, startDate, endDate, projectUUIDs, store.subscriptionsCollection) + if err != nil { + store.logErrorAndCrash(ctx, "SubscriptionsCount", err) + } + return res, err +} + +// ModAck modifies the subscription's ack timeout field in mongodb +func (store *MongoStoreWithOfficialDriver) ModAck(ctx context.Context, projectUUID string, name string, ack int) error { + doc := bson.M{"project_uuid": projectUUID, "name": name} + change := bson.M{"$set": bson.M{"ack": ack}} + _, err := store.subscriptionsCollection.UpdateOne(ctx, doc, change) + if err != nil { + store.logErrorAndCrash(ctx, "ModAck", err) + } + return err +} + +// UpdateSubOffset updates a subscription offset +func (store *MongoStoreWithOfficialDriver) UpdateSubOffset(ctx context.Context, projectUUID string, name string, offset int64) { + doc := bson.M{"project_uuid": projectUUID, "name": name} + change := bson.M{"$set": bson.M{"offset": offset, "next_offset": 0, "pending_ack": ""}} + _, err := store.subscriptionsCollection.UpdateOne(ctx, doc, change) + if err != nil { + store.logErrorAndCrash(ctx, "UpdateSubOffset", err) + } +} + +func (store *MongoStoreWithOfficialDriver) UpdateSubPull(ctx context.Context, projectUUID string, name string, + nextOff int64, ts string) error { + + doc := bson.M{"project_uuid": projectUUID, "name": name} + change := bson.M{"$set": bson.M{"next_offset": nextOff, "pending_ack": ts}} + _, err := store.subscriptionsCollection.UpdateOne(ctx, doc, change) + if err != nil { + store.logErrorAndCrash(ctx, "UpdateSubPull", err) + return err + } + return nil +} + +func (store *MongoStoreWithOfficialDriver) UpdateSubOffsetAck(ctx context.Context, projectUUID string, name string, + offset int64, ts string) error { + + // Get Info + res := QSub{} + query := bson.M{"project_uuid": projectUUID, "name": name} + results, err := store.subsFindQueryProcessor.execute(ctx, query) + if err != nil { + store.logErrorAndCrash(ctx, "UpdateSubOffsetAck", err) + return err + } + if len(results) == 0 { + return errors.New("sub not found during UpdateSubOffsetAck") + } + res = results[0] + + // check if no ack pending + if res.NextOffset == 0 { + return errors.New("no ack pending") + } + + // check if ack offset is wrong - wrong ack + if offset <= res.Offset || offset > res.NextOffset { + return errors.New("wrong ack") + } + + // check if ack has timeout + zSec := "2006-01-02T15:04:05Z" + timeGiven, _ := time.Parse(zSec, ts) + timeRef, _ := time.Parse(zSec, res.PendingAck) + durSec := timeGiven.Sub(timeRef).Seconds() + + if int(durSec) > res.Ack { + return errors.New("ack timeout") + } + + doc := bson.M{"project_uuid": projectUUID, "name": name} + change := bson.M{ + "$set": bson.M{ + "offset": offset, + "next_offset": 0, + "pending_ack": "", + }, + } + _, err = store.subscriptionsCollection.UpdateOne(ctx, doc, change) + if err != nil { + store.logErrorAndCrash(ctx, "UpdateSubOffsetAck", err) + } + return err +} + +// ModSubPush modifies the push configuration +func (store *MongoStoreWithOfficialDriver) ModSubPush(ctx context.Context, projectUUID string, + name string, pushCfg QPushConfig) error { + + doc := bson.M{ + "project_uuid": projectUUID, + "name": name, + } + + change := bson.M{"$set": bson.M{ + "push_type": pushCfg.Type, + "push_endpoint": pushCfg.PushEndpoint, + "authorization_type": pushCfg.AuthorizationType, + "authorization_header": pushCfg.AuthorizationHeader, + "max_messages": pushCfg.MaxMessages, + "retry_policy": pushCfg.RetPolicy, + "retry_period": pushCfg.RetPeriod, + "verification_hash": pushCfg.VerificationHash, + "verified": pushCfg.Verified, + "mattermost_url": pushCfg.MattermostUrl, + "mattermost_username": pushCfg.MattermostUsername, + "mattermost_channel": pushCfg.MattermostChannel, + "base_64_decode": pushCfg.Base64Decode, + }, + } + ur, err := store.subscriptionsCollection.UpdateOne(ctx, doc, change) + if err != nil { + store.logErrorAndCrash(ctx, "ModSubPush", err) + return err + } + if ur.MatchedCount == 0 { + return DocNotFound{} + } + + return nil +} + +// ###### TOPIC QUERIES ###### + +func (store *MongoStoreWithOfficialDriver) LinkTopicSchema(ctx context.Context, projectUUID, name, schemaUUID string) error { + doc := bson.M{"project_uuid": projectUUID, "name": name} + change := bson.M{"$set": bson.M{"schema_uuid": schemaUUID}} + _, err := store.topicsCollection.UpdateOne(ctx, doc, change) + if err != nil { + store.logErrorAndCrash(ctx, "LinkTopicSchema", err) + return err + + } + return nil +} + +// QueryTopicsByACL returns topics that a specific username has access to +func (store *MongoStoreWithOfficialDriver) QueryTopicsByACL(ctx context.Context, projectUUID, user string) ([]QTopic, error) { + + // By default, return all topics of a given project + query := bson.M{"project_uuid": projectUUID} + + // If name is given return only the specific topic + if user != "" { + query = bson.M{"project_uuid": projectUUID, "acl": user} + } + + results, err := store.topicsFindQueryProcessor.execute(ctx, query) + + if err != nil { + store.logErrorAndCrash(ctx, "QueryTopicsByACL", err) + } + + return results, err + +} + +func (store *MongoStoreWithOfficialDriver) QueryTopics(ctx context.Context, projectUUID string, userUUID string, name string, pageToken string, pageSize int64) ([]QTopic, int64, string, error) { + + var err error + var totalSize int64 + var limit int64 + var nextPageToken string + var qTopics []QTopic + + // By default, return all topics of a given project + query := bson.M{"project_uuid": projectUUID} + + // find all the topics for a specific user + if userUUID != "" { + query["acl"] = bson.M{"$in": []string{userUUID}} + } + + // if the page size is other than zero(where zero means, no limit), try to grab one more document to check if there + // will be a next page after the current one + if pageSize > 0 { + + limit = pageSize + 1 + + } + + // first check if an pageToken is provided and whether is a valid bson ID + if pageToken != "" { + bsonID, err := primitive.ObjectIDFromHex(pageToken) + if err != nil { + err = fmt.Errorf("page token %s is not a valid bson ObjectId. %s", pageToken, err.Error()) + log.WithFields( + log.Fields{ + "type": "backend_log", + "trace_id": ctx.Value("trace_id"), + "backend_service": "mongo", + "page_token": pageToken, + "err": err.Error(), + }, + ).Error("Page token is not a valid bson ObjectId") + return qTopics, totalSize, nextPageToken, err + } + + query["_id"] = bson.M{"$lte": bsonID} + + } else if name != "" { + + query["name"] = name + } + + limitFindOptions := options.Find().SetLimit(limit).SetSort(bson.M{"_id": -1}) + qTopics, err = store.topicsFindQueryProcessor.execute(ctx, query, limitFindOptions) + + if name == "" { + + countQuery := bson.M{"project_uuid": projectUUID} + if userUUID != "" { + countQuery["acl"] = bson.M{"$in": []string{userUUID}} + } + + totalSize, err = store.topicsCollection.CountDocuments(ctx, countQuery) + + if err != nil { + store.logErrorAndCrash(ctx, "QueryTopics-2", err) + + } + + // if the amount of topics that were found was equal to the limit, its a sign that there are topics to populate the next page + // so pick the last element's pageToken to use as the starting point for the next page + // and eliminate the extra element from the current response + if len(qTopics) > 0 && len(qTopics) == int(limit) { + + nextPageToken = qTopics[limit-1].ID.(primitive.ObjectID).Hex() + qTopics = qTopics[:len(qTopics)-1] + } + } + + return qTopics, totalSize, nextPageToken, err + +} + +func (store *MongoStoreWithOfficialDriver) UpdateTopicLatestPublish(ctx context.Context, projectUUID string, name string, date time.Time) error { + doc := bson.M{ + "project_uuid": projectUUID, + "name": name, + } + + change := bson.M{ + "$set": bson.M{ + "latest_publish": date, + }, + } + + _, err := store.topicsCollection.UpdateOne(ctx, doc, change) + if err != nil { + store.logErrorAndCrash(ctx, "UpdateTopicLatestPublish", err) + } + return err +} + +func (store *MongoStoreWithOfficialDriver) UpdateTopicPublishRate(ctx context.Context, projectUUID string, name string, rate float64) error { + doc := bson.M{ + "project_uuid": projectUUID, + "name": name, + } + + change := bson.M{ + "$set": bson.M{ + "publish_rate": rate, + }, + } + + _, err := store.topicsCollection.UpdateOne(ctx, doc, change) + if err != nil { + store.logErrorAndCrash(ctx, "UpdateTopicPublishRate", err) + } + return err +} + +func (store *MongoStoreWithOfficialDriver) RemoveTopic(ctx context.Context, projectUUID string, name string) error { + query := bson.M{"project_uuid": projectUUID, "name": name} + err := store.deleteOne(ctx, store.topicsCollection, query) + if err != nil { + if (err == DocNotFound{}) { + return err + } else { + store.logErrorAndCrash(ctx, "RemoveTopic", err) + return err + } + } + return nil +} + +func (store *MongoStoreWithOfficialDriver) InsertTopic(ctx context.Context, projectUUID string, name string, schemaUUID string, createdOn time.Time) error { + topic := QTopic{ + ProjectUUID: projectUUID, + Name: name, + MsgNum: 0, + TotalBytes: 0, + LatestPublish: time.Time{}, + PublishRate: 0, + SchemaUUID: schemaUUID, + CreatedOn: createdOn, + ACL: []string{}, + } + _, err := store.topicsCollection.InsertOne(ctx, topic) + if err != nil { + store.logErrorAndCrash(ctx, "InsertTopic", err) + } + return err +} + +// TopicsCount returns the amount of topics created in the given time period per project +func (store *MongoStoreWithOfficialDriver) TopicsCount(ctx context.Context, startDate, endDate time.Time, projectUUIDs []string) (map[string]int64, error) { + res, err := store.getDocCountForCollectionPerProject(ctx, startDate, endDate, projectUUIDs, store.topicsCollection) + if err != nil { + store.logErrorAndCrash(ctx, "TopicsCount", err) + } + return res, err +} + +// QueryDailyTopicMsgCount returns results regarding the number of messages published to a topic +func (store *MongoStoreWithOfficialDriver) QueryDailyTopicMsgCount(ctx context.Context, projectUUID string, topicName string, date time.Time) ([]QDailyTopicMsgCount, error) { + + var err error + var qDailyTopicMsgCount []QDailyTopicMsgCount + var query bson.M + + // represents an empty time object + var zeroValueTime time.Time + + query = bson.M{"date": date, "project_uuid": projectUUID, "topic_name": topicName} + + // if nothing's specified return the whole collection + if projectUUID == "" && topicName == "" && date == zeroValueTime { + query = bson.M{} + } + + if projectUUID != "" && topicName != "" && date == zeroValueTime { + query = bson.M{"project_uuid": projectUUID, "topic_name": topicName} + } + + limit := int64(30) + findOptions := options.Find().SetLimit(limit).SetSort(bson.M{"date": -1}) + cursor, err := store.topicsDailyMsgCountCollection.Find(ctx, query, findOptions) + if err != nil { + store.logErrorAndCrash(ctx, "QueryDailyTopicMsgCount", err) + return qDailyTopicMsgCount, err + } + + err = cursor.All(ctx, &qDailyTopicMsgCount) + if err != nil { + store.logErrorAndCrash(ctx, "QueryDailyTopicMsgCount-2", err) + return qDailyTopicMsgCount, err + } + + return qDailyTopicMsgCount, err +} + +// IncrementTopicMsgNum increments the number of messages published in a topic +func (store *MongoStoreWithOfficialDriver) IncrementTopicMsgNum(ctx context.Context, projectUUID string, + name string, num int64) error { + + doc := bson.M{"project_uuid": projectUUID, "name": name} + change := bson.M{"$inc": bson.M{"msg_num": num}} + + _, err := store.topicsCollection.UpdateOne(ctx, doc, change) + if err != nil { + store.logErrorAndCrash(ctx, "IncrementTopicMsgNum", err) + } + return err +} + +// IncrementDailyTopicMsgCount increments the daily count of published messages to a specific topic +func (store *MongoStoreWithOfficialDriver) IncrementDailyTopicMsgCount(ctx context.Context, projectUUID string, + topicName string, num int64, date time.Time) error { + + doc := bson.M{"date": date, "project_uuid": projectUUID, "topic_name": topicName} + change := bson.M{"$inc": bson.M{"msg_count": num}} + err := store.upsert(ctx, doc, change, store.topicsDailyMsgCountCollection) + if err != nil { + store.logErrorAndCrash(ctx, "IncrementDailyTopicMsgCount", err) + } + return err +} + +// IncrementTopicBytes increases the total number of bytes published in a topic +func (store *MongoStoreWithOfficialDriver) IncrementTopicBytes(ctx context.Context, projectUUID string, + name string, totalBytes int64) error { + + doc := bson.M{"project_uuid": projectUUID, "name": name} + change := bson.M{"$inc": bson.M{"total_bytes": totalBytes}} + + _, err := store.topicsCollection.UpdateOne(ctx, doc, change) + if err != nil { + store.logErrorAndCrash(ctx, "IncrementTopicBytes", err) + } + return err +} diff --git a/stores/mongo_store_integration_test.go b/stores/mongo_store_integration_test.go index f050977b..709f3bf0 100644 --- a/stores/mongo_store_integration_test.go +++ b/stores/mongo_store_integration_test.go @@ -8,7 +8,7 @@ import ( "fmt" "github.com/stretchr/testify/suite" "github.com/testcontainers/testcontainers-go" - "gopkg.in/mgo.v2/bson" + "go.mongodb.org/mongo-driver/bson/primitive" "testing" "time" ) @@ -70,7 +70,7 @@ func (suite *QListReverser[T]) reverse(s []T) []T { } func (suite *MongoStoreIntegrationTestSuite) assertSchemasEqual(expected []QSchema, actual []QSchema) { - suite.True(len(actual) == len(expected)) + suite.Equal(len(expected), len(actual)) for idx, schema := range expected { suite.Equal(schema.UUID, actual[idx].UUID, schema.Name) suite.Equal(schema.Name, actual[idx].Name, schema.Name) @@ -81,7 +81,7 @@ func (suite *MongoStoreIntegrationTestSuite) assertSchemasEqual(expected []QSche } func (suite *MongoStoreIntegrationTestSuite) assertUsersEqual(expected []QUser, actual []QUser) { - suite.True(len(actual) == len(expected)) + suite.Equal(len(expected), len(actual)) for idx, user := range expected { suite.Equal(user.UUID, actual[idx].UUID, user.Name) suite.Equal(user.Name, actual[idx].Name, user.Name) @@ -96,12 +96,13 @@ func (suite *MongoStoreIntegrationTestSuite) assertUsersEqual(expected []QUser, suite.Equal(user.CreatedOn, actual[idx].CreatedOn, user.Name) suite.Equal(user.ModifiedOn, actual[idx].ModifiedOn, user.Name) suite.Equal(user.Token, actual[idx].Token, user.Name) - suite.True((actual[idx].ID.(bson.ObjectId)).Valid(), user.Name) + _, isObjectId := actual[idx].ID.(primitive.ObjectID) + suite.True(isObjectId, user.Name) } } func (suite *MongoStoreIntegrationTestSuite) assertTopicsEqual(expected []QTopic, actual []QTopic) { - suite.True(len(actual) == len(expected)) + suite.Equal(len(expected), len(actual)) for idx, topic := range expected { suite.Equal(topic.Name, actual[idx].Name, topic.Name) suite.Equal(topic.ProjectUUID, actual[idx].ProjectUUID, topic.Name) @@ -112,12 +113,13 @@ func (suite *MongoStoreIntegrationTestSuite) assertTopicsEqual(expected []QTopic suite.Equal(topic.MsgNum, actual[idx].MsgNum, topic.Name) suite.Equal(topic.TotalBytes, actual[idx].TotalBytes, topic.Name) suite.Equal(topic.ACL, actual[idx].ACL, topic.Name) - suite.True((actual[idx].ID.(bson.ObjectId)).Valid(), topic.Name) + _, isObjectId := actual[idx].ID.(primitive.ObjectID) + suite.True(isObjectId, topic.Name) } } func (suite *MongoStoreIntegrationTestSuite) assertSubsEqual(expected []QSub, actual []QSub) { - suite.True(len(actual) == len(expected)) + suite.Equal(len(expected), len(actual)) for idx, sub := range expected { suite.Equal(sub.Name, actual[idx].Name, sub.Name) suite.Equal(sub.ProjectUUID, actual[idx].ProjectUUID, sub.Name) @@ -130,7 +132,8 @@ func (suite *MongoStoreIntegrationTestSuite) assertSubsEqual(expected []QSub, ac suite.Equal(sub.MsgNum, actual[idx].MsgNum, sub.Name) suite.Equal(sub.TotalBytes, actual[idx].TotalBytes, sub.Name) suite.Equal(sub.ACL, actual[idx].ACL, sub.Name) - suite.True((actual[idx].ID.(bson.ObjectId)).Valid(), sub.Name) + _, isObjectId := actual[idx].ID.(primitive.ObjectID) + suite.True(isObjectId, sub.Name) suite.Equal(sub.PushType, actual[idx].PushType, sub.Name) suite.Equal(sub.PushEndpoint, actual[idx].PushEndpoint, sub.Name) suite.Equal(sub.AuthorizationType, actual[idx].AuthorizationType, sub.Name) @@ -146,8 +149,8 @@ func (suite *MongoStoreIntegrationTestSuite) assertSubsEqual(expected []QSub, ac func (suite *MongoStoreIntegrationTestSuite) initDB() { - created := time.Date(2009, time.November, 10, 23, 0, 0, 0, time.Local) - modified := time.Date(2009, time.November, 10, 23, 0, 0, 0, time.Local) + created := time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC) + modified := time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC) // populate Projects qPr := QProject{ @@ -201,10 +204,10 @@ func (suite *MongoStoreIntegrationTestSuite) initDB() { Name: "topic4", MsgNum: 0, TotalBytes: 0, - LatestPublish: time.Date(1970, time.January, 1, 2, 0, 0, 0, time.Local), + LatestPublish: time.Date(1970, time.January, 1, 2, 0, 0, 0, time.UTC), PublishRate: 0, SchemaUUID: "", - CreatedOn: time.Date(2020, 11, 19, 0, 0, 0, 0, time.Local), + CreatedOn: time.Date(2020, 11, 19, 0, 0, 0, 0, time.UTC), ACL: []string{}, } qtop3 := QTopic{ @@ -212,10 +215,10 @@ func (suite *MongoStoreIntegrationTestSuite) initDB() { Name: "topic3", MsgNum: 0, TotalBytes: 0, - LatestPublish: time.Date(2019, 5, 7, 0, 0, 0, 0, time.Local), + LatestPublish: time.Date(2019, 5, 7, 0, 0, 0, 0, time.UTC), PublishRate: 8.99, SchemaUUID: "schema_uuid_3", - CreatedOn: time.Date(2020, 11, 20, 0, 0, 0, 0, time.Local), + CreatedOn: time.Date(2020, 11, 20, 0, 0, 0, 0, time.UTC), ACL: qTopicACL03.ACL, } qtop2 := QTopic{ @@ -223,10 +226,10 @@ func (suite *MongoStoreIntegrationTestSuite) initDB() { Name: "topic2", MsgNum: 0, TotalBytes: 0, - LatestPublish: time.Date(2019, 5, 8, 0, 0, 0, 0, time.Local), + LatestPublish: time.Date(2019, 5, 8, 0, 0, 0, 0, time.UTC), PublishRate: 5.45, SchemaUUID: "schema_uuid_1", - CreatedOn: time.Date(2020, 11, 21, 0, 0, 0, 0, time.Local), + CreatedOn: time.Date(2020, 11, 21, 0, 0, 0, 0, time.UTC), ACL: qTopicACL02.ACL, } qtop1 := QTopic{ @@ -234,10 +237,10 @@ func (suite *MongoStoreIntegrationTestSuite) initDB() { Name: "topic1", MsgNum: 0, TotalBytes: 0, - LatestPublish: time.Date(2019, 5, 6, 0, 0, 0, 0, time.Local), + LatestPublish: time.Date(2019, 5, 6, 0, 0, 0, 0, time.UTC), PublishRate: 10, SchemaUUID: "", - CreatedOn: time.Date(2020, 11, 22, 0, 0, 0, 0, time.Local), + CreatedOn: time.Date(2020, 11, 22, 0, 0, 0, 0, time.UTC), ACL: qTopicACL01.ACL, } suite.TopicList = append(suite.TopicList, qtop1) @@ -282,9 +285,9 @@ func (suite *MongoStoreIntegrationTestSuite) initDB() { Name: "sub1", Topic: "topic1", Ack: 10, - LatestConsume: time.Date(2019, 5, 6, 0, 0, 0, 0, time.Local), + LatestConsume: time.Date(2019, 5, 6, 0, 0, 0, 0, time.UTC), ConsumeRate: 10, - CreatedOn: time.Date(2020, 11, 19, 0, 0, 0, 0, time.Local), + CreatedOn: time.Date(2020, 11, 19, 0, 0, 0, 0, time.UTC), ACL: qSubACL01.ACL, } @@ -294,9 +297,9 @@ func (suite *MongoStoreIntegrationTestSuite) initDB() { Name: "sub2", Topic: "topic2", Ack: 10, - LatestConsume: time.Date(2019, 5, 7, 0, 0, 0, 0, time.Local), + LatestConsume: time.Date(2019, 5, 7, 0, 0, 0, 0, time.UTC), ConsumeRate: 8.99, - CreatedOn: time.Date(2020, 11, 20, 0, 0, 0, 0, time.Local), + CreatedOn: time.Date(2020, 11, 20, 0, 0, 0, 0, time.UTC), ACL: qSubACL02.ACL, } @@ -306,9 +309,9 @@ func (suite *MongoStoreIntegrationTestSuite) initDB() { Name: "sub3", Topic: "topic3", Ack: 10, - LatestConsume: time.Date(2019, 5, 8, 0, 0, 0, 0, time.Local), + LatestConsume: time.Date(2019, 5, 8, 0, 0, 0, 0, time.UTC), ConsumeRate: 5.45, - CreatedOn: time.Date(2020, 11, 21, 0, 0, 0, 0, time.Local), + CreatedOn: time.Date(2020, 11, 21, 0, 0, 0, 0, time.UTC), ACL: qSubACL03.ACL, } @@ -328,7 +331,7 @@ func (suite *MongoStoreIntegrationTestSuite) initDB() { VerificationHash: "push-id-1", Verified: true, Base64Decode: true, - CreatedOn: time.Date(2020, 11, 22, 0, 0, 0, 0, time.Local), + CreatedOn: time.Date(2020, 11, 22, 0, 0, 0, 0, time.UTC), ACL: qSubACL04.ACL, } @@ -433,25 +436,25 @@ func (suite *MongoStoreIntegrationTestSuite) initDB() { // populate daily msg count for topics dc1 := QDailyTopicMsgCount{ - Date: time.Date(2018, 10, 1, 0, 0, 0, 0, time.Local), + Date: time.Date(2018, 10, 1, 0, 0, 0, 0, time.UTC), ProjectUUID: "argo_uuid", TopicName: "topic1", NumberOfMessages: 40, } dc2 := QDailyTopicMsgCount{ - Date: time.Date(2018, 10, 2, 0, 0, 0, 0, time.Local), + Date: time.Date(2018, 10, 2, 0, 0, 0, 0, time.UTC), ProjectUUID: "argo_uuid", TopicName: "topic1", NumberOfMessages: 30, } dc3 := QDailyTopicMsgCount{ - Date: time.Date(2018, 10, 1, 0, 0, 0, 0, time.Local), + Date: time.Date(2018, 10, 1, 0, 0, 0, 0, time.UTC), ProjectUUID: "argo_uuid", TopicName: "topic2", NumberOfMessages: 70, } dc4 := QDailyTopicMsgCount{ - Date: time.Date(2018, 10, 1, 0, 0, 0, 0, time.Local), + Date: time.Date(2018, 10, 1, 0, 0, 0, 0, time.UTC), ProjectUUID: "argo_uuid", TopicName: "topic3", } @@ -633,6 +636,46 @@ func (suite *MongoStoreIntegrationTestSuite) initDB() { } +func (suite *MongoStoreIntegrationTestSuite) TestQuerySubsByACL() { + eSubList1 := []QSub{suite.SubList[0], suite.SubList[1], suite.SubList[2]} + subList, _ := suite.store.QuerySubsByACL(suite.ctx, "argo_uuid", "uuid1") + suite.assertSubsEqual(eSubList1, subList) +} + +func (suite *MongoStoreIntegrationTestSuite) TestIncrementSubBytes() { + _ = suite.store.IncrementSubBytes(suite.ctx, "argo_uuid", "sub1", 50) + sub, _ := suite.store.QueryOneSub(suite.ctx, "argo_uuid", "sub1") + suite.Equal(int64(50), sub.TotalBytes) + _ = suite.store.IncrementSubBytes(suite.ctx, "argo_uuid", "sub1", -50) +} + +func (suite *MongoStoreIntegrationTestSuite) TestIncrementSubMsgNum() { + _ = suite.store.IncrementSubMsgNum(suite.ctx, "argo_uuid", "sub1", 50) + sub, _ := suite.store.QueryOneSub(suite.ctx, "argo_uuid", "sub1") + suite.Equal(int64(50), sub.MsgNum) + _ = suite.store.IncrementSubMsgNum(suite.ctx, "argo_uuid", "sub1", -50) +} + +func (suite *MongoStoreIntegrationTestSuite) TestQueryTopicsByACL() { + eTopList1st1 := []QTopic{suite.TopicList[0], suite.TopicList[1]} + tpList, _ := suite.store.QueryTopicsByACL(suite.ctx, "argo_uuid", "uuid1") + suite.assertTopicsEqual(eTopList1st1, tpList) +} + +func (suite *MongoStoreIntegrationTestSuite) TestIncrementTopicBytes() { + _ = suite.store.IncrementTopicBytes(suite.ctx, "argo_uuid", "topic1", 50) + tpList4, _, _, _ := suite.store.QueryTopics(suite.ctx, "argo_uuid", "", "topic1", "", 0) + suite.Equal(int64(50), tpList4[0].TotalBytes) + _ = suite.store.IncrementTopicBytes(suite.ctx, "argo_uuid", "topic1", -50) +} + +func (suite *MongoStoreIntegrationTestSuite) TestIncrementTopicMsgNum() { + _ = suite.store.IncrementTopicMsgNum(suite.ctx, "argo_uuid", "topic1", 50) + tpList4, _, _, _ := suite.store.QueryTopics(suite.ctx, "argo_uuid", "", "topic1", "", 0) + suite.Equal(int64(50), tpList4[0].MsgNum) + _ = suite.store.IncrementTopicMsgNum(suite.ctx, "argo_uuid", "topic1", -50) +} + func (suite *MongoStoreIntegrationTestSuite) TestQueryTopics() { qTopicListReverser := QListReverser[QTopic]{} @@ -648,14 +691,14 @@ func (suite *MongoStoreIntegrationTestSuite) TestQueryTopics() { tpList2, ts2, pg2, _ := suite.store.QueryTopics(suite.ctx, "argo_uuid", "", "", "", 2) suite.assertTopicsEqual(eTopList1st2, tpList2) suite.Equal(int64(4), ts2) - suite.True(bson.IsObjectIdHex(pg2)) + suite.True(primitive.IsValidObjectID(pg2)) // retrieve the next one eTopList3 := []QTopic{suite.TopicList[1]} tpList3, ts3, pg3, _ := suite.store.QueryTopics(suite.ctx, "argo_uuid", "", "", pg2, 1) suite.assertTopicsEqual(eTopList3, tpList3) suite.Equal(int64(4), ts3) - suite.True(bson.IsObjectIdHex(pg3)) + suite.True(primitive.IsValidObjectID(pg3)) // retrieve a single topic eTopList4 := []QTopic{suite.TopicList[0]} @@ -676,7 +719,7 @@ func (suite *MongoStoreIntegrationTestSuite) TestQueryTopics() { tpList6, ts6, pg6, _ := suite.store.QueryTopics(suite.ctx, "argo_uuid", "uuid1", "", "", 1) suite.assertTopicsEqual(eTopList6, tpList6) suite.Equal(int64(2), ts6) - suite.True(bson.IsObjectIdHex(pg6)) + suite.True(primitive.IsValidObjectID(pg6)) } func (suite *MongoStoreIntegrationTestSuite) TestQuerySubs() { @@ -696,7 +739,7 @@ func (suite *MongoStoreIntegrationTestSuite) TestQuerySubs() { subList2, ts2, pg2, err2 := suite.store.QuerySubs(suite.ctx, "argo_uuid", "", "", "", 2) suite.assertSubsEqual(eSubListFirstPage, subList2) suite.Equal(int64(4), ts2) - suite.True(bson.IsObjectIdHex(pg2)) + suite.True(primitive.IsValidObjectID(pg2)) // retrieve next 2 subs eSubListNextPage := []QSub{ @@ -718,9 +761,9 @@ func (suite *MongoStoreIntegrationTestSuite) TestQuerySubs() { subList4, ts4, pg4, err4 := suite.store.QuerySubs(suite.ctx, "argo_uuid", "uuid1", "", "", 0) + suite.assertSubsEqual(eSubList4, subList4) suite.Equal(int64(3), ts4) suite.Equal("", pg4) - suite.assertSubsEqual(eSubList4, subList4) // retrieve user's subs eSubList5 := []QSub{ @@ -730,7 +773,7 @@ func (suite *MongoStoreIntegrationTestSuite) TestQuerySubs() { subList5, ts5, pg5, err5 := suite.store.QuerySubs(suite.ctx, "argo_uuid", "uuid1", "", "", 2) suite.Equal(int64(3), ts5) - suite.True(bson.IsObjectIdHex(pg5)) + suite.True(primitive.IsValidObjectID(pg5)) suite.assertSubsEqual(eSubList5, subList5) suite.Nil(err1) @@ -756,13 +799,17 @@ func (suite *MongoStoreIntegrationTestSuite) TestDailyTopicMsgCount() { suite.Equal(suite.DailyTopicMsgCount, qdsAll) // test daily count - _ = suite.store.IncrementDailyTopicMsgCount(suite.ctx, "argo_uuid", "topic1", 40, time.Date(2018, 10, 1, 0, 0, 0, 0, time.Local)) - qds, _ := suite.store.QueryDailyTopicMsgCount(suite.ctx, "argo_uuid", "topic1", time.Date(2018, 10, 1, 0, 0, 0, 0, time.Local)) - suite.Equal(int64(80), qds[0].NumberOfMessages) + _ = suite.store.IncrementDailyTopicMsgCount(suite.ctx, "argo_uuid_2", "topic1", + 40, time.Date(2018, 10, 1, 0, 0, 0, 0, time.UTC)) + qds, _ := suite.store.QueryDailyTopicMsgCount(suite.ctx, "argo_uuid_2", "topic1", + time.Date(2018, 10, 1, 0, 0, 0, 0, time.UTC)) + suite.Equal(int64(40), qds[0].NumberOfMessages) // check if it was inserted since it wasn't present - _ = suite.store.IncrementDailyTopicMsgCount(suite.ctx, "argo_uuid", "some_other_topic", 70, time.Date(2018, 10, 1, 0, 0, 0, 0, time.Local)) - qds2, _ := suite.store.QueryDailyTopicMsgCount(suite.ctx, "argo_uuid", "some_other_topic", time.Date(2018, 10, 1, 0, 0, 0, 0, time.Local)) + _ = suite.store.IncrementDailyTopicMsgCount(suite.ctx, "argo_uuid_2", "some_other_topic", + 70, time.Date(2018, 10, 1, 0, 0, 0, 0, time.UTC)) + qds2, _ := suite.store.QueryDailyTopicMsgCount(suite.ctx, "argo_uuid_2", + "some_other_topic", time.Date(2018, 10, 1, 0, 0, 0, 0, time.UTC)) suite.Equal(int64(70), qds2[0].NumberOfMessages) } @@ -778,6 +825,11 @@ func (suite *MongoStoreIntegrationTestSuite) TestHasResourceRoles() { } +func (suite *MongoStoreIntegrationTestSuite) TestGetAllRoles() { + roles := suite.store.GetAllRoles(suite.ctx) + suite.Equal([]string{"admin", "publisher", "reader"}, roles) +} + func (suite *MongoStoreIntegrationTestSuite) TestHasProject() { suite.True(suite.store.HasProject(suite.ctx, "ARGO")) suite.False(suite.store.HasProject(suite.ctx, "FOO")) @@ -791,7 +843,7 @@ func (suite *MongoStoreIntegrationTestSuite) TestGetUserRoles() { } func (suite *MongoStoreIntegrationTestSuite) TestRemoveSub() { - _ = suite.store.InsertSub(suite.ctx, "argo_uuid", "subFresh", "topicFresh", 0, 10, QPushConfig{}, time.Date(2020, 12, 19, 0, 0, 0, 0, time.Local)) + _ = suite.store.InsertSub(suite.ctx, "argo_uuid", "subFresh", "topicFresh", 0, 10, QPushConfig{}, time.Date(2020, 12, 19, 0, 0, 0, 0, time.UTC)) err := suite.store.RemoveSub(suite.ctx, "argo_uuid", "subFresh") suite.Equal(nil, err) subList, _, _, _ := suite.store.QuerySubs(suite.ctx, "argo_uuid", "", "", "", 0) @@ -801,7 +853,7 @@ func (suite *MongoStoreIntegrationTestSuite) TestRemoveSub() { } func (suite *MongoStoreIntegrationTestSuite) TestRemoveTopic() { - _ = suite.store.InsertTopic(suite.ctx, "argo_uuid", "topicFresh", "", time.Date(2020, 9, 11, 0, 0, 0, 0, time.Local)) + _ = suite.store.InsertTopic(suite.ctx, "argo_uuid", "topicFresh", "", time.Date(2020, 9, 11, 0, 0, 0, 0, time.UTC)) err := suite.store.RemoveTopic(suite.ctx, "argo_uuid", "topicFresh") suite.Equal(nil, err) qTopicListReverser := QListReverser[QTopic]{} @@ -817,8 +869,70 @@ func (suite *MongoStoreIntegrationTestSuite) TestModAck() { suite.Equal(66, subAck.Ack) } +func (suite *MongoStoreIntegrationTestSuite) TestUpdateSubOffset() { + _ = suite.store.InsertSub(suite.ctx, "argo_uuid", "subFresh", "topicFresh", 0, 1000, + QPushConfig{}, time.Date(2020, 12, 19, 0, 0, 0, 0, time.UTC)) + zSec := "2006-01-02T15:04:05Z" + t := time.Now().UTC() + ts := t.Format(zSec) + _ = suite.store.UpdateSubPull(suite.ctx, "argo_uuid", "subFresh", 99, ts) + sub, _ := suite.store.QueryOneSub(suite.ctx, "argo_uuid", "subFresh") + suite.Equal(int64(99), sub.NextOffset) + suite.Equal(ts, sub.PendingAck) + suite.Equal(int64(0), sub.Offset) + + suite.store.UpdateSubOffset(suite.ctx, "argo_uuid", "subFresh", 99) + sub, _ = suite.store.QueryOneSub(suite.ctx, "argo_uuid", "subFresh") + suite.Equal(int64(0), sub.NextOffset) + suite.Equal("", sub.PendingAck) + suite.Equal(int64(99), sub.Offset) + + // clear state + _ = suite.store.RemoveSub(suite.ctx, "argo_uuid", "subFresh") +} + +func (suite *MongoStoreIntegrationTestSuite) TestUpdateSubOffsetAck() { + _ = suite.store.InsertSub(suite.ctx, "argo_uuid", "subFresh", "topicFresh", 0, 1000, + QPushConfig{}, time.Date(2020, 12, 19, 0, 0, 0, 0, time.UTC)) + zSec := "2006-01-02T15:04:05Z" + + err := suite.store.UpdateSubOffsetAck(suite.ctx, "argo_uuid", "subFresh", 99, "") + suite.Equal("no ack pending", err.Error()) + + t := time.Now().UTC() + + // time stamp which is ahead of t combined with the appropriate ack deadline + tErr := t.Add(1200 * 1000 * 1000 * 1000).Format(zSec) + ts := t.Format(zSec) + tComplete := t.Add(1000).Format(zSec) + _ = suite.store.UpdateSubPull(suite.ctx, "argo_uuid", "subFresh", 99, ts) + sub, _ := suite.store.QueryOneSub(suite.ctx, "argo_uuid", "subFresh") + suite.Equal(int64(99), sub.NextOffset) + suite.Equal(ts, sub.PendingAck) + suite.Equal(int64(0), sub.Offset) + + err2 := suite.store.UpdateSubOffsetAck(suite.ctx, "argo_uuid", "subFresh", -1, "") + suite.Equal("wrong ack", err2.Error()) + + err3 := suite.store.UpdateSubOffsetAck(suite.ctx, "argo_uuid", "subFresh", 100, "") + suite.Equal("wrong ack", err3.Error()) + + err4 := suite.store.UpdateSubOffsetAck(suite.ctx, "argo_uuid", "subFresh", 88, tErr) + suite.Equal("ack timeout", err4.Error()) + + err5 := suite.store.UpdateSubOffsetAck(suite.ctx, "argo_uuid", "subFresh", 88, tComplete) + sub2, _ := suite.store.QueryOneSub(suite.ctx, "argo_uuid", "subFresh") + suite.Nil(err5) + suite.Equal(int64(88), sub2.Offset) + suite.Equal("", sub2.PendingAck) + suite.Equal(int64(99), sub.NextOffset) + + // clear state + _ = suite.store.RemoveSub(suite.ctx, "argo_uuid", "subFresh") +} + func (suite *MongoStoreIntegrationTestSuite) TestModPushSub() { - _ = suite.store.InsertSub(suite.ctx, "argo_uuid", "subFresh", "topicFresh", 0, 10, QPushConfig{}, time.Date(2020, 12, 19, 0, 0, 0, 0, time.Local)) + _ = suite.store.InsertSub(suite.ctx, "argo_uuid", "subFresh", "topicFresh", 0, 10, QPushConfig{}, time.Date(2020, 12, 19, 0, 0, 0, 0, time.UTC)) qCfg := QPushConfig{ Type: "http_endpoint", PushEndpoint: "example.com", @@ -899,12 +1013,12 @@ func (suite *MongoStoreIntegrationTestSuite) TestQueryACL() { QAcl09, err09 := suite.store.QueryACL(suite.ctx, "argo_uuid", "subscriptions", "sub4ss") suite.Equal(QAcl{}, QAcl09) - suite.Equal(errors.New("not found"), err09) + suite.Equal(DocNotFound{}, err09) } func (suite *MongoStoreIntegrationTestSuite) TestUpdateUser() { - created := time.Date(2009, time.November, 10, 23, 0, 0, 0, time.Local) - modified := time.Date(2009, time.November, 10, 23, 0, 0, 0, time.Local) + created := time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC) + modified := time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC) qRoles := []QProjectRoles{QProjectRoles{"argo_uuid", []string{"admin"}}, QProjectRoles{"argo_uuid2", []string{"admin", "viewer"}}} _ = suite.store.InsertUser(suite.ctx, "user_uuid11", qRoles, "newUser2", "", "", "", "", "BX312Z34NLQ", "fake@email.com", []string{}, created, modified, "uuid1") usrUpdated := QUser{UUID: "user_uuid11", Projects: qRoles, Name: "updated_name", Token: "BX312Z34NLQ", Email: "fake@email.com", ServiceRoles: []string{"service_admin"}, CreatedOn: created, ModifiedOn: modified, CreatedBy: "uuid1"} @@ -941,6 +1055,15 @@ func (suite *MongoStoreIntegrationTestSuite) TestGetUserFromToken() { suite.assertUsersEqual([]QUser{suite.UserList[0]}, []QUser{usrGet}) } +func (suite *MongoStoreIntegrationTestSuite) TestUpdateUserFromToken() { + _ = suite.store.UpdateUserToken(suite.ctx, suite.UserList[0].UUID, "S3CR3T-v2") + _, e1 := suite.store.GetUserFromToken(suite.ctx, "S3CR3T") + suite.Equal("not found", e1.Error()) + usrGet, _ := suite.store.GetUserFromToken(suite.ctx, "S3CR3T-v2") + suite.Equal(suite.UserList[0].UUID, usrGet.UUID) + _ = suite.store.UpdateUserToken(suite.ctx, suite.UserList[0].UUID, "S3CR3T") +} + func (suite *MongoStoreIntegrationTestSuite) TestPaginatedQueryUsers() { reverser := QListReverser[QUser]{} @@ -962,7 +1085,7 @@ func (suite *MongoStoreIntegrationTestSuite) TestPaginatedQueryUsers() { suite.Equal(int64(9), ts1) suite.assertUsersEqual([]QUser{suite.UserList[8], suite.UserList[7]}, qUsers2) - suite.True(bson.IsObjectIdHex(pg2)) + suite.True(primitive.IsValidObjectID(pg2)) suite.Equal(int64(9), ts2) suite.Equal(0, len(qUsers3)) @@ -970,14 +1093,14 @@ func (suite *MongoStoreIntegrationTestSuite) TestPaginatedQueryUsers() { suite.Equal(int64(0), ts3) suite.assertUsersEqual([]QUser{suite.UserList[6], suite.UserList[5]}, qUsers4) - suite.True(bson.IsObjectIdHex(pg4)) + suite.True(primitive.IsValidObjectID(pg4)) suite.Equal(int64(9), ts4) } func (suite *MongoStoreIntegrationTestSuite) TestACLModificationActions() { - _ = suite.store.InsertSub(suite.ctx, "argo_uuid", "subFresh", "topicFresh", 0, 10, QPushConfig{}, time.Date(2020, 12, 19, 0, 0, 0, 0, time.Local)) - _ = suite.store.InsertTopic(suite.ctx, "argo_uuid", "topicFresh", "", time.Date(2020, 9, 11, 0, 0, 0, 0, time.Local)) + _ = suite.store.InsertSub(suite.ctx, "argo_uuid", "subFresh", "topicFresh", 0, 10, QPushConfig{}, time.Date(2020, 12, 19, 0, 0, 0, 0, time.UTC)) + _ = suite.store.InsertTopic(suite.ctx, "argo_uuid", "topicFresh", "", time.Date(2020, 9, 11, 0, 0, 0, 0, time.UTC)) // test mod acl ExpectedACL01 := QAcl{[]string{"uuid1", "uuid2"}} @@ -1068,10 +1191,10 @@ func (suite *MongoStoreIntegrationTestSuite) TestCRUDProjects() { suite.Equal(expProj4, projectOut8) suite.Equal(errors.New("not found"), err) - created := time.Date(2009, time.November, 10, 23, 0, 0, 0, time.Local) - modified := time.Date(2009, time.November, 10, 23, 0, 0, 0, time.Local) + created := time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC) + modified := time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC) _ = suite.store.InsertProject(suite.ctx, "argo_uuid3", "ARGO3", created, modified, "uuid1", "simple project") - modified = time.Date(2010, time.November, 10, 23, 0, 0, 0, time.Local) + modified = time.Date(2010, time.November, 10, 23, 0, 0, 0, time.UTC) expPr1 := QProject{UUID: "argo_uuid3", Name: "ARGO3", CreatedOn: created, ModifiedOn: modified, CreatedBy: "uuid1", Description: "a modified description"} _ = suite.store.UpdateProject(suite.ctx, "argo_uuid3", "", "a modified description", modified) prUp1, _ := suite.store.QueryProjects(suite.ctx, "argo_uuid3", "") @@ -1084,17 +1207,36 @@ func (suite *MongoStoreIntegrationTestSuite) TestCRUDProjects() { _ = suite.store.UpdateProject(suite.ctx, "argo_uuid3", "ARGO_3", "a newly modified description", modified) prUp3, _ := suite.store.QueryProjects(suite.ctx, "argo_uuid3", "") suite.Equal(expPr3, prUp3[0]) - // Test RemoveProject + + // Test RemoveProject and all its related resources + t := time.Now() + _ = suite.store.InsertSub(suite.ctx, "argo_uuid3", "subDel", "topicFresh", 0, 10, QPushConfig{}, time.Date(2020, 12, 19, 0, 0, 0, 0, time.UTC)) + _ = suite.store.InsertTopic(suite.ctx, "argo_uuid3", "topicDel", "", time.Now().UTC()) + _ = suite.store.IncrementDailyTopicMsgCount(suite.ctx, "argo_uuid3", "topicDel", 100, t) + r, _ := suite.store.QueryDailyProjectMsgCount(suite.ctx, "argo_uuid3") + suite.Equal( + 1, + len(r), + ) + suite.Equal(t.Day(), r[0].Date.Day()) _ = suite.store.RemoveProject(suite.ctx, "argo_uuid3") + _ = suite.store.RemoveProjectTopics(suite.ctx, "argo_uuid3") + _ = suite.store.RemoveProjectSubs(suite.ctx, "argo_uuid3") + _ = suite.store.RemoveProjectDailyMessageCounters(suite.ctx, "argo_uuid3") _, err = suite.store.QueryProjects(suite.ctx, "argo_uuid3", "") + r2, _ := suite.store.QueryDailyProjectMsgCount(suite.ctx, "argo_uuid3") + suite.Empty(r2) suite.Equal(errors.New("not found"), err) } func (suite *MongoStoreIntegrationTestSuite) TestQueryTotalMessagesPerProject() { expectedQpmc := []QProjectMessageCount{ - {ProjectUUID: "argo_uuid", NumberOfMessages: 30, AverageDailyMessages: 7.5}, + {ProjectUUID: "argo_uuid", NumberOfMessages: 140, AverageDailyMessages: 35}, } - qpmc, qpmcerr1 := suite.store.QueryTotalMessagesPerProject(suite.ctx, []string{"argo_uuid"}, time.Date(2018, 10, 1, 0, 0, 0, 0, time.UTC), time.Date(2018, 10, 4, 0, 0, 0, 0, time.UTC)) + qpmc, qpmcerr1 := suite.store.QueryTotalMessagesPerProject(suite.ctx, []string{"argo_uuid"}, + time.Date(2018, 10, 1, 0, 0, 0, 0, time.UTC), + time.Date(2018, 10, 4, 0, 0, 0, 0, time.UTC), + ) suite.Equal(expectedQpmc, qpmc) suite.Nil(qpmcerr1) } @@ -1127,7 +1269,7 @@ func (suite *MongoStoreIntegrationTestSuite) TestCRUDSchemas() { suite.Nil(eis) // test update schema - _ = suite.store.InsertTopic(suite.ctx, "argo_uuid", "topicFresh", "", time.Date(2020, 9, 11, 0, 0, 0, 0, time.Local)) + _ = suite.store.InsertTopic(suite.ctx, "argo_uuid", "topicFresh", "", time.Date(2020, 9, 11, 0, 0, 0, 0, time.UTC)) _ = suite.store.UpdateSchema(suite.ctx, "uuid1", "new-name", "new-type", "new-raw-schema") eus := QSchema{UUID: "uuid1", ProjectUUID: "argo_uuid", Type: "new-type", Name: "new-name", RawSchema: "new-raw-schema"} qus, _ := suite.store.QuerySchemas(suite.ctx, "argo_uuid", "uuid1", "") @@ -1210,6 +1352,24 @@ func (suite *MongoStoreIntegrationTestSuite) TestResourcesCounters() { suite.Equal(map[string]int64{"argo_uuid": 7, "argo_uuid2": 1}, uc) } +func (suite *MongoStoreIntegrationTestSuite) TestOpMetrics() { + _ = suite.store.InsertOpMetric(suite.ctx, "host1", 1.1, 0.8) + _ = suite.store.InsertOpMetric(suite.ctx, "host2", 1.2, 1.3) + expectedOpMetrics := []QopMetric{ + { + Hostname: "host1", + CPU: 1.1, + MEM: 0.8, + }, + { + Hostname: "host2", + CPU: 1.2, + MEM: 1.3, + }, + } + suite.Equal(expectedOpMetrics, suite.store.GetOpMetrics(suite.ctx)) +} + func (suite *MongoStoreIntegrationTestSuite) SetupSuite() { suite.ctx = context.Background() suite.store.Initialize() @@ -1229,12 +1389,13 @@ func TestMongoStoreIntegrationTestSuite(t *testing.T) { p, _ := container.MappedPort(context.Background(), "27017/tcp") - mongoDBUri := fmt.Sprintf("mongodb://localhost:%s", p.Port()) + mongoDBUri := fmt.Sprintf("localhost:%s", p.Port()) + mongoDatabase := "argo_ams" - mongoStore := &MongoStore{ - Server: mongoDBUri, - Database: "argo_ams", - } + mongoStore := NewMongoStoreWithOfficialDriver( + mongoDBUri, + mongoDatabase, + ) suite.Run(t, &MongoStoreIntegrationTestSuite{ store: mongoStore, ctx: context.Background(), diff --git a/stores/store.go b/stores/store.go index 5d115d4b..00465cfc 100644 --- a/stores/store.go +++ b/stores/store.go @@ -8,24 +8,60 @@ import ( // Store encapsulates the generic store interface type Store interface { Initialize() - QuerySubsByTopic(ctx context.Context, projectUUID, topic string) ([]QSub, error) + Clone() Store + Close() + + // ###### TOPIC QUERIES ###### + + LinkTopicSchema(ctx context.Context, projectUUID, name, schemaUUID string) error QueryTopicsByACL(ctx context.Context, projectUUID, user string) ([]QTopic, error) - QuerySubsByACL(ctx context.Context, projectUUID, user string) ([]QSub, error) - QuerySubs(ctx context.Context, projectUUID string, userUUID string, name string, pageToken string, pageSize int64) ([]QSub, int64, string, error) QueryTopics(ctx context.Context, projectUUID string, userUUID string, name string, pageToken string, pageSize int64) ([]QTopic, int64, string, error) - QueryDailyTopicMsgCount(ctx context.Context, projectUUID string, name string, date time.Time) ([]QDailyTopicMsgCount, error) UpdateTopicLatestPublish(ctx context.Context, projectUUID string, name string, date time.Time) error UpdateTopicPublishRate(ctx context.Context, projectUUID string, name string, rate float64) error + RemoveTopic(ctx context.Context, projectUUID string, name string) error + InsertTopic(ctx context.Context, projectUUID string, name string, schemaUUID string, createdOn time.Time) error + TopicsCount(ctx context.Context, startDate, endDate time.Time, projectUUIDs []string) (map[string]int64, error) + QueryDailyTopicMsgCount(ctx context.Context, projectUUID string, name string, date time.Time) ([]QDailyTopicMsgCount, error) + IncrementTopicMsgNum(ctx context.Context, projectUUID string, name string, num int64) error + IncrementDailyTopicMsgCount(ctx context.Context, projectUUID string, topicName string, num int64, date time.Time) error + IncrementTopicBytes(ctx context.Context, projectUUID string, name string, totalBytes int64) error + + // ###### SUBSCRIPTION QUERIES ###### + + QuerySubsByTopic(ctx context.Context, projectUUID, topic string) ([]QSub, error) + QuerySubsByACL(ctx context.Context, projectUUID, user string) ([]QSub, error) + QuerySubs(ctx context.Context, projectUUID string, userUUID string, name string, pageToken string, pageSize int64) ([]QSub, int64, string, error) UpdateSubLatestConsume(ctx context.Context, projectUUID string, name string, date time.Time) error UpdateSubConsumeRate(ctx context.Context, projectUUID string, name string, rate float64) error - RemoveTopic(ctx context.Context, projectUUID string, name string) error RemoveSub(ctx context.Context, projectUUID string, name string) error + IncrementSubBytes(ctx context.Context, projectUUID string, name string, totalBytes int64) error + IncrementSubMsgNum(ctx context.Context, projectUUID string, name string, num int64) error + InsertSub(ctx context.Context, projectUUID string, name string, topic string, offest int64, ack int, pushCfg QPushConfig, createdOn time.Time) error + QueryOneSub(ctx context.Context, projectUUID string, name string) (QSub, error) + QueryPushSubs(ctx context.Context) []QSub + SubscriptionsCount(ctx context.Context, startDate, endDate time.Time, projectUUIDs []string) (map[string]int64, error) + ModAck(ctx context.Context, projectUUID string, name string, ack int) error + UpdateSubOffset(ctx context.Context, projectUUID string, name string, offset int64) + UpdateSubPull(ctx context.Context, projectUUID string, name string, offset int64, ts string) error + UpdateSubOffsetAck(ctx context.Context, projectUUID string, name string, offset int64, ts string) error + ModSubPush(ctx context.Context, projectUUID string, name string, pushCfg QPushConfig) error + + // ###### USER QUERIES ###### + + HasUsers(ctx context.Context, projectUUID string, users []string) (bool, []string) PaginatedQueryUsers(ctx context.Context, pageToken string, pageSize int64, projectUUID string) ([]QUser, int64, string, error) QueryUsers(ctx context.Context, projectUUID string, uuid string, name string) ([]QUser, error) UpdateUser(ctx context.Context, uuid, fname, lname, org, desc string, projects []QProjectRoles, name string, email string, serviceRoles []string, modifiedOn time.Time) error AppendToUserProjects(ctx context.Context, userUUID string, projectUUID string, pRoles ...string) error UpdateUserToken(ctx context.Context, uuid string, token string) error RemoveUser(ctx context.Context, uuid string) error + InsertUser(ctx context.Context, uuid string, projects []QProjectRoles, name string, firstName string, lastName string, org string, desc string, token string, email string, serviceRoles []string, createdOn time.Time, modifiedOn time.Time, createdBy string) error + GetUserFromToken(ctx context.Context, token string) (QUser, error) + UsersCount(ctx context.Context, startDate, endDate time.Time, projectUUIDs []string) (map[string]int64, error) + GetUserRoles(ctx context.Context, projectUUID string, token string) ([]string, string) + + // ##### PROJECT QUERIES ##### + QueryProjects(ctx context.Context, uuid string, name string) ([]QProject, error) UpdateProject(ctx context.Context, projectUUID string, name string, description string, modifiedOn time.Time) error RemoveProject(ctx context.Context, uuid string) error @@ -34,48 +70,36 @@ type Store interface { RemoveProjectDailyMessageCounters(ctx context.Context, projectUUID string) error QueryDailyProjectMsgCount(ctx context.Context, projectUUID string) ([]QDailyProjectMsgCount, error) QueryTotalMessagesPerProject(ctx context.Context, projectUUIDs []string, startDate time.Time, endDate time.Time) ([]QProjectMessageCount, error) + InsertProject(ctx context.Context, uuid string, name string, createdOn time.Time, modifiedOn time.Time, createdBy string, description string) error + HasProject(ctx context.Context, name string) bool + + // ##### USER REGISTRATIONS QUERIES ##### + RegisterUser(ctx context.Context, uuid, name, firstName, lastName, email, org, desc, registeredAt, atkn, status string) error DeleteRegistration(ctx context.Context, uuid string) error QueryRegistrations(ctx context.Context, regUUID, status, activationToken, name, email, org string) ([]QUserRegistration, error) UpdateRegistration(ctx context.Context, regUUID, status, declineComment, modifiedBy, modifiedAt string) error - InsertUser(ctx context.Context, uuid string, projects []QProjectRoles, name string, firstName string, lastName string, org string, desc string, token string, email string, serviceRoles []string, createdOn time.Time, modifiedOn time.Time, createdBy string) error - InsertProject(ctx context.Context, uuid string, name string, createdOn time.Time, modifiedOn time.Time, createdBy string, description string) error - InsertOpMetric(ctx context.Context, hostname string, cpu float64, mem float64) error - InsertTopic(ctx context.Context, projectUUID string, name string, schemaUUID string, createdOn time.Time) error - LinkTopicSchema(ctx context.Context, projectUUID, name, schemaUUID string) error - IncrementTopicMsgNum(ctx context.Context, projectUUID string, name string, num int64) error - IncrementDailyTopicMsgCount(ctx context.Context, projectUUID string, topicName string, num int64, date time.Time) error - IncrementTopicBytes(ctx context.Context, projectUUID string, name string, totalBytes int64) error - IncrementSubBytes(ctx context.Context, projectUUID string, name string, totalBytes int64) error - IncrementSubMsgNum(ctx context.Context, projectUUID string, name string, num int64) error - InsertSub(ctx context.Context, projectUUID string, name string, topic string, offest int64, ack int, pushCfg QPushConfig, createdOn time.Time) error - HasProject(ctx context.Context, name string) bool - HasUsers(ctx context.Context, projectUUID string, users []string) (bool, []string) - QueryOneSub(ctx context.Context, projectUUID string, name string) (QSub, error) - QueryPushSubs(ctx context.Context) []QSub - HasResourceRoles(ctx context.Context, resource string, roles []string) bool - InsertResourceRoles(ctx context.Context, resource string, roles []string) error - GetOpMetrics(ctx context.Context) []QopMetric - GetUserRoles(ctx context.Context, projectUUID string, token string) ([]string, string) - GetUserFromToken(ctx context.Context, token string) (QUser, error) - UpdateSubOffset(ctx context.Context, projectUUID string, name string, offset int64) - UpdateSubPull(ctx context.Context, projectUUID string, name string, offset int64, ts string) error - UpdateSubOffsetAck(ctx context.Context, projectUUID string, name string, offset int64, ts string) error - ModSubPush(ctx context.Context, projectUUID string, name string, pushCfg QPushConfig) error + + // ##### SCHEMA QUERIES ##### + + InsertSchema(ctx context.Context, projectUUID, schemaUUID, name, schemaType, rawSchemaString string) error + QuerySchemas(ctx context.Context, projectUUID, schemaUUID, name string) ([]QSchema, error) + UpdateSchema(ctx context.Context, schemaUUID, name, schemaType, rawSchemaString string) error + DeleteSchema(ctx context.Context, schemaUUID string) error + + // ##### ACL QUERIES ###### QueryACL(ctx context.Context, projectUUID string, resource string, name string) (QAcl, error) ExistsInACL(ctx context.Context, projectUUID string, resource string, resourceName string, userUUID string) error ModACL(ctx context.Context, projectUUID string, resource string, name string, acl []string) error AppendToACL(ctx context.Context, projectUUID string, resource string, name string, acl []string) error RemoveFromACL(ctx context.Context, projectUUID string, resource string, name string, acl []string) error - ModAck(ctx context.Context, projectUUID string, name string, ack int) error + + // ##### ROLES QUERIES ##### + HasResourceRoles(ctx context.Context, resource string, roles []string) bool + InsertResourceRoles(ctx context.Context, resource string, roles []string) error GetAllRoles(ctx context.Context) []string - InsertSchema(ctx context.Context, projectUUID, schemaUUID, name, schemaType, rawSchemaString string) error - QuerySchemas(ctx context.Context, projectUUID, schemaUUID, name string) ([]QSchema, error) - UpdateSchema(ctx context.Context, schemaUUID, name, schemaType, rawSchemaString string) error - DeleteSchema(ctx context.Context, schemaUUID string) error - UsersCount(ctx context.Context, startDate, endDate time.Time, projectUUIDs []string) (map[string]int64, error) - TopicsCount(ctx context.Context, startDate, endDate time.Time, projectUUIDs []string) (map[string]int64, error) - SubscriptionsCount(ctx context.Context, startDate, endDate time.Time, projectUUIDs []string) (map[string]int64, error) - Clone() Store - Close() + + // ##### OP METRICS QUERIES ##### + InsertOpMetric(ctx context.Context, hostname string, cpu float64, mem float64) error + GetOpMetrics(ctx context.Context) []QopMetric } From 24f39c7b213c8a7d371b4bafe8a325239f57202f Mon Sep 17 00:00:00 2001 From: agelostsal Date: Mon, 30 Oct 2023 14:04:14 +0200 Subject: [PATCH 5/9] Bumping gorilla handlers and context to latest versions --- go.mod | 5 +++-- go.sum | 7 +++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 2fcf344f..8c695a74 100644 --- a/go.mod +++ b/go.mod @@ -5,8 +5,8 @@ go 1.19 require ( github.com/Shopify/sarama v1.22.1 github.com/golang/protobuf v1.5.3 - github.com/gorilla/context v0.0.0-20150820051245-1c83b3eabd45 - github.com/gorilla/handlers v0.0.0-20160816184729-a5775781a543 + github.com/gorilla/context v1.1.1 + github.com/gorilla/handlers v1.5.1 github.com/gorilla/mux v1.8.0 github.com/linkedin/goavro v2.1.0+incompatible github.com/samuel/go-zookeeper v0.0.0-20160616024954-e64db453f351 @@ -39,6 +39,7 @@ require ( github.com/eapache/go-resiliency v1.1.0 // indirect github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect github.com/eapache/queue v1.1.0 // indirect + github.com/felixge/httpsnoop v1.0.3 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/gogo/protobuf v1.3.2 // indirect diff --git a/go.sum b/go.sum index 8c42242f..16a2ceab 100644 --- a/go.sum +++ b/go.sum @@ -99,6 +99,9 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= +github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= @@ -179,8 +182,12 @@ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5m github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/gorilla/context v0.0.0-20150820051245-1c83b3eabd45 h1:f69IpTg7NxbCoH51W7oD+gPkojLLvi6ybJ6Iya76wB8= github.com/gorilla/context v0.0.0-20150820051245-1c83b3eabd45/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= +github.com/gorilla/context v1.1.1 h1:AWwleXJkX/nhcU9bZSnZoi3h/qGYqQAGhq6zZe/aQW8= +github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/handlers v0.0.0-20160816184729-a5775781a543 h1:dAB4uWBz7LFnjYiZpwFhSJ2fqI23B3mE3XJmz22Zc+g= github.com/gorilla/handlers v0.0.0-20160816184729-a5775781a543/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= +github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH4= +github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= From 060eaf644575f6eb798303d2cea389deb60c9a60 Mon Sep 17 00:00:00 2001 From: agelostsal Date: Wed, 29 Nov 2023 13:52:32 +0200 Subject: [PATCH 6/9] Communication & Training Material Docs --- COMMUNICATION.md | 8 ++++++++ TRAINING_MATERIAL.md | 15 +++++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 COMMUNICATION.md create mode 100644 TRAINING_MATERIAL.md diff --git a/COMMUNICATION.md b/COMMUNICATION.md new file mode 100644 index 00000000..f550f771 --- /dev/null +++ b/COMMUNICATION.md @@ -0,0 +1,8 @@ +# Communication Channels + +There are two ways you can initiate communication with the team behind the +Argo Messaging Service. + +1) If you are coming through EOSC, you should use the +official [EOSC Helpdesk portal.](https://eosc-helpdesk.eosc-portal.eu/#login) +2) Otherwise you can reach out through **_argo at grnet.gr_** \ No newline at end of file diff --git a/TRAINING_MATERIAL.md b/TRAINING_MATERIAL.md new file mode 100644 index 00000000..15ccbe82 --- /dev/null +++ b/TRAINING_MATERIAL.md @@ -0,0 +1,15 @@ +# Training Resources + +We have compiled a list of resources that will assist any newcomer +who wants to understand the messaging service and integrate with it. +The list includes in general presentations of how AMS is being used +while also presenting detailed guides on its core features. + +1) [Getting started with the Argo Messaging Service](./website/docs/howto/how_to_use.md) +2) [Argo Messaging Quick Intro](https://docs.google.com/presentation/d/e/2PACX-1vSIiRzcmgBpn4VyhkBqHrbagNvt_BWBNCAn9f__I9m2vvzRSd2ol2w8-nHhtqbklZvDkyk47a8n65eD/pub?start=false&loop=false&delayms=3000&slide=id.g19e940dd15d_0_45) +3) [AMS Hackathon](https://docs.google.com/presentation/d/e/2PACX-1vSd4wEW6iiU13qT1WDRD_ZsQP7m3v2mlIBGctNW5HpjCS-HlJ9Xb4Kvn15ScSzMwyDfm1YREgtEJzhi/pub?start=false&loop=false&delayms=3000&slide=id.gddb93bcb25_0_645) +4) [Active Data](https://docs.google.com/presentation/d/1zKUM-95qRXszSrLweFdbsgYhvH7UDWoq3hkDWgy2TBM/edit#slide=id.p) +5) [2021/11/03 - EOSC-FUTURE-T5.4 - AMS](https://docs.google.com/presentation/d/e/2PACX-1vQDCfzPFI4vi4A2_yT8PLH00h2_qmbZ27ZxYYSxCobWSCq_tAjIn9j6jTsa9fA1XPggoBidhm5547WK/pub?start=false&loop=false&delayms=3000&slide=id.p) +6) [Publisher Guide - How to send data](./website/docs/guides/publisher.md) +7) [Consumer guide - How to pull data](./website/docs/guides/subscriber_guide.md) +8) [Live updates through Mattermost Integration](./website/docs/guides/mattermost-integration_guide.md) \ No newline at end of file From 9c07cf745971f61c76a04b763dba266811c9f2e5 Mon Sep 17 00:00:00 2001 From: agelostsal Date: Wed, 29 Nov 2023 14:25:03 +0200 Subject: [PATCH 7/9] Use cases documentation --- USE_CASES.md | 66 +++++++++++++++++++++++++++ website/docs/api_advanced/api_subs.md | 1 - 2 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 USE_CASES.md diff --git a/USE_CASES.md b/USE_CASES.md new file mode 100644 index 00000000..ee5466e1 --- /dev/null +++ b/USE_CASES.md @@ -0,0 +1,66 @@ +# Use cases for the Argo Messaging Service + +The integration between different core services using the ARGO Messaging Service (AMS) as transport layer was one of our +main goals. The main services are: + +1) **_EOSC Marketplace (beta)_**: It uses the AMS Service to exchange information about the + orders. +2) **_AAI Federation Registry (beta)_**: It uses the AMS Service to exchange information with the different + deployers (ex, SimpleSamlPhp, Mitre Id, Keycloak). +3) **_Operations Portal_**: Reads the alarms from predefined topics, stores + them in a database and displays them in the operations portal. +4) **_Accounting_**: Use of AMS as a transport layer for + collecting accounting data from the Sites. The accounting information is gathered from different collectors into a + central accounting repository where it is processed to generate statistical summaries that are available through the + EGI + Accounting Portal. +5) **_FedCloud_**: Use of AMS as a transport layer of the cloud information system. It makes use of the + ams-authN. The entry point for users, topics and subscriptions is GOCDB. +6) **_ARGO Availability and Reliability Monitoring + Service_**: It uses the AMS service to send the messages from the monitoring engine to other components. + +### AAI Federation Registry Integration + +The Federation Registry is a portal designed to manage service providers (SPs). It enables service owners to configure +federated access for their services using the OIDC and SAML protocols by providing a centralized location for managing +the service configuration. +Access management is handled by a different component which can differ based on the installation (Keycloak, SSP, +MitreID). Service configurations have to be updated on the Access Managment component every time a change is made using +the Federation Registry Portal. +Argo messaging is the message-oriented middleware technology that is used for this communication between the two +parties. +The use of Argo Messaging Service allows for: + +- **_Flexibility_**: It enables interoperability and integration between different components and systems, regardless of + their + underlying technologies or platforms. +- **_Asynchronous communication_**: Messages can be sent and received at different times and speeds, without blocking or + waiting + for a response. This improves the responsiveness throughout the system. +- **_Security_**: It provides built-in security features, such as authentication, authorization, encryption, and digital + signatures, which help protect the confidentiality, integrity, and authenticity of the messages exchanged between the + Federation Registry and the given Component. + +Federation Registry has multiple instances and with the integration of the Argo Messaging Service we have managed to +organise and monitor +communication between our components. +Configuring Argo Messaging and managing topics and subscription was made easy through the use of AMS Admin Ui app and +through the API. + +### Live Updates through our Mattermost integration + +While the Argo Messaging Service is primarily used for scenarios +where data is being published by one entity and consumed by another, +in order for systems to achieve async event based workflows,the existence +of [push enabled subscriptions](./website/docs/api_advanced/api_subs.md#push-enabled-subscriptions), gives the ability +to the system itself, +to forward messages to remote destination when they arrive, without having clients +constantly asking for new data. + +One use case of this flow, is the ability to deliver data to a specific +mattermost channel. + +We have an [mattermost example](./website/docs/guides/mattermost-integration_guide.md) that mirrors a real use case +where we needed to filter and reformat +specific messages that were actual alerts, that also needed to be delivered +to a mattermost channel in order for issues to be handled as fast as possible. diff --git a/website/docs/api_advanced/api_subs.md b/website/docs/api_advanced/api_subs.md index 55a0d769..57050689 100644 --- a/website/docs/api_advanced/api_subs.md +++ b/website/docs/api_advanced/api_subs.md @@ -234,7 +234,6 @@ This request triggers the process of verifying the ownership of a registered pus curl -X POST "https://{URL}/v1/projects/BRAND_NEW/subscriptions/alert_engine:verifyPushEndpoint?key=S3CR3T"` ``` -### Push Enabled Subscriptions Whenever a subscription is created with a valid push configuration, the service will also generate a unique hash that should be later used to validate the ownership of the registered push endpoint, and will mark the subscription as unverified. From 2753de4e5b9c5ea54d2ad685b7ea13708e553291 Mon Sep 17 00:00:00 2001 From: agelostsal Date: Thu, 30 Nov 2023 14:12:21 +0200 Subject: [PATCH 8/9] Introduce Communication, Use cases And Training Material Docs to the website --- website/docs/communication/_category_.json | 7 ++++++ .../docs/communication/communication.md | 9 +++++++- .../docs/training_material/_category_.json | 7 ++++++ .../training_material/training_material.md | 23 +++++++++++++++---- website/docs/use_cases/_category_.json | 7 ++++++ .../docs/use_cases/use_cases.md | 18 +++++++++++---- 6 files changed, 61 insertions(+), 10 deletions(-) create mode 100644 website/docs/communication/_category_.json rename COMMUNICATION.md => website/docs/communication/communication.md (63%) create mode 100644 website/docs/training_material/_category_.json rename TRAINING_MATERIAL.md => website/docs/training_material/training_material.md (72%) create mode 100644 website/docs/use_cases/_category_.json rename USE_CASES.md => website/docs/use_cases/use_cases.md (92%) diff --git a/website/docs/communication/_category_.json b/website/docs/communication/_category_.json new file mode 100644 index 00000000..009e0ba9 --- /dev/null +++ b/website/docs/communication/_category_.json @@ -0,0 +1,7 @@ +{ + "label": "Communication", + "position": 11, + "link": { + "type": "generated-index" + } +} \ No newline at end of file diff --git a/COMMUNICATION.md b/website/docs/communication/communication.md similarity index 63% rename from COMMUNICATION.md rename to website/docs/communication/communication.md index f550f771..2044a57b 100644 --- a/COMMUNICATION.md +++ b/website/docs/communication/communication.md @@ -1,3 +1,9 @@ +--- +id: communication_channels +title: Communication Channels +sidebar_position: 1 +--- + # Communication Channels There are two ways you can initiate communication with the team behind the @@ -5,4 +11,5 @@ Argo Messaging Service. 1) If you are coming through EOSC, you should use the official [EOSC Helpdesk portal.](https://eosc-helpdesk.eosc-portal.eu/#login) -2) Otherwise you can reach out through **_argo at grnet.gr_** \ No newline at end of file + +2) Otherwise, you can reach out through **_argo at grnet.gr_** \ No newline at end of file diff --git a/website/docs/training_material/_category_.json b/website/docs/training_material/_category_.json new file mode 100644 index 00000000..3eea12a5 --- /dev/null +++ b/website/docs/training_material/_category_.json @@ -0,0 +1,7 @@ +{ + "label": "Training Material", + "position": 10, + "link": { + "type": "generated-index" + } +} \ No newline at end of file diff --git a/TRAINING_MATERIAL.md b/website/docs/training_material/training_material.md similarity index 72% rename from TRAINING_MATERIAL.md rename to website/docs/training_material/training_material.md index 15ccbe82..47e264be 100644 --- a/TRAINING_MATERIAL.md +++ b/website/docs/training_material/training_material.md @@ -1,3 +1,9 @@ +--- +id: training_material +title: Training Material +sidebar_position: 1 +--- + # Training Resources We have compiled a list of resources that will assist any newcomer @@ -5,11 +11,18 @@ who wants to understand the messaging service and integrate with it. The list includes in general presentations of how AMS is being used while also presenting detailed guides on its core features. -1) [Getting started with the Argo Messaging Service](./website/docs/howto/how_to_use.md) +1) [Getting started with the Argo Messaging Service](../howto/how_to_use.md) + 2) [Argo Messaging Quick Intro](https://docs.google.com/presentation/d/e/2PACX-1vSIiRzcmgBpn4VyhkBqHrbagNvt_BWBNCAn9f__I9m2vvzRSd2ol2w8-nHhtqbklZvDkyk47a8n65eD/pub?start=false&loop=false&delayms=3000&slide=id.g19e940dd15d_0_45) -3) [AMS Hackathon](https://docs.google.com/presentation/d/e/2PACX-1vSd4wEW6iiU13qT1WDRD_ZsQP7m3v2mlIBGctNW5HpjCS-HlJ9Xb4Kvn15ScSzMwyDfm1YREgtEJzhi/pub?start=false&loop=false&delayms=3000&slide=id.gddb93bcb25_0_645) + +3) [AMS Hackathon](https://docs.google.com/presentation/d/e/2PACX-1vSd4wEW6iiU13qT1WDRD_ZsQP7m3v2mlIBGctNW5HpjCS-HlJ9Xb4Kvn15ScSzMwyDfm1YREgtEJzhi/pub?start=false&loop=false&delayms=3000&slide=id.p) + 4) [Active Data](https://docs.google.com/presentation/d/1zKUM-95qRXszSrLweFdbsgYhvH7UDWoq3hkDWgy2TBM/edit#slide=id.p) + 5) [2021/11/03 - EOSC-FUTURE-T5.4 - AMS](https://docs.google.com/presentation/d/e/2PACX-1vQDCfzPFI4vi4A2_yT8PLH00h2_qmbZ27ZxYYSxCobWSCq_tAjIn9j6jTsa9fA1XPggoBidhm5547WK/pub?start=false&loop=false&delayms=3000&slide=id.p) -6) [Publisher Guide - How to send data](./website/docs/guides/publisher.md) -7) [Consumer guide - How to pull data](./website/docs/guides/subscriber_guide.md) -8) [Live updates through Mattermost Integration](./website/docs/guides/mattermost-integration_guide.md) \ No newline at end of file + +6) [Publisher Guide - How to send data](../guides/publisher.md) + +7) [Consumer guide - How to pull data](../guides/subscriber_guide.md) + +8) [Live updates through Mattermost Integration](../guides/mattermost-integration_guide.md) diff --git a/website/docs/use_cases/_category_.json b/website/docs/use_cases/_category_.json new file mode 100644 index 00000000..24cd787b --- /dev/null +++ b/website/docs/use_cases/_category_.json @@ -0,0 +1,7 @@ +{ + "label": "Use Cases", + "position": 9, + "link": { + "type": "generated-index" + } +} \ No newline at end of file diff --git a/USE_CASES.md b/website/docs/use_cases/use_cases.md similarity index 92% rename from USE_CASES.md rename to website/docs/use_cases/use_cases.md index ee5466e1..0b6d7f13 100644 --- a/USE_CASES.md +++ b/website/docs/use_cases/use_cases.md @@ -1,3 +1,9 @@ +--- +id: use_cases +title: Use cases +sidebar_position: 1 +--- + # Use cases for the Argo Messaging Service The integration between different core services using the ARGO Messaging Service (AMS) as transport layer was one of our @@ -5,17 +11,21 @@ main goals. The main services are: 1) **_EOSC Marketplace (beta)_**: It uses the AMS Service to exchange information about the orders. + 2) **_AAI Federation Registry (beta)_**: It uses the AMS Service to exchange information with the different deployers (ex, SimpleSamlPhp, Mitre Id, Keycloak). + 3) **_Operations Portal_**: Reads the alarms from predefined topics, stores them in a database and displays them in the operations portal. + 4) **_Accounting_**: Use of AMS as a transport layer for collecting accounting data from the Sites. The accounting information is gathered from different collectors into a central accounting repository where it is processed to generate statistical summaries that are available through the - EGI - Accounting Portal. + EGI Accounting Portal. + 5) **_FedCloud_**: Use of AMS as a transport layer of the cloud information system. It makes use of the ams-authN. The entry point for users, topics and subscriptions is GOCDB. + 6) **_ARGO Availability and Reliability Monitoring Service_**: It uses the AMS service to send the messages from the monitoring engine to other components. @@ -52,7 +62,7 @@ through the API. While the Argo Messaging Service is primarily used for scenarios where data is being published by one entity and consumed by another, in order for systems to achieve async event based workflows,the existence -of [push enabled subscriptions](./website/docs/api_advanced/api_subs.md#push-enabled-subscriptions), gives the ability +of [push enabled subscriptions](../api_advanced/api_subs.md#push-enabled-subscriptions), gives the ability to the system itself, to forward messages to remote destination when they arrive, without having clients constantly asking for new data. @@ -60,7 +70,7 @@ constantly asking for new data. One use case of this flow, is the ability to deliver data to a specific mattermost channel. -We have an [mattermost example](./website/docs/guides/mattermost-integration_guide.md) that mirrors a real use case +We have an [mattermost example](../guides/mattermost-integration_guide.md) that mirrors a real use case where we needed to filter and reformat specific messages that were actual alerts, that also needed to be delivered to a mattermost channel in order for issues to be handled as fast as possible. From 0038db0cf8fff153045c0ab712dabf2cf86f1fbb Mon Sep 17 00:00:00 2001 From: agelostsal Date: Thu, 7 Dec 2023 12:15:34 +0200 Subject: [PATCH 9/9] New Release 1.6.0 --- argo-messaging.spec | 4 +++- version/version.go | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/argo-messaging.spec b/argo-messaging.spec index 7f2bb5a1..5c9bb5eb 100644 --- a/argo-messaging.spec +++ b/argo-messaging.spec @@ -3,7 +3,7 @@ Name: argo-messaging Summary: ARGO Messaging API for broker network -Version: 1.5.0 +Version: 1.6.0 Release: 1%{?dist} License: ASL 2.0 Buildroot: %{_tmppath}/%{name}-buildroot @@ -63,6 +63,8 @@ go clean %attr(0644,root,root) /usr/lib/systemd/system/argo-messaging.service %changelog +* Thu Dec 7 2023 Agelos Tsalapatis 1.6.0-1%{?dist} +- AMS release 1.6.0 * Thu May 18 2023 Agelos Tsalapatis 1.5.0-1%{?dist} - AMS release 1.5.0 * Mon Oct 10 2022 Agelos Tsalapatis 1.4.0-1%{?dist} diff --git a/version/version.go b/version/version.go index 69553bc3..60fca494 100644 --- a/version/version.go +++ b/version/version.go @@ -10,7 +10,7 @@ import ( var ( // Release version of the service. Bump it up during new version release - Release = "1.5.0" + Release = "1.6.0" // Commit hash provided during build Commit = "Unknown" // BuildTime provided during build