comments: Allow retrieving all comment votes.

**Background:**

The comment votes endpoint currently only allows you to retrieve comment
votes for a specific user ID. This initial implementation was
restrictive because that was all that politeiagui needs. Opening up the
endpoint to allow a client to retrieve all comment votes made on a
proposal, without needing to provide a user ID, has been requested in
order to help analyze the proposal process. Comment votes are already
public, so there is no reason to not allow this.

**Implementation:**

- Make the user ID field optional.
- Return paginated comment votes. 
- Add A page number field to the command payload, which defaults to 1 if
  not provided.
- Add new `votesPageSize` comments plugin setting to use it in both of
`politeiawww` & `politeiad`
- Add the new `votesPageSize` plugin setting to the comments' policy reply.
- Update comments plugin command in politeiad to reflect the new
  implementation.
- Update `pictl commentvotes` command to reflect new changes.

politeiad/backendv2/tstorebe/plugins/comments/cmds.go

@@ -1183,28 +1183,21 @@ func (p *commentsPlugin) cmdVotes(token []byte, payload string) (string, error)
 		return "", err
 	}
 
-	// Compile the comment vote digests for all votes that were cast
-	// by the specified user.
-	digests := make([][]byte, 0, 256)
-	for _, cidx := range ridx.Comments {
-		voteIdxs, ok := cidx.Votes[v.UserID]
-		if !ok {
-			// User has not cast any votes for this comment
-			continue
-		}
-
-		// User has cast votes on this comment
-		for _, vidx := range voteIdxs {
-			digests = append(digests, vidx.Digest)
-		}
-	}
+	// Collect the requested page of comment vote digests
+	digests := collectVoteDigestsPage(ridx.Comments, v.UserID, v.Page,
+		p.votesPageSize)
 
 	// Lookup votes
 	votes, err := p.commentVotes(token, digests)
 	if err != nil {
 		return "", fmt.Errorf("commentVotes: %v", err)
 	}
 
+	// Sort comment votes by timestamp from newest to oldest.
+	sort.SliceStable(votes, func(i, j int) bool {
+		return votes[i].Timestamp > votes[j].Timestamp
+	})
+
 	// Prepare reply
 	vr := comments.VotesReply{
 		Votes: votes,
@@ -1217,6 +1210,94 @@ func (p *commentsPlugin) cmdVotes(token []byte, payload string) (string, error)
 	return string(reply), nil
 }
 
+// collectVoteDigestsPage accepts a map of all comment indexes with a
+// filtering criteria and it collects the requested page.
+func collectVoteDigestsPage(commentIdxes map[uint32]commentIndex, userID string, page, pageSize uint32) [][]byte {
+	// Default to first page if page is not provided
+	if page == 0 {
+		page = 1
+	}
+
+	digests := make([][]byte, 0, pageSize)
+	var (
+		pageFirstIndex uint32 = (page - 1) * pageSize
+		pageLastIndex  uint32 = page * pageSize
+		idx            uint32 = 0
+		filterByUserID        = userID != ""
+	)
+
+	// Iterate over record index comments map deterministically; start from
+	// comment id 1 upwards.
+	for commentID := 1; commentID <= len(commentIdxes); commentID++ {
+		cidx := commentIdxes[uint32(commentID)]
+		switch {
+		// User ID filtering criteria is applied. Collect the requested page of
+		// the user's comment votes.
+		case filterByUserID:
+			voteIdxs, ok := cidx.Votes[userID]
+			if !ok {
+				// User has not cast any votes for this comment
+				continue
+			}
+			for _, vidx := range voteIdxs {
+				// Add digest if it's part of the requested page
+				if isInPageRange(idx, pageFirstIndex, pageLastIndex) {
+					digests = append(digests, vidx.Digest)
+
+					// If digests page is full, then we are done
+					if len(digests) == int(pageSize) {
+						return digests
+					}
+				}
+				idx++
+			}
+
+		// No filtering criteria is applied. The votes are indexed by user ID and
+		// saved in a map. In order to return a page of votes in a deterministic
+		// manner, the user IDs must first be sorted, then the pagination is
+		// applied.
+		default:
+			userIDs := getSortedUserIDs(cidx.Votes)
+			for _, userID := range userIDs {
+				for _, vidx := range cidx.Votes[userID] {
+					// Add digest if it's part of the requested page
+					if isInPageRange(idx, pageFirstIndex, pageLastIndex) {
+						digests = append(digests, vidx.Digest)
+
+						// If digests page is full, then we are done
+						if len(digests) == int(pageSize) {
+							return digests
+						}
+					}
+					idx++
+				}
+			}
+		}
+	}
+
+	return digests
+}
+
+// getSortedUserIDs accepts a map of comment vote indexes indexed by user IDs,
+// it collects the keys, sorts them and finally returns them as sorted slice.
+func getSortedUserIDs(m map[string][]voteIndex) []string {
+	userIDs := make([]string, 0, len(m))
+	for userID := range m {
+		userIDs = append(userIDs, userID)
+	}
+
+	// Sort keys
+	sort.Strings(userIDs)
+
+	return userIDs
+}
+
+// isInPageRange determines whether the given index is part of the given
+// page range.
+func isInPageRange(idx, pageFirstIndex, pageLastIndex uint32) bool {
+	return idx >= pageFirstIndex && idx <= pageLastIndex
+}
+
 // cmdTimestamps retrieves the timestamps for the comments of a record.
 func (p *commentsPlugin) cmdTimestamps(token []byte, payload string) (string, error) {
 	// Decode payload

politeiad/backendv2/tstorebe/plugins/comments/cmds_test.go

@@ -0,0 +1,166 @@
+// Copyright (c) 2021 The Decred developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+package comments
+
+import (
+	"encoding/hex"
+	"testing"
+
+	"github.com/decred/politeia/politeiad/plugins/comments"
+)
+
+func TestCollectVoteDigestsPage(t *testing.T) {
+	// Setup test data
+	userIDs := []string{"user1", "user2", "user3"}
+	commentIDs := []uint32{1, 2, 3}
+	token := "testtoken"
+	// Use page size 2 for testing
+	pageSize := uint32(2)
+
+	// Create a comment indexes map for testing with three comment IDs,
+	// which has one comment vote on the first comment from "user1",
+	// another two comment votes on the second second comment from
+	// "user1" and "user2", and lastly another three comment votes on
+	// the third comment from all three test users.
+	commentIdxes := make(map[uint32]commentIndex, len(commentIDs))
+	for _, commentID := range commentIDs {
+		// Prepare comment index Votes map
+		commentIdx := commentIndex{
+			Votes: make(map[string][]voteIndex, commentID),
+		}
+
+		users := userIDs[:commentID]
+		for _, userID := range users {
+			be, err := convertBlobEntryFromCommentVote(comments.CommentVote{
+				UserID:    userID,
+				State:     comments.RecordStateVetted,
+				Token:     token,
+				CommentID: commentID,
+				Vote:      comments.VoteUpvote,
+				PublicKey: "pubkey",
+				Signature: "signature",
+				Timestamp: 1,
+				Receipt:   "receipt",
+			})
+			if err != nil {
+				t.Error(err)
+			}
+			d, err := hex.DecodeString(be.Digest)
+			if err != nil {
+				t.Error(err)
+			}
+			commentIdx.Votes[userID] = []voteIndex{
+				{
+					Digest: d,
+					Vote:   comments.VoteUpvote,
+				},
+			}
+		}
+
+		commentIdxes[commentID] = commentIdx
+	}
+
+	// Setup tests
+	tests := []struct {
+		name                 string
+		page                 uint32
+		userID               string
+		resultExpectedLength int
+	}{
+		{
+			name:                 "first user's first page",
+			page:                 1,
+			userID:               userIDs[0],
+			resultExpectedLength: 2,
+		},
+		{
+			name:                 "first user's second page",
+			page:                 2,
+			userID:               userIDs[0],
+			resultExpectedLength: 1,
+		},
+		{
+			name:                 "first user's third page",
+			page:                 3,
+			userID:               userIDs[0],
+			resultExpectedLength: 0,
+		},
+		{
+			name:                 "second user's first page",
+			page:                 1,
+			userID:               userIDs[1],
+			resultExpectedLength: 2,
+		},
+		{
+			name:                 "second user's second page",
+			page:                 2,
+			userID:               userIDs[1],
+			resultExpectedLength: 0,
+		},
+		{
+			name:                 "third user's first page",
+			page:                 1,
+			userID:               userIDs[2],
+			resultExpectedLength: 1,
+		},
+		{
+			name:                 "third user's second page",
+			page:                 2,
+			userID:               userIDs[2],
+			resultExpectedLength: 0,
+		},
+		{
+			name:                 "all votes first page",
+			page:                 1,
+			userID:               "",
+			resultExpectedLength: 2,
+		},
+		{
+			name:                 "all votes second page",
+			page:                 2,
+			userID:               "",
+			resultExpectedLength: 2,
+		},
+		{
+			name:                 "all votes third page",
+			page:                 3,
+			userID:               "",
+			resultExpectedLength: 2,
+		},
+		{
+			name:                 "all votes forth page",
+			page:                 4,
+			userID:               "",
+			resultExpectedLength: 0,
+		},
+		{
+			name:                 "default to first page with filtering criteria",
+			page:                 0,
+			userID:               userIDs[2],
+			resultExpectedLength: 1,
+		},
+		{
+			name:                 "default to first page w/o filtering criteria",
+			page:                 0,
+			userID:               "",
+			resultExpectedLength: 2,
+		},
+	}
+
+	// Run tests
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			// Run test
+			digests := collectVoteDigestsPage(commentIdxes, tc.userID, tc.page,
+				pageSize)
+
+			// Verify length of returned page
+			if len(digests) != tc.resultExpectedLength {
+				t.Errorf("unexpected result length; want %v, got %v",
+					commentIdxes, digests)
+			}
+		})
+	}
+}

politeiad/backendv2/tstorebe/plugins/comments/comments.go

@@ -43,6 +43,7 @@ type commentsPlugin struct {
 	commentLengthMax uint32
 	voteChangesMax   uint32
 	allowExtraData   bool
+	votesPageSize    uint32
 }
 
 // Setup performs any plugin setup that is required.
@@ -155,6 +156,10 @@ func (p *commentsPlugin) Settings() []backend.PluginSetting {
 			Key:   comments.SettingKeyAllowExtraData,
 			Value: strconv.FormatBool(p.allowExtraData),
 		},
+		{
+			Key:   comments.SettingKeyVotesPageSize,
+			Value: strconv.FormatUint(uint64(p.votesPageSize), 10),
+		},
 	}
 }
 
@@ -172,6 +177,7 @@ func New(tstore plugins.TstoreClient, settings []backend.PluginSetting, dataDir
 		commentLengthMax = comments.SettingCommentLengthMax
 		voteChangesMax   = comments.SettingVoteChangesMax
 		allowExtraData   = comments.SettingAllowExtraData
+		votesPageSize    = comments.SettingVotesPageSize
 	)
 
 	// Override defaults with any passed in settings
@@ -198,6 +204,13 @@ func New(tstore plugins.TstoreClient, settings []backend.PluginSetting, dataDir
 					v.Key, v.Value, err)
 			}
 			allowExtraData = b
+		case comments.SettingKeyVotesPageSize:
+			u, err := strconv.ParseUint(v.Value, 10, 64)
+			if err != nil {
+				return nil, errors.Errorf("invalid plugin setting %v '%v': %v",
+					v.Key, v.Value, err)
+			}
+			votesPageSize = uint32(u)
 		default:
 			return nil, errors.Errorf("invalid comments plugin setting '%v'", v.Key)
 		}
@@ -210,5 +223,6 @@ func New(tstore plugins.TstoreClient, settings []backend.PluginSetting, dataDir
 		commentLengthMax: commentLengthMax,
 		voteChangesMax:   voteChangesMax,
 		allowExtraData:   allowExtraData,
+		votesPageSize:    votesPageSize,
 	}, nil
 }

politeiad/plugins/comments/comments.go

@@ -38,10 +38,14 @@ const (
 	// SettingKeyAllowExtraData is the plugin setting key for the
 	// SettingAllowExtraData plugin setting.
 	SettingKeyAllowExtraData = "allowextradata"
+
+	// SettingKeyVotesPageSize is the plugin setting key for the
+	// SettingVotesPageSize plugin setting.
+	SettingKeyVotesPageSize = "votespagesize"
 )
 
-// Plugin setting default values. These can be overridden by providing a plugin
-// setting key and value to the plugin on startup.
+// Plugin setting default values. These can be overridden by providing a
+// plugin setting key and value to the plugin on startup.
 const (
 	// SettingCommentLengthMax is the default maximum number of
 	// characters that are allowed in a comment.
@@ -55,6 +59,13 @@ const (
 	// SettingAllowExtraData is the default value of the bool flag which
 	// determines whether posting extra data along with the comment is allowed.
 	SettingAllowExtraData = false
+
+	// SettingVotesPageSize is the default maximum number of comment votes
+	// that can be returned at any one time. It defaults to 2500 to limit the
+	// comment votes route payload size to be ~1MiB, as each comment vote size
+	// is expected to be around 400 bytes which means:
+	// 2500 * 400 byte = 1000000 byte = ~1MiB.
+	SettingVotesPageSize uint32 = 2500
 )
 
 // ErrorCodeT represents a error that was caused by the user.
@@ -463,9 +474,14 @@ type CountReply struct {
 	Count uint32 `json:"count"`
 }
 
-// Votes retrieves the comment votes that meet the provided filtering criteria.
+// Votes retrieves the record's comment votes that meet the provided filtering
+// criteria. If no filtering criteria is provided then it rerieves all comment
+// votes. This command is paginated, if no page is provided, then the first
+// page is returned. If the requested page does not exist an empty page
+// is returned.
 type Votes struct {
-	UserID string `json:"userid"`
+	UserID string `json:"userid,omitempty"`
+	Page   uint32 `json:"page,omitempty"`
 }
 
 // VotesReply is the reply to the Votes command.