Skip to content
This repository was archived by the owner on Nov 1, 2022. It is now read-only.

Commit fe4b178

Browse files
linabutlerlinabutler
committed
Record Sync engine telemetry in Glean.
This commit adds two Glean pings to the Storage-Sync browser component: one for history, and one for bookmarks. Both pings record the same set of base metrics, including incoming and outgoing counts, sync duration, the hashed FxA UID, and, most importantly, the failure reason if the store fails to sync. The bookmarks ping records additional validation data. The Glean schema is a flattened version of the Sync ping that's currently sent on Firefox Desktop and Firefox for iOS. Instead of sending a single ping with multiple syncs, each having multiple engines, we send one Glean ping per engine per sync. The pings are recorded directly in the component, so any app that consumes it should get pings for free.
1 parent 2e06f7f commit fe4b178

File tree

10 files changed

+477
-16
lines changed

10 files changed

+477
-16
lines changed

‎buildSrc/src/main/java/Dependencies.kt

+1
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,7 @@ object Dependencies {
111111
const val mozilla_rustlog = "org.mozilla.appservices:rustlog:${Versions.mozilla_appservices}"
112112
const val mozilla_servo_arm = "org.mozilla.servoview:servoview-armv7:${Versions.servo}"
113113
const val mozilla_servo_x86 = "org.mozilla.servoview:servoview-x86:${Versions.servo}"
114+
const val mozilla_sync15 = "org.mozilla.appservices:sync15:${Versions.mozilla_appservices}"
114115

115116
const val thirdparty_okhttp = "com.squareup.okhttp3:okhttp:${Versions.okhttp}"
116117
const val thirdparty_okhttp_urlconnection = "com.squareup.okhttp3:okhttp-urlconnection:${Versions.okhttp}"

‎components/browser/storage-sync/build.gradle

+8
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,10 @@
22
* License, v. 2.0. If a copy of the MPL was not distributed with this
33
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
44

5+
plugins {
6+
id "com.jetbrains.python.envs" version "0.0.26"
7+
}
8+
59
apply plugin: 'com.android.library'
610
apply plugin: 'kotlin-android'
711

@@ -46,7 +50,9 @@ dependencies {
4650
api project(':concept-storage')
4751
api project(':concept-sync')
4852

53+
implementation project(':service-glean')
4954
implementation project(':support-utils')
55+
implementation Dependencies.mozilla_sync15
5056

5157
implementation Dependencies.kotlin_stdlib
5258

@@ -65,3 +71,5 @@ dependencies {
6571

6672
apply from: '../../../publish.gradle'
6773
ext.configurePublish(config.componentsGroupId, archivesBaseName, project.ext.description)
74+
75+
apply from: '../../service/glean/scripts/sdk_generator.gradle'

‎components/browser/storage-sync/metrics.yaml

+257
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,257 @@
1+
# This Source Code Form is subject to the terms of the Mozilla Public
2+
# License, v. 2.0. If a copy of the MPL was not distributed with this
3+
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
4+
5+
$schema: moz://mozilla.org/schemas/glean/metrics/1-0-0
6+
7+
# The `history_sync` and `bookmarks_sync` metrics use the same structure, but, since are
8+
# sent in separate pings, after each sync. This means we submit one ping per
9+
# store per sync.
10+
history_sync:
11+
uid:
12+
type: string
13+
description: >
14+
The user's hashed Firefox Account ID.
15+
send_in_pings:
16+
- history_sync
17+
bugs:
18+
- 3092
19+
data_reviews:
20+
- https://github.com/mozilla-mobile/android-components/pull/3092
21+
notification_emails:
22+
- sync-core@mozilla.com
23+
expires: never
24+
started_at:
25+
type: datetime
26+
time_unit: millisecond
27+
description: >
28+
Records when the history sync started.
29+
send_in_pings:
30+
- history_sync
31+
bugs:
32+
- 3092
33+
data_reviews:
34+
- https://github.com/mozilla-mobile/android-components/pull/3092
35+
notification_emails:
36+
- sync-core@mozilla.com
37+
expires: never
38+
finished_at:
39+
type: datetime
40+
time_unit: millisecond
41+
description: >
42+
Records when the history sync finished. This includes the time to
43+
download, apply, and upload all records.
44+
send_in_pings:
45+
- history_sync
46+
bugs:
47+
- 3092
48+
data_reviews:
49+
- https://github.com/mozilla-mobile/android-components/pull/3092
50+
notification_emails:
51+
- sync-core@mozilla.com
52+
expires: never
53+
incoming:
54+
type: labeled_counter
55+
labels:
56+
- applied
57+
- failed_to_apply
58+
- reconciled
59+
description: >
60+
Records incoming history record counts. `applied` is the number of
61+
incoming history pages that were successfully stored or updated in the
62+
local database. `failed_to_apply` is the number of pages that were
63+
ignored due to errors. `reconciled` is the number of pages with new visits
64+
locally and remotely, and had their visits merged.
65+
send_in_pings:
66+
- history_sync
67+
bugs:
68+
- 3092
69+
data_reviews:
70+
- https://github.com/mozilla-mobile/android-components/pull/3092
71+
notification_emails:
72+
- sync-core@mozilla.com
73+
expires: never
74+
outgoing:
75+
type: labeled_counter
76+
labels:
77+
- uploaded
78+
- failed_to_upload
79+
description: >
80+
Records outgoing history record counts. `uploaded` is the number of
81+
records that were successfully sent to the server. `failed_to_upload`
82+
is the number of records that weren't uploaded, and will be retried
83+
on the next sync.
84+
send_in_pings:
85+
- history_sync
86+
bugs:
87+
- 3092
88+
data_reviews:
89+
- https://github.com/mozilla-mobile/android-components/pull/3092
90+
notification_emails:
91+
- sync-core@mozilla.com
92+
expires: never
93+
outgoing_batches:
94+
type: counter
95+
description: >
96+
Records the number of batches needed to upload all outgoing records. The
97+
Sync server has a hard limit on the number of records (and request body
98+
bytes) on the number of records that can fit into a single batch, and
99+
large syncs may require multiple batches.
100+
send_in_pings:
101+
- history_sync
102+
bugs:
103+
- 3092
104+
data_reviews:
105+
- https://github.com/mozilla-mobile/android-components/pull/3092
106+
notification_emails:
107+
- sync-core@mozilla.com
108+
expires: never
109+
failure_reason:
110+
type: labeled_string
111+
labels:
112+
- other
113+
- unexpected
114+
- auth
115+
description: >
116+
Records why the history sync failed: either due to an authentication
117+
error, unexpected exception, or other error. The error strings are
118+
truncated and sanitized to omit PII, like URLs and file system paths.
119+
send_in_pings:
120+
- history_sync
121+
bugs:
122+
- 3092
123+
data_reviews:
124+
- https://github.com/mozilla-mobile/android-components/pull/3092
125+
notification_emails:
126+
- sync-core@mozilla.com
127+
expires: never
128+
129+
bookmarks_sync:
130+
uid:
131+
type: string
132+
description: >
133+
The user's hashed Firefox Account ID.
134+
send_in_pings:
135+
- bookmarks_sync
136+
bugs:
137+
- 3092
138+
data_reviews:
139+
- https://github.com/mozilla-mobile/android-components/pull/3092
140+
notification_emails:
141+
- sync-core@mozilla.com
142+
expires: never
143+
started_at:
144+
type: datetime
145+
time_unit: millisecond
146+
description: >
147+
Records when the bookmark sync started.
148+
send_in_pings:
149+
- bookmarks_sync
150+
bugs:
151+
- 3092
152+
data_reviews:
153+
- https://github.com/mozilla-mobile/android-components/pull/3092
154+
notification_emails:
155+
- sync-core@mozilla.com
156+
expires: never
157+
finished_at:
158+
type: datetime
159+
time_unit: millisecond
160+
description: >
161+
Records when the bookmark sync finished.
162+
send_in_pings:
163+
- bookmarks_sync
164+
bugs:
165+
- 3092
166+
data_reviews:
167+
- https://github.com/mozilla-mobile/android-components/pull/3092
168+
notification_emails:
169+
- sync-core@mozilla.com
170+
expires: never
171+
incoming:
172+
type: labeled_counter
173+
labels:
174+
- applied
175+
- failed_to_apply
176+
- reconciled
177+
description: >
178+
Records incoming bookmark record counts.
179+
send_in_pings:
180+
- bookmarks_sync
181+
bugs:
182+
- 3092
183+
data_reviews:
184+
- https://github.com/mozilla-mobile/android-components/pull/3092
185+
notification_emails:
186+
- sync-core@mozilla.com
187+
expires: never
188+
outgoing:
189+
type: labeled_counter
190+
labels:
191+
- uploaded
192+
- failed_to_upload
193+
description: >
194+
Records outgoing bookmark record counts.
195+
send_in_pings:
196+
- bookmarks_sync
197+
bugs:
198+
- 3092
199+
data_reviews:
200+
- https://github.com/mozilla-mobile/android-components/pull/3092
201+
notification_emails:
202+
- sync-core@mozilla.com
203+
expires: never
204+
outgoing_batches:
205+
type: counter
206+
description: >
207+
Records the number of batches needed to upload all outgoing records.
208+
send_in_pings:
209+
- bookmarks_sync
210+
bugs:
211+
- 3092
212+
data_reviews:
213+
- https://github.com/mozilla-mobile/android-components/pull/3092
214+
notification_emails:
215+
- sync-core@mozilla.com
216+
expires: never
217+
failure_reason:
218+
type: labeled_string
219+
labels:
220+
- other
221+
- unexpected
222+
- auth
223+
description: >
224+
Records bookmark sync failure reasons.
225+
send_in_pings:
226+
- bookmarks_sync
227+
bugs:
228+
- 3092
229+
data_reviews:
230+
- https://github.com/mozilla-mobile/android-components/pull/3092
231+
notification_emails:
232+
- sync-core@mozilla.com
233+
expires: never
234+
remote_tree_problems:
235+
type: labeled_counter
236+
version: 3
237+
labels:
238+
- orphans
239+
- misparented_roots
240+
- multiple_parents_by_children
241+
- missing_parent_guids
242+
- non_folder_parent_guids
243+
- parent_child_disagreements
244+
- missing_children
245+
description: >
246+
Records counts for structure problems and divergences in the remote
247+
bookmarks tree. These are documented in
248+
https://github.com/mozilla/dogear/blob/fbade15f2a4f11215e30b8f428a0a8df3defeaec/src/tree.rs#L1273-L1294.
249+
send_in_pings:
250+
- bookmarks_sync
251+
bugs:
252+
- 3092
253+
data_reviews:
254+
- https://github.com/mozilla-mobile/android-components/pull/3092
255+
notification_emails:
256+
- sync-core@mozilla.com
257+
expires: never

‎components/browser/storage-sync/pings.yaml

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
# This Source Code Form is subject to the terms of the Mozilla Public
2+
# License, v. 2.0. If a copy of the MPL was not distributed with this
3+
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
4+
5+
$schema: moz://mozilla.org/schemas/glean/pings/1-0-0
6+
7+
history_sync:
8+
description: >
9+
A ping sent for every history sync. It doesn't include the `client_id`
10+
because it reports a hashed version of the user's Firefox Account ID.
11+
include_client_id: false
12+
bugs:
13+
- 3092
14+
notification_emails:
15+
- sync-core@mozilla.com
16+
data_reviews:
17+
- https://github.com/mozilla-mobile/android-components/pull/3092
18+
bookmarks_sync:
19+
description: >
20+
A ping sent for every bookmarks sync. It doesn't include the `client_id`
21+
because it reports a hashed version of the user's Firefox Account ID.
22+
include_client_id: false
23+
bugs:
24+
- 3092
25+
notification_emails:
26+
- sync-core@mozilla.com
27+
data_reviews:
28+
- https://github.com/mozilla-mobile/android-components/pull/3092

‎components/browser/storage-sync/src/main/java/mozilla/components/browser/storage/sync/Connection.kt

+8-7
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import androidx.annotation.GuardedBy
88
import mozilla.appservices.places.PlacesApi
99
import mozilla.appservices.places.PlacesReaderConnection
1010
import mozilla.appservices.places.PlacesWriterConnection
11+
import mozilla.appservices.sync15.SyncTelemetryPing
1112
import java.io.Closeable
1213
import java.io.File
1314

@@ -22,14 +23,14 @@ const val DB_NAME = "places.sqlite"
2223
*
2324
* Writer is always the same, as guaranteed by [PlacesApi].
2425
*/
25-
interface Connection : Closeable {
26+
internal interface Connection : Closeable {
2627
fun reader(): PlacesReaderConnection
2728
fun writer(): PlacesWriterConnection
2829

2930
// Until we get a real SyncManager in application-services libraries, we'll have to live with this
3031
// strange split that doesn't quite map all that well to our internal storage model.
31-
fun syncHistory(syncInfo: SyncAuthInfo)
32-
fun syncBookmarks(syncInfo: SyncAuthInfo)
32+
fun syncHistory(syncInfo: SyncAuthInfo): SyncTelemetryPing
33+
fun syncBookmarks(syncInfo: SyncAuthInfo): SyncTelemetryPing
3334
}
3435

3536
/**
@@ -65,14 +66,14 @@ internal object RustPlacesConnection : Connection {
6566
return api!!.getWriter()
6667
}
6768

68-
override fun syncHistory(syncInfo: SyncAuthInfo) {
69+
override fun syncHistory(syncInfo: SyncAuthInfo): SyncTelemetryPing {
6970
check(api != null) { "must call init first" }
70-
api!!.syncHistory(syncInfo)
71+
return api!!.syncHistory(syncInfo)
7172
}
7273

73-
override fun syncBookmarks(syncInfo: SyncAuthInfo) {
74+
override fun syncBookmarks(syncInfo: SyncAuthInfo): SyncTelemetryPing {
7475
check(api != null) { "must call init first" }
75-
api!!.syncBookmarks(syncInfo)
76+
return api!!.syncBookmarks(syncInfo)
7677
}
7778

7879
override fun close() = synchronized(this) {

0 commit comments

Comments
 (0)