Skip to content

Commit

Permalink
Merge pull request #1 from t3n/options
Browse files Browse the repository at this point in the history 
Added options
  • Loading branch information
@helmecke
helmecke authored May 6, 2020
2 parents f214626 + 8ad7538 commit 36b922c
Show file tree
Hide file tree
Showing 3 changed files with 67 additions and 50 deletions.
10 changes: 10 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,18 @@ Alpha, everything is subject to change.

## Install

```bash
$ docker pull t3nde/gtmetrix-bq
```

## Usage

```bash
# copy example config and edit
$ cp example-config.yaml config.yaml; vim config.yaml
$ docker run t3nde/gtmetrix-bq -v ${PWD}/config.yaml:/app/config.yaml --env GTMETRIX_REST_API_EMAIL=<email> --env GTMETRIX_REST_API_KEY=<key> t3nde/gtmetrix-bq
```

## Contributing

PRs accepted.
Expand Down
8 changes: 6 additions & 2 deletions example-config.yaml
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
bq:
dataset: 'gtmetrix'
table: 'test'
urls:
- https://google.com/
tests:
- url: https://google.com/
options:
x-metrix-adblock: 1
- url: https:/google.com/
options: {}
99 changes: 51 additions & 48 deletions gtmetrix-bq.py
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
#!/usr/bin/env python

from datetime import datetime
from yaml import load, BaseLoader
import yaml
import json
from gtmetrix import GTmetrixInterface
from google.cloud import bigquery
from google.api_core.exceptions import NotFound
Expand All @@ -16,7 +17,7 @@ def bq_create_dataset(dataset):
except NotFound:
dataset = bigquery.Dataset(dataset_ref)
dataset = bigquery_client.create_dataset(dataset)
print('Dataset {} created.'.format(dataset.dataset_id))
print("Dataset {} created.".format(dataset.dataset_id))


def bq_create_table(dataset, table):
Expand All @@ -30,33 +31,35 @@ def bq_create_table(dataset, table):
bigquery_client.get_table(table_ref)
except NotFound:
schema = [
bigquery.SchemaField('timestamp', 'DATETIME', mode='REQUIRED'),
bigquery.SchemaField('url', 'STRING', mode='REQUIRED'),
bigquery.SchemaField('adblock', 'BOOLEAN', mode='REQUIRED'),
bigquery.SchemaField('onload_time', 'INTEGER', mode='REQUIRED'),
bigquery.SchemaField('first_contentful_paint_time', 'INTEGER', mode='REQUIRED'),
bigquery.SchemaField('page_elements', 'INTEGER', mode='REQUIRED'),
bigquery.SchemaField('report_url', 'STRING', mode='REQUIRED'),
bigquery.SchemaField('redirect_duration', 'INTEGER', mode='REQUIRED'),
bigquery.SchemaField('first_paint_time', 'INTEGER', mode='REQUIRED'),
bigquery.SchemaField('dom_content_loaded_duration', 'INTEGER'),
bigquery.SchemaField('dom_content_loaded_time', 'INTEGER', mode='REQUIRED'),
bigquery.SchemaField('dom_interactive_time', 'INTEGER', mode='REQUIRED'),
bigquery.SchemaField('page_bytes', 'INTEGER', mode='REQUIRED'),
bigquery.SchemaField('page_load_time', 'INTEGER', mode='REQUIRED'),
bigquery.SchemaField('html_bytes', 'INTEGER', mode='REQUIRED'),
bigquery.SchemaField('fully_loaded_time', 'INTEGER', mode='REQUIRED'),
bigquery.SchemaField('html_load_time', 'INTEGER', mode='REQUIRED'),
bigquery.SchemaField('rum_speed_index', 'INTEGER', mode='REQUIRED'),
bigquery.SchemaField('yslow_score', 'INTEGER', mode='REQUIRED'),
bigquery.SchemaField('pagespeed_score', 'INTEGER', mode='REQUIRED'),
bigquery.SchemaField('backend_duration', 'INTEGER', mode='REQUIRED'),
bigquery.SchemaField('onload_duration', 'INTEGER', mode='REQUIRED'),
bigquery.SchemaField('connect_duration', 'INTEGER', mode='REQUIRED'),
bigquery.SchemaField("timestamp", "DATETIME", mode="REQUIRED"),
bigquery.SchemaField("url", "STRING", mode="REQUIRED"),
bigquery.SchemaField("options", "STRING", mode="REQUIRED"),
bigquery.SchemaField("onload_time", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField(
"first_contentful_paint_time", "INTEGER", mode="REQUIRED"
),
bigquery.SchemaField("page_elements", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("report_url", "STRING", mode="REQUIRED"),
bigquery.SchemaField("redirect_duration", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("first_paint_time", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("dom_content_loaded_duration", "INTEGER"),
bigquery.SchemaField("dom_content_loaded_time", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("dom_interactive_time", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("page_bytes", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("page_load_time", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("html_bytes", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("fully_loaded_time", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("html_load_time", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("rum_speed_index", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("yslow_score", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("pagespeed_score", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("backend_duration", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("onload_duration", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField("connect_duration", "INTEGER", mode="REQUIRED"),
]
table = bigquery.Table(table_ref, schema=schema)
table = bigquery_client.create_table(table)
print('table {} created.'.format(table.table_id))
print("table {} created.".format(table.table_id))


def bq_insert_rows(dataset, table, rows_to_insert):
Expand All @@ -73,30 +76,30 @@ def bq_insert_rows(dataset, table, rows_to_insert):
assert errors == []


if __name__ == '__main__':
with open("config.yaml", 'r') as stream:
config = load(stream, BaseLoader)
if __name__ == "__main__":
with open("config.yaml", "r") as stream:
config = yaml.load(stream, yaml.BaseLoader)

dataset_id = config['bq']['dataset']
table_id = config['bq']['table']
dataset_id = config["bq"]["dataset"]
table_id = config["bq"]["table"]
bq_create_dataset(dataset_id)
bq_create_table(dataset_id, table_id)
gt = GTmetrixInterface()

for url in config['urls']:
for options in [{}, {'x-metrix-adblock': 1}]:
test = gt.start_test(url, **options)
results = test.fetch_results('results')['results']
results['url'] = url
results['timestamp'] = datetime.utcnow()

if not bool(options):
results['adblock'] = False
else:
results['adblock'] = True

for k, v in results.items():
if v is None:
results[k] = 0

bq_insert_rows(dataset_id, table_id, [results])
for test in config["tests"]:
print(
"Testing {} with {} options...".format(
test["url"], json.dumps(test["options"])
)
)
gt_test = gt.start_test(test["url"], **test["options"])
results = gt_test.fetch_results("results")["results"]
results["url"] = test["url"]
results["timestamp"] = datetime.utcnow()
results["options"] = json.dumps(test["options"])

for k, v in results.items():
if v is None:
results[k] = 0

bq_insert_rows(dataset_id, table_id, [results])

0 comments on commit 36b922c

Please # to comment.