Skip to content

Commit

Permalink
Move all schemas to a single file schemas.py
Browse files Browse the repository at this point in the history 
This reduces clutter and avoids duplicate definitions.

Also:

The run schema has been tightened.

Various tweaks in other schemas.

Requires upgrading the vtjson package.
  • Loading branch information
@vdbergh @ppigazzini
vdbergh authored and ppigazzini committed Apr 4, 2024
1 parent 7943cb1 commit c59a834
Show file tree
Hide file tree
Showing 10 changed files with 668 additions and 533 deletions.
175 changes: 12 additions & 163 deletions server/fishtest/actiondb.py
View file
Original file line number Diff line number Diff line change
@@ -1,170 +1,10 @@
from datetime import datetime, timezone

from bson.objectid import ObjectId
from fishtest.schemas import action_schema
from fishtest.util import hex_print, worker_name
from pymongo import DESCENDING
from vtjson import regex, union, validate

run_id = regex(r"[a-f0-9]{24}", name="run_id")
run_name = regex(r".*-[a-f0-9]{7}", name="run_name")
short_worker_name = regex(r".*-[\d]+cores-[a-zA-Z0-9]{2,8}", name="short_worker_name")
long_worker_name = regex(
r".*-[\d]+cores-[a-zA-Z0-9]{2,8}-[a-f0-9]{4}\*?", name="long_worker_name"
)

schema = union(
{
"_id?": ObjectId,
"time": float,
"action": "failed_task",
"username": str,
"worker": long_worker_name,
"run_id": run_id,
"run": run_name,
"task_id": int,
"message": str,
},
{
"_id?": ObjectId,
"time": float,
"action": "crash_or_time",
"username": str,
"worker": long_worker_name,
"run_id": run_id,
"run": run_name,
"task_id": int,
"message": str,
},
{
"_id?": ObjectId,
"time": float,
"action": "dead_task",
"username": str,
"worker": long_worker_name,
"run_id": run_id,
"run": run_name,
"task_id": int,
},
{
"_id?": ObjectId,
"time": float,
"action": "system_event",
"username": "fishtest.system",
"message": str,
},
{
"_id?": ObjectId,
"time": float,
"action": "new_run",
"username": str,
"run_id": run_id,
"run": run_name,
"message": str,
},
{
"_id?": ObjectId,
"time": float,
"action": "upload_nn",
"username": str,
"nn": str,
},
{
"_id?": ObjectId,
"time": float,
"action": "modify_run",
"username": str,
"run_id": run_id,
"run": run_name,
"message": str,
},
{
"_id?": ObjectId,
"time": float,
"action": "delete_run",
"username": str,
"run_id": run_id,
"run": run_name,
},
{
"_id?": ObjectId,
"time": float,
"action": "stop_run",
"username": str,
"worker": long_worker_name,
"run_id": run_id,
"run": run_name,
"task_id": int,
"message": str,
},
{
"_id?": ObjectId,
"time": float,
"action": "stop_run",
"username": str,
"run_id": run_id,
"run": run_name,
"message": str,
},
{
"_id?": ObjectId,
"time": float,
"action": "finished_run",
"username": str,
"run_id": run_id,
"run": run_name,
"message": str,
},
{
"_id?": ObjectId,
"time": float,
"action": "approve_run",
"username": str,
"run_id": run_id,
"run": run_name,
},
{
"_id?": ObjectId,
"time": float,
"action": "purge_run",
"username": str,
"run_id": run_id,
"run": run_name,
"message": str,
},
{
"_id?": ObjectId,
"time": float,
"action": "block_user",
"username": str,
"user": str,
"message": union("blocked", "unblocked"),
},
{
"_id?": ObjectId,
"time": float,
"action": "accept_user",
"username": str,
"user": str,
"message": "accepted",
},
{
"_id?": ObjectId,
"time": float,
"action": "block_worker",
"username": str,
"worker": short_worker_name,
"message": union("blocked", "unblocked"),
},
{
"_id?": ObjectId,
"time": float,
"action": "log_message",
"username": str,
"message": str,
},
)

del long_worker_name, run_id, run_name, short_worker_name
from vtjson import ValidationError, validate


def run_name(run):
Expand Down Expand Up @@ -376,5 +216,14 @@ def insert_action(self, **action):
if "run_id" in action:
action["run_id"] = str(action["run_id"])
action["time"] = datetime.now(timezone.utc).timestamp()
validate(schema, action, "action") # may raise exception
try:
validate(action_schema, action, "action")
except ValidationError:
message = f"Internal Error. Request {str(action)} does not validate"
print(message, flush=True)
self.log_message(
username="fishtest.system",
message=message,
)
return
self.actions.insert_one(action)
91 changes: 6 additions & 85 deletions server/fishtest/api.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import re
from datetime import datetime, timezone

from fishtest.schemas import api_access_schema, api_schema
from fishtest.stats.stat_util import SPRT_elo
from fishtest.util import worker_name
from pyramid.httpexceptions import (
Expand All @@ -14,7 +15,7 @@
)
from pyramid.response import Response
from pyramid.view import exception_view_config, view_config, view_defaults
from vtjson import ValidationError, compile, intersect, interval, lax, regex, validate
from vtjson import ValidationError, validate

"""
Important note
Expand All @@ -33,83 +34,6 @@

WORKER_VERSION = 232

"""
begin api_schema
"""

run_id = regex(r"[a-f0-9]{24}", name="run_id")
uuid = regex(r"[0-9a-zA-Z]{2,8}(-[a-f0-9]{4}){3}-[a-f0-9]{12}", name="uuid")

uint = intersect(int, interval(0, ...))
suint = intersect(int, interval(1, ...))
ufloat = intersect(float, interval(0.0, ...))


def valid_results(R):
l, d, w = R["losses"], R["draws"], R["wins"]
R = R["pentanomial"]
return (
l + d + w == 2 * sum(R)
and w - l == 2 * R[4] + R[3] - R[1] - 2 * R[0]
and R[3] + 2 * R[2] + R[1] >= d >= R[3] + R[1]
)


def valid_spsa_results(R):
return R["wins"] + R["losses"] + R["draws"] == R["num_games"]


api_schema = {
"password": str,
"run_id?": run_id,
"task_id?": uint,
"pgn?": str,
"message?": str,
"worker_info": {
"uname": str,
"architecture": [str, str],
"concurrency": suint,
"max_memory": suint,
"min_threads": suint,
"username": str,
"version": uint,
"python_version": [uint, uint, uint],
"gcc_version": [uint, uint, uint],
"compiler": {"g++", "clang++"},
"unique_key": uuid,
"modified": bool,
"near_github_api_limit": bool,
"ARCH": str,
"nps": ufloat,
},
"spsa?": intersect(
{
"wins": uint,
"losses": uint,
"draws": uint,
"num_games": uint,
},
valid_spsa_results,
),
"stats?": intersect(
{
"wins": uint,
"losses": uint,
"draws": uint,
"crashes": uint,
"time_losses": uint,
"pentanomial": [uint, uint, uint, uint, uint],
},
valid_results,
),
}

api_schema = compile(api_schema)

"""
end api_schema
"""


def validate_request(request):
validate(api_schema, request, "request")
Expand Down Expand Up @@ -180,9 +104,8 @@ def validate_username_password(self, api):
self.handle_error("request is not json encoded")

# Is the request syntactically correct?
schema = lax({"password": str, "worker_info": {"username": str}})
try:
validate(schema, self.request_body, "request")
validate(api_access_schema, self.request_body, "request")
except ValidationError as e:
self.handle_error(str(e))

Expand Down Expand Up @@ -218,13 +141,11 @@ def validate_request(self, api):
self.handle_error("Invalid run_id: {}".format(run_id))
self.__run = run

# if a task_id is present then there should be a run_id, and
# the unique_key should correspond to the unique_key of the
# task
# if a task_id is present then the unique_key should correspond
# to the unique_key of the task

if "task_id" in self.request_body:
task_id = self.request_body["task_id"]
if "run_id" not in self.request_body:
self.handle_error("The request has a task_id but no run_id")

if task_id < 0 or task_id >= len(run["tasks"]):
self.handle_error(
Expand Down
Loading

0 comments on commit c59a834

Please # to comment.