From b8318d809113d21a754e7b960fb1fe60de9d5d9c Mon Sep 17 00:00:00 2001 From: EmmaLRussell Date: Mon, 12 Feb 2024 14:56:48 +0000 Subject: [PATCH 1/6] surface error details on unknown project hash --- beebop/app.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beebop/app.py b/beebop/app.py index fe366cd5..71be4220 100644 --- a/beebop/app.py +++ b/beebop/app.py @@ -304,8 +304,8 @@ def get_status_job(job, p_hash, redis): return {"assign": status_assign, "microreact": status_microreact, "network": status_network} - except AttributeError: - return {"error": "Unknown project hash"} + except AttributeError as err: + return {"error": f"Unknown project hash error: {str(err)}"} # get job result From 49efdd938b44587ae911d46ad4d36d7aa6756906 Mon Sep 17 00:00:00 2001 From: EmmaLRussell Date: Fri, 16 Feb 2024 16:22:28 +0000 Subject: [PATCH 2/6] include job id in error info --- beebop/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beebop/app.py b/beebop/app.py index 71be4220..6139e2fb 100644 --- a/beebop/app.py +++ b/beebop/app.py @@ -305,7 +305,7 @@ def get_status_job(job, p_hash, redis): "microreact": status_microreact, "network": status_network} except AttributeError as err: - return {"error": f"Unknown project hash error: {str(err)}"} + return {"error": f"Unknown project hash error (for job {job}): {str(err)}"} # get job result From cc8f25b2b8c9d09d2c3f822cbf153faec56fdd7f Mon Sep 17 00:00:00 2001 From: EmmaLRussell Date: Fri, 16 Feb 2024 16:28:35 +0000 Subject: [PATCH 3/6] message fix --- beebop/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beebop/app.py b/beebop/app.py index 6139e2fb..4a174ab0 100644 --- a/beebop/app.py +++ b/beebop/app.py @@ -305,7 +305,7 @@ def get_status_job(job, p_hash, redis): "microreact": status_microreact, "network": status_network} except AttributeError as err: - return {"error": f"Unknown project hash error (for job {job}): {str(err)}"} + return {"error": f"Unknown project hash error (for hash {p_hash}): {str(err)}"} # get job result From 75c6f315e93e57ea9b3138204c7f17ed34b36296 Mon Sep 17 00:00:00 2001 From: EmmaLRussell Date: Mon, 19 Feb 2024 09:42:28 +0000 Subject: [PATCH 4/6] print job info to console when run poppunk --- beebop/app.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/beebop/app.py b/beebop/app.py index 4a174ab0..8dffe7f5 100644 --- a/beebop/app.py +++ b/beebop/app.py @@ -225,6 +225,9 @@ def run_poppunk_internal(sketches: dict, job_timeout=job_timeout) # save p-hash with job.id in redis server redis.hset("beebop:hash:job:assign", p_hash, job_assign.id) + # check we've actually saved the value + test_value = redis.hget("beebop:hash:job:assign", p_hash) + print(f"Read test_value {test_value} for job id {job_assign.id}") # create visualisations # network job_network = q.enqueue(visualise.network, From 10d3bc2d4c827c64c216d41685a2294f97c3dd15 Mon Sep 17 00:00:00 2001 From: EmmaLRussell Date: Mon, 19 Feb 2024 11:16:10 +0000 Subject: [PATCH 5/6] more debug --- beebop/app.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/beebop/app.py b/beebop/app.py index 8dffe7f5..2a33a6da 100644 --- a/beebop/app.py +++ b/beebop/app.py @@ -165,6 +165,7 @@ def run_poppunk() -> json: :return json: [response object with all job IDs stored in 'data'] """ + print("running poppunk") sketches = request.json['sketches'].items() p_hash = request.json['projectHash'] name_mapping = request.json['names'] @@ -194,12 +195,16 @@ def run_poppunk_internal(sketches: dict, :param q: [redis queue] :return json: [response object with all job IDs stored in 'data'] """ + print("running poppunk internal") # create FS fs = PoppunkFileStore(storage_location) + print("created fs") # read arguments args = get_args() + print("got args") # set database paths db_paths = DatabaseFileStore(database_location) + print("set db paths") # store json sketches in storage, and store an initial output_cluster file # to record sample hashes for the project hashes_list = [] @@ -213,8 +218,10 @@ def run_poppunk_internal(sketches: dict, fs.ensure_output_dir_exists(p_hash) with open(fs.output_cluster(p_hash), 'wb') as f: pickle.dump(initial_output, f) + print("dumped sketches") # check connection to redis check_connection(redis) + print("checked redis") # submit list of hashes to redis worker job_assign = q.enqueue(assignClusters.get_clusters, hashes_list, From 6923ae04b992cc70ca2c98236b426e90c6e2a217 Mon Sep 17 00:00:00 2001 From: EmmaLRussell Date: Mon, 19 Feb 2024 14:03:20 +0000 Subject: [PATCH 6/6] more logging --- beebop/app.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/beebop/app.py b/beebop/app.py index 2a33a6da..526205a4 100644 --- a/beebop/app.py +++ b/beebop/app.py @@ -165,7 +165,7 @@ def run_poppunk() -> json: :return json: [response object with all job IDs stored in 'data'] """ - print("running poppunk") + logit("running poppunk") sketches = request.json['sketches'].items() p_hash = request.json['projectHash'] name_mapping = request.json['names'] @@ -173,6 +173,8 @@ def run_poppunk() -> json: return run_poppunk_internal(sketches, p_hash, name_mapping, storage_location, redis, q) +def logit(s: str): + print(s, flush=True) def run_poppunk_internal(sketches: dict, p_hash: str, @@ -195,16 +197,16 @@ def run_poppunk_internal(sketches: dict, :param q: [redis queue] :return json: [response object with all job IDs stored in 'data'] """ - print("running poppunk internal") + logit("running poppunk internal") # create FS fs = PoppunkFileStore(storage_location) - print("created fs") + logit("created fs") # read arguments args = get_args() - print("got args") + logit("got args") # set database paths db_paths = DatabaseFileStore(database_location) - print("set db paths") + logit("set db paths") # store json sketches in storage, and store an initial output_cluster file # to record sample hashes for the project hashes_list = [] @@ -218,10 +220,10 @@ def run_poppunk_internal(sketches: dict, fs.ensure_output_dir_exists(p_hash) with open(fs.output_cluster(p_hash), 'wb') as f: pickle.dump(initial_output, f) - print("dumped sketches") + logit("dumped sketches") # check connection to redis check_connection(redis) - print("checked redis") + logit("checked redis") # submit list of hashes to redis worker job_assign = q.enqueue(assignClusters.get_clusters, hashes_list, @@ -234,7 +236,7 @@ def run_poppunk_internal(sketches: dict, redis.hset("beebop:hash:job:assign", p_hash, job_assign.id) # check we've actually saved the value test_value = redis.hget("beebop:hash:job:assign", p_hash) - print(f"Read test_value {test_value} for job id {job_assign.id}") + logit(f"Read test_value {test_value} for job id {job_assign.id}") # create visualisations # network job_network = q.enqueue(visualise.network,