Merge pull request #49 from bacpop/bacpop-202-fallback-to-refs

Bacpop-202 Assign fallback to refs to db
bacpop · Dec 6, 2024 · af7603d · af7603d
2 parents b93b922 + 3e9d0d9
commit af7603d
Show file tree

Hide file tree

Showing 12 changed files with 1,941 additions and 387 deletions.
diff --git a/README.md b/README.md
@@ -108,6 +108,11 @@ Testing can be done in a second terminal (make sure to activate 'beebop_py') by
 TESTING=True poetry run pytest
 ```
 
+### Diagrams
+
+- There is a .drawio graph in the `diagrams` folder illustrating the process of running a analysis. This includes
+all the files created and how they are used in each job. You can open and view the diagram at [draw.io](https://draw.io).
+
 ## Use/Deploy specific version of PopPUNK
 
 To use a specific version, commit or branch of PopPUNK in a beebop_py deployment, you can update `POPPUNK_VERSION` in `common`.

diff --git a/beebop/app.py b/beebop/app.py
@@ -4,7 +4,7 @@
 from redis import Redis
 import redis.exceptions as redis_exceptions
 from rq import Queue
-from rq.job import Job
+from rq.job import Job, Dependency
 import os
 from io import BytesIO
 import zipfile
@@ -169,7 +169,7 @@ def get_species_config() -> json:
     """
     all_species_args = vars(get_args().species)
     species_config = {
-        species: get_species_kmers(args.dbname)
+        species: get_species_kmers(args.refdb)
         for species, args in all_species_args.items()
     }
     return jsonify(response_success(species_config))
@@ -248,8 +248,13 @@ def run_poppunk_internal(sketches: dict,
             400,
         )
 
-    db_fs = DatabaseFileStore(
-        f"{dbs_location}/{species_args.dbname}",
+    # pass in both full and refs to assign
+    ref_db_fs = DatabaseFileStore(
+        f"{dbs_location}/{species_args.refdb}",
+        species_args.external_clusters_file,
+    )
+    full_db_fs = DatabaseFileStore(
+        f"{dbs_location}/{species_args.fulldb}",
         species_args.external_clusters_file,
     )
 
@@ -279,7 +284,8 @@ def run_poppunk_internal(sketches: dict,
                            hashes_list,
                            p_hash,
                            fs,
-                           db_fs,
+                           ref_db_fs,
+                           full_db_fs,
                            args,
                            species,
                            **queue_kwargs)
@@ -290,7 +296,7 @@ def run_poppunk_internal(sketches: dict,
     job_network = q.enqueue(visualise.network,
                             args=(p_hash,
                                   fs,
-                                  db_fs,
+                                  full_db_fs,
                                   args,
                                   name_mapping,
                                   species),
@@ -299,16 +305,21 @@ def run_poppunk_internal(sketches: dict,
     # microreact
     # delete all previous microreact cluster job results for this project
     redis.delete(f"beebop:hash:job:microreact:{p_hash}")
-    job_microreact = q.enqueue(visualise.microreact,
-                               args=(p_hash,
-                                     fs,
-                                     db_fs,
-                                     args,
-                                     name_mapping,
-                                     species,
-                                     redis_host,
-                                     queue_kwargs),
-                               depends_on=job_network, **queue_kwargs)
+    job_microreact = q.enqueue(
+        visualise.microreact,
+        args=(
+            p_hash,
+            fs,
+            full_db_fs,
+            args,
+            name_mapping,
+            species,
+            redis_host,
+            queue_kwargs,
+        ),
+        depends_on=Dependency([job_assign, job_network], allow_failure=True),
+        **queue_kwargs,
+    )
     redis.hset("beebop:hash:job:microreact", p_hash, job_microreact.id)
     return jsonify(
         response_success(