Bacpop-205 Fix network

.github/workflows/build_and_push.yml

@@ -21,6 +21,6 @@ jobs:
           password: ${{ secrets.GITHUB_TOKEN }}
       # add --with-dev to below commands to build & push the dev image
       - name: Build docker image
-        run: ./docker/build 
+        run: ./docker/build --with-dev # TODO: remove --with-dev before merge
       - name: Push docker image
-        run: ./docker/push 
+        run: ./docker/push --with-dev # TODO: remove --with-dev before merge

.github/workflows/pytest.yml

@@ -89,8 +89,8 @@ jobs:
         sudo apt-get install -y '^libxcb.*-dev' libx11-xcb-dev libglu1-mesa-dev libxrender-dev libxi-dev libxkbcommon-dev libxkbcommon-x11-dev
         pip install joblib==1.1.0
         conda install -c bioconda pp-sketchlib=2.0.0
-        pip3 install git+https://github.com/bacpop/PopPUNK@v2.7.2#egg=PopPUNK 
-
+        pip3 install git+https://github.com/bacpop/PopPUNK@network_relabelling#egg=PopPUNK
+# TODO revert above branch before merge
     - name: Install Poetry
       uses: snok/install-poetry@v1
       with:

beebop/app.py

@@ -410,11 +410,11 @@ def get_network_graphs(p_hash) -> json:
     """
     fs = PoppunkFileStore(storage_location)
     try:
-        cluster_result = get_clusters_internal(p_hash, storage_location)
+        cluster_result = get_cluster_assignments(p_hash, storage_location)
         graphmls = {}
         for cluster_info in cluster_result.values():
             cluster = cluster_info["cluster"]
-            path = fs.network_output_component(
+            path = fs.pruned_network_output_component(
                 p_hash, get_cluster_num(cluster)
             )
             with open(path, "r") as graphml_file:
@@ -494,9 +494,9 @@ def get_results(result_type) -> json:
                                                 storage_location)
 
 
-def get_clusters_internal(p_hash: str, storage_location: str) -> dict:
+def get_cluster_assignments(p_hash: str, storage_location: str) -> dict:
     """
-    [returns cluster assignment results ]
+    [returns cluster assignment results]
 
     :param p_hash: [project hash]
     :param storage_location: [storage location]
@@ -516,7 +516,7 @@ def get_clusters_json(p_hash: str, storage_location: str) -> json:
     :param storage_location: [storage location]
     :return json: [response object with cluster results stored in 'data']
     """
-    cluster_result = get_clusters_internal(p_hash, storage_location)
+    cluster_result = get_cluster_assignments(p_hash, storage_location)
     cluster_dict = {value['hash']: value for value in cluster_result.values()}
     failed_samples = get_failed_samples_internal(p_hash, storage_location)
 
@@ -620,7 +620,7 @@ def get_project(p_hash) -> json:
     if "error" in status:
         return jsonify(error=response_failure(status)), 500
     else:
-        clusters_result = get_clusters_internal(p_hash, storage_location)
+        clusters_result = get_cluster_assignments(p_hash, storage_location)
         failed_samples = get_failed_samples_internal(p_hash, storage_location)
 
         fs = PoppunkFileStore(storage_location)

beebop/assignClusters.py

@@ -208,7 +208,7 @@ def handle_external_clusters(
                 sketches_dict,
                 not_found_query_names,
                 output_full_tmp,
-                not_found_query_clusters
+                not_found_query_clusters,
             )
         )
         queries_names.extend(not_found_query_names_new)
@@ -269,13 +269,11 @@ def handle_not_found_queries(
     assign_query_clusters(
         config, config.full_db_fs, not_found_query_names, output_full_tmp
     )
-    query_names, query_clusters, _, _, _, _, _ = (
-        summarise_clusters(
-            output_full_tmp,
-            config.species,
-            config.full_db_fs.db,
-            not_found_query_names,
-        )
+    query_names, query_clusters, _, _, _, _, _ = summarise_clusters(
+        output_full_tmp,
+        config.species,
+        config.full_db_fs.db,
+        not_found_query_names,
     )
 
     handle_files_manipulation(
@@ -347,15 +345,17 @@ def update_external_clusters(
     not_found_prev_querying = config.fs.external_previous_query_clustering_tmp(
         config.p_hash
     )
-    external_clusters_not_found, _ = get_external_clusters_from_file(
+
+    update_external_clusters_csv(
+        previous_query_clustering,
         not_found_prev_querying,
         not_found_query_names,
-        config.external_clusters_prefix,
     )
-    update_external_clusters_csv(
-        previous_query_clustering,
+
+    external_clusters_not_found, _ = get_external_clusters_from_file(
+        not_found_prev_querying,
         not_found_query_names,
-        external_clusters_not_found,
+        config.external_clusters_prefix,
     )
     external_clusters.update(external_clusters_not_found)
 
@@ -399,9 +399,7 @@ def copy_include_files(output_full_tmp: str, outdir: str) -> None:
             merge_txt_files(dest_file, source_file)
             os.remove(source_file)
         else:
-            os.rename(
-                source_file, dest_file
-            )
+            os.rename(source_file, dest_file)
 
 
 def filter_queries(

beebop/filestore.py

@@ -230,6 +230,24 @@ def network_output_component(self, p_hash, component_number) -> str:
             )
         )
 
+    def pruned_network_output_component(self, p_hash, component_number) -> str:
+        """
+        [Generates the path to the pruned network component file
+        for the given project hash and component number.]
+
+        :param p_hash: [project hash]
+        :param component_number: [component number,
+            which is the same as cluster number]
+        :return str: [path to pruned network component file]
+        """
+        return str(
+            PurePath(
+                self.output(p_hash),
+                "network",
+                f"pruned_network_component_{component_number}.graphml",
+            )
+        )
+
     def tmp(self, p_hash) -> str:
         """
         :param p_hash: [project hash]

beebop/poppunkWrapper.py

@@ -108,7 +108,7 @@ def create_microreact(self, cluster: str, internal_cluster: str) -> None:
             ),
             previous_mst=None,
             previous_distances=None,
-            network_file=self.fs.network_file(self.p_hash),
+            network_file=None,
             gpu_graph=self.args.visualise.gpu_graph,
             info_csv=self.args.visualise.info_csv,
             rapidnj=shutil.which("rapidnj"),
@@ -120,6 +120,7 @@ def create_microreact(self, cluster: str, internal_cluster: str) -> None:
             recalculate_distances=self.args.visualise.recalculate_distances,
             use_partial_query_graph=self.fs.partial_query_graph(self.p_hash),
             tmp=self.fs.tmp(self.p_hash),
+            extend_query_graph=True
         )
 
     def create_network(self) -> None:
@@ -152,7 +153,7 @@ def create_network(self) -> None:
             ),
             previous_mst=None,
             previous_distances=None,
-            network_file=self.fs.network_file(self.p_hash),
+            network_file=None,
             gpu_graph=self.args.visualise.gpu_graph,
             info_csv=self.args.visualise.info_csv,
             rapidnj=shutil.which("rapidnj"),
@@ -164,4 +165,5 @@ def create_network(self) -> None:
             recalculate_distances=self.args.visualise.recalculate_distances,
             use_partial_query_graph=self.fs.partial_query_graph(self.p_hash),
             tmp=self.fs.tmp(self.p_hash),
+            extend_query_graph=True
         )