semseg refactors - kamangir/bolt#746

README.md

@@ -15,4 +15,4 @@ pip install roofAI
 
 [![pylint](https://github.com/kamangir/roofAI/actions/workflows/pylint.yml/badge.svg)](https://github.com/kamangir/roofAI/actions/workflows/pylint.yml) [![pytest](https://github.com/kamangir/roofAI/actions/workflows/pytest.yml/badge.svg)](https://github.com/kamangir/roofAI/actions/workflows/pytest.yml) [![bashtest](https://github.com/kamangir/roofAI/actions/workflows/bashtest.yml/badge.svg)](https://github.com/kamangir/roofAI/actions/workflows/bashtest.yml) [![PyPI version](https://img.shields.io/pypi/v/roofAI.svg)](https://pypi.org/project/roofAI/) [![PyPI - Downloads](https://img.shields.io/pypi/dd/roofAI)](https://pypistats.org/packages/roofAI)
 
-built by 🌀 [`blue_options-4.185.1`](https://github.com/kamangir/awesome-bash-cli), based on 🏛️ [`roofAI-5.22.1`](https://github.com/kamangir/roofAI).
+built by 🌀 [`blue_options-4.186.1`](https://github.com/kamangir/awesome-bash-cli), based on 🏛️ [`roofAI-5.23.1`](https://github.com/kamangir/roofAI).

roofAI/.abcli/dataset/ingest.sh

@@ -6,7 +6,7 @@ function roofAI_dataset_ingest() {
     local options=$1
 
     if [ $(abcli_option_int "$options" help 0) == 1 ]; then
-        local common_options="dryrun,open,register,suffix=<v1>,upload"
+        local common_options="dryrun,open,upload"
 
         local options="source=AIRS,$common_options,target=sagemaker|torch"
         local args="[--test_count <10>]$ABCUL[--train_count <10>]$ABCUL[--val_count <10>]"
@@ -20,9 +20,7 @@ function roofAI_dataset_ingest() {
 
     local do_dryrun=$(abcli_option_int "$options" dryrun 0)
     local do_open=$(abcli_option_int "$options" open 0)
-    local do_register=$(abcli_option_int "$options" register 0)
     local do_upload=$(abcli_option_int "$options" upload 0)
-    local suffix=$(abcli_option "$options" suffix v1)
     local source=$(abcli_option "$options" source)
     local target=$(abcli_option "$options" target torch)
 
@@ -50,13 +48,13 @@ function roofAI_dataset_ingest() {
     fi
 
     if [ "$source" == "AIRS" ]; then
-        local cache_object_name=$(abcli_cache read $cache_keyword)
+        local cache_object_name=$(abcli_mlflow_cache read $cache_keyword)
 
         local cache_from_source=0
         if [[ -z "$cache_object_name" ]]; then
             local cache_object_name=roofAI_ingest_${source}_cache_$(abcli_string_timestamp)
 
-            abcli_cache write \
+            abcli_mlflow_cache write \
                 $cache_keyword \
                 $cache_object_name
 
@@ -104,8 +102,5 @@ function roofAI_dataset_ingest() {
     [[ "$do_upload" == 1 ]] &&
         abcli_upload - $object_name
 
-    [[ "$do_register" == 1 ]] &&
-        abcli_cache write roofAI_ingest_${source}_${suffix} $object_name
-
     return 0
 }

roofAI/.abcli/semseg.sh

@@ -10,26 +10,7 @@ function roofAI_semseg() {
     local device=$(abcli_option "$options" device $device)
     local do_dryrun=$(abcli_option_int "$options" dryrun 0)
     local do_download=$(abcli_option_int "$options" download $(abcli_not $do_dryrun))
-    local do_register=$(abcli_option_int "$options" register 0)
-    local do_upload=$(abcli_option_int "$options" upload $do_register)
-
-    if [ "$task" == "list" ]; then
-        if [ $(abcli_option_int "$options" help 0) == 1 ]; then
-            abcli_show_usage "semseg list" \
-                "list registered semseg models."
-            return
-        fi
-
-        local reference
-        for reference in $(abcli_tags search \
-            registered_semseg_model \
-            --delim space \
-            --log 0); do
-            abcli_log "⚡️ $reference: $(abcli_cache read $reference)"
-        done
-
-        return
-    fi
+    local do_upload=$(abcli_option_int "$options" upload $(abcli_not $do_dryrun))
 
     if [ "$task" == "predict" ]; then
         if [ $(abcli_option_int "$options" help 0) == 1 ]; then
@@ -69,7 +50,7 @@ function roofAI_semseg() {
 
     if [ "$task" == "train" ]; then
         if [ $(abcli_option_int "$options" help 0) == 1 ]; then
-            local options="device=cpu|cuda,~download,dryrun,profile=$semseg_profiles,register,suffix=<v1>,upload"
+            local options="device=cpu|cuda,~download,dryrun,profile=$semseg_profiles,upload"
             local args="[--activation <sigmoid>]$ABCUL[--classes <one+two+three+four>]$ABCUL[--encoder_name <se_resnext50_32x4d>]$ABCUL[--encoder_weights <imagenet>]"
             abcli_show_usage "semseg train$ABCUL[$options]$ABCUL<dataset-object-name>$ABCUL<model-object-name>$ABCUL$args" \
                 "semseg.train(<dataset-object-name>) -> <model-object-name>."
@@ -90,8 +71,6 @@ function roofAI_semseg() {
             --dataset_path $ABCLI_OBJECT_ROOT/$dataset_object_name \
             --model_path $ABCLI_OBJECT_ROOT/$model_object_name \
             --profile $(abcli_option "$options" profile VALIDATION) \
-            --register $do_register \
-            --suffix $(abcli_option "$options" suffix v1) \
             "${@:5}"
         local status="$?"
 

roofAI/__init__.py

@@ -4,8 +4,12 @@
 
 DESCRIPTION = f"{ICON} everything AI about roofs."
 
-VERSION = "5.22.1"
+VERSION = "5.23.1"
 
 MARQUEE = "https://github.com/kamangir/assets/blob/main/2023-10-28-16-28-36-88493-predict.gif?raw=true"
 
 REPO_NAME = NAME
+
+
+def fullname() -> str:
+    return f"{NAME}-{VERSION}"

roofAI/dataset/README.md

@@ -33,7 +33,7 @@ roof dataset review open \
 
 ```bash
 roofAI dataset ingest \
-    source=AIRS,register \
+    source=AIRS \
     - \
     --test_count 250 \
     --train_count 350 \
@@ -88,5 +88,5 @@ roof dataset review open \
 
 ```bash
 roofAI dataset ingest \
-    source=CamVid,register
+    source=CamVid
 ```

roofAI/host.py

@@ -0,0 +1,9 @@
+from typing import List
+
+from abcli.host import signature as abcli_signature
+
+from roofAI import fullname
+
+
+def signature() -> List[str]:
+    return [fullname()] + abcli_signature()

roofAI/semseg/README.md

@@ -6,7 +6,7 @@ A Semantic Segmenter based on [segmentation_models.pytorch](<https://github.com/
 
 ```bash
 roofAI semseg train \
-    profile=FULL,register,suffix=full_v2 \
+    profile=FULL \
     $(@ref roofAI_ingest_AIRS_v2) \
     $(@timestamp) \
     --classes roof

roofAI/semseg/__main__.py

@@ -67,18 +67,6 @@
     default="VALIDATION",
     help="FULL|QUICK|VALIDATION",
 )
-parser.add_argument(
-    "--register",
-    type=int,
-    default=0,
-    help="0|1",
-)
-parser.add_argument(
-    "--suffix",
-    type=str,
-    default="",
-    help="suffix to register the model",
-)
 args = parser.parse_args()
 
 success = True
@@ -111,8 +99,6 @@
             classes=args.classes.split("+"),
             activation=args.activation,
             device=args.device,
-            do_register=bool(args.register),
-            suffix=args.suffix,
             profile=profile,
         )
     else:

roofAI/semseg/interface.py

@@ -35,11 +35,9 @@ def train(
     encoder_weights="imagenet",
     activation="sigmoid",  # could be None for logits or 'softmax2d' for multi-class segmentation
     device="cpu",  # 'cuda'
-    do_register: bool = False,
-    suffix: str = "v1",
     profile: Profile = Profile.VALIDATION,
     in_notebook: bool = False,
-) -> Tuple[bool, Any]:
+) -> SemSegModel:
     trainer = SemSegModelTrainer(
         dataset_path=dataset_path,
         model_path=model_path,
@@ -54,8 +52,6 @@ def train(
         encoder_weights=encoder_weights,
         activation=activation,
         device=device,
-        do_register=do_register,
-        suffix=suffix,
         in_notebook=in_notebook,
     )
 

roofAI/semseg/train.py

@@ -1,20 +1,15 @@
-"""
-Copied with modification from ../../notebooks/semseg.ipynb
-"""
-
 import os
 import matplotlib.pyplot as plt
 import time
 from typing import List
 import torch
 from torch.utils.data import DataLoader
+import segmentation_models_pytorch as smp
+from segmentation_models_pytorch import utils
 
 from blue_options import string
 from blue_objects import file, path
-from blue_objects.mysql import cache, tags
 
-import segmentation_models_pytorch as smp
-from segmentation_models_pytorch import utils
 from roofAI.semseg.augmentation import (
     get_training_augmentation,
     get_validation_augmentation,
@@ -110,14 +105,11 @@ def train(
         encoder_weights="imagenet",
         activation="sigmoid",  # could be None for logits or 'softmax2d' for multi-class segmentation
         device="cpu",  # 'cuda'
-        do_register: bool = False,
-        suffix: str = "v1",
         in_notebook: bool = False,
     ):
         logger.info(
-            "{}.train{} -{}:{}-> {}[{}]: {}".format(
+            "{}.train{} -{}:{}-> {}[{}]".format(
                 self.__class__.__name__,
-                f"[{suffix}]" if suffix else "",
                 device,
                 activation,
                 encoder_name,
@@ -337,13 +329,4 @@ def train(
 
         # TODO: semseg_model.predict(...)
 
-        if do_register:
-            keyword = f"roofAI_semseg_model_{self.dataset.source}_{suffix}"
-            cache.write(
-                keyword,
-                path.name(self.model_path),
-            )
-
-            tags.set_(keyword, "registered_semseg_model")
-
         return semseg_model

roofAI/tests/test_host.py

@@ -0,0 +1,5 @@
+from roofAI.host import signature
+
+
+def test_signature():
+    assert signature()

roofAI/tests/test_semseg_train.py

@@ -34,8 +34,6 @@ def test_semseg_train(dataset_source, classes):
             dataset_path=objects.object_path(dataset_object_name),
             model_path=objects.object_path(objects.unique_object()),
             classes=classes,
-            do_register=True,
-            suffix=prefix,
         ),
         SemSegModel,
     )