From d98b5ee7e3be2734ed834ead5bed6450ffc7d621 Mon Sep 17 00:00:00 2001 From: Martin Kozlovsky Date: Wed, 17 Jan 2024 16:47:15 +0100 Subject: [PATCH] cleaned up config, renamed some fields --- configs/README.md | 18 ++--- configs/classification_model.yaml | 2 +- configs/coco_model.yaml | 2 +- configs/detection_model.yaml | 2 +- configs/example_export.yaml | 2 +- configs/example_tuning.yaml | 2 +- configs/keypoint_bbox_model.yaml | 2 +- configs/segmentation_model.yaml | 2 +- luxonis_train/__main__.py | 4 +- .../callbacks/export_on_train_end.py | 2 +- luxonis_train/callbacks/metadata_logger.py | 3 +- luxonis_train/callbacks/test_on_train_end.py | 28 ++++---- luxonis_train/core/core.py | 4 +- luxonis_train/core/tuner.py | 17 +++-- luxonis_train/models/luxonis_model.py | 4 +- .../predefined_models/classification_model.py | 12 ++-- .../predefined_models/detection_model.py | 12 ++-- .../keypoint_detection_model.py | 12 ++-- .../predefined_models/segmentation_model.py | 12 ++-- luxonis_train/nodes/README.md | 8 ++- luxonis_train/utils/config.py | 69 +++++++------------ pyproject.toml | 2 +- tools/main.py | 4 +- 23 files changed, 105 insertions(+), 120 deletions(-) diff --git a/configs/README.md b/configs/README.md index 25ebd5bc..27e2fb6e 100644 --- a/configs/README.md +++ b/configs/README.md @@ -63,7 +63,7 @@ For list of all nodes, see [nodes](../luxonis_train/nodes/README.md). | Key | Type | Default value | Description | | ----------------------- | -------------------- | ------------- | ------------------------------------------------------------------------------------------------------------------------------------------- | | name | str | | name of the node | -| override_name | str | None | custom name for the node | +| alias | str | None | custom name for the node | | params | dict | {} | parameters for the node | | inputs | list | \[\] | list of input nodes for this node, if empty, the node is understood to be an input node of the model | | freezing.active | bool | False | whether to freeze the modules so the weights are not updated | @@ -73,12 +73,12 @@ For list of all nodes, see [nodes](../luxonis_train/nodes/README.md). Modules that are attached to a node. This include losses, metrics and visualziers. -| Key | Type | Default value | Description | -| ------------- | ---- | ------------- | ------------------------------------------- | -| name | str | | name of the module | -| attached_to | str | | Name of the node the module is attached to. | -| override_name | str | None | custom name for the module | -| params | dict | {} | parameters of the module | +| Key | Type | Default value | Description | +| ----------- | ---- | ------------- | ------------------------------------------- | +| name | str | | name of the module | +| attached_to | str | | Name of the node the module is attached to. | +| alias | str | None | custom name for the module | +| params | dict | {} | parameters of the module | #### Losses @@ -128,9 +128,9 @@ To store and load the data we use LuxonisDataset and LuxonisLoader. For specific | Key | Type | Default value | Description | | -------------- | ---------------------------------------- | ------------------- | ---------------------------------------------- | -| dataset_name | str \| None | None | name of the dataset | +| name | str \| None | None | name of the dataset | +| id | str \| None | None | id of the dataset | | team_id | str \| None | None | team under which you can find all datasets | -| dataset_id | str \| None | None | id of the dataset | | bucket_type | Literal\["intenal", "external"\] | internal | type of underlying storage | | bucket_storage | Literal\["local", "s3", "gcc", "azure"\] | BucketStorage.LOCAL | underlying object storage for a bucket | | train_view | str | train | view to use for training | diff --git a/configs/classification_model.yaml b/configs/classification_model.yaml index 205ee53d..62c1014e 100755 --- a/configs/classification_model.yaml +++ b/configs/classification_model.yaml @@ -16,7 +16,7 @@ model: include_plot: True dataset: - dataset_name: cifar10_test + name: cifar10_test trainer: preprocessing: diff --git a/configs/coco_model.yaml b/configs/coco_model.yaml index 86e65611..491152ce 100755 --- a/configs/coco_model.yaml +++ b/configs/coco_model.yaml @@ -96,7 +96,7 @@ tracker: is_mlflow: False dataset: - dataset_name: coco_test + name: coco_test train_view: train val_view: val test_view: test diff --git a/configs/detection_model.yaml b/configs/detection_model.yaml index f17567c6..8d7f9c25 100755 --- a/configs/detection_model.yaml +++ b/configs/detection_model.yaml @@ -11,7 +11,7 @@ model: use_neck: True dataset: - dataset_name: coco_test + name: coco_test trainer: preprocessing: diff --git a/configs/example_export.yaml b/configs/example_export.yaml index a35ca148..a999a2bd 100755 --- a/configs/example_export.yaml +++ b/configs/example_export.yaml @@ -13,7 +13,7 @@ model: task: binary dataset: - dataset_name: coco_test + name: coco_test trainer: preprocessing: diff --git a/configs/example_tuning.yaml b/configs/example_tuning.yaml index 3ef75221..980036ae 100755 --- a/configs/example_tuning.yaml +++ b/configs/example_tuning.yaml @@ -12,7 +12,7 @@ model: task: binary dataset: - dataset_name: coco_test + name: coco_test trainer: preprocessing: diff --git a/configs/keypoint_bbox_model.yaml b/configs/keypoint_bbox_model.yaml index acf28f07..dc4fe3d7 100755 --- a/configs/keypoint_bbox_model.yaml +++ b/configs/keypoint_bbox_model.yaml @@ -9,7 +9,7 @@ model: name: KeypointDetectionModel dataset: - dataset_name: coco_test + name: coco_test trainer: preprocessing: diff --git a/configs/segmentation_model.yaml b/configs/segmentation_model.yaml index d9d0f50b..c26fb0cc 100755 --- a/configs/segmentation_model.yaml +++ b/configs/segmentation_model.yaml @@ -12,7 +12,7 @@ model: task: binary dataset: - dataset_name: coco_test + name: coco_test trainer: preprocessing: diff --git a/luxonis_train/__main__.py b/luxonis_train/__main__.py index f1f7891e..73843593 100644 --- a/luxonis_train/__main__.py +++ b/luxonis_train/__main__.py @@ -126,9 +126,9 @@ def inspect( image_size = cfg.trainer.preprocessing.train_image_size dataset = LuxonisDataset( - dataset_name=cfg.dataset.dataset_name, + dataset_name=cfg.dataset.name, team_id=cfg.dataset.team_id, - dataset_id=cfg.dataset.dataset_id, + dataset_id=cfg.dataset.id, bucket_type=cfg.dataset.bucket_type, bucket_storage=cfg.dataset.bucket_storage, ) diff --git a/luxonis_train/callbacks/export_on_train_end.py b/luxonis_train/callbacks/export_on_train_end.py index 3aa55309..de5fde88 100644 --- a/luxonis_train/callbacks/export_on_train_end.py +++ b/luxonis_train/callbacks/export_on_train_end.py @@ -49,7 +49,7 @@ def on_train_end(self, trainer: pl.Trainer, pl_module: pl.LightningModule) -> No cfg: Config = pl_module.cfg cfg.model.weights = best_model_path if self.upload_to_mlflow: - if pl_module.cfg.tracker.is_mlflow: + if cfg.tracker.is_mlflow: tracker = cast(LuxonisTrackerPL, trainer.logger) new_upload_directory = f"mlflow://{tracker.project_id}/{tracker.run_id}" cfg.exporter.upload_directory = new_upload_directory diff --git a/luxonis_train/callbacks/metadata_logger.py b/luxonis_train/callbacks/metadata_logger.py index e36c0c30..5ccf542f 100644 --- a/luxonis_train/callbacks/metadata_logger.py +++ b/luxonis_train/callbacks/metadata_logger.py @@ -5,6 +5,7 @@ import pkg_resources import yaml +from luxonis_train.utils.config import Config from luxonis_train.utils.registry import CALLBACKS @@ -23,7 +24,7 @@ def __init__(self, hyperparams: list[str]): self.hyperparams = hyperparams def on_fit_start(self, trainer: pl.Trainer, pl_module: pl.LightningModule) -> None: - cfg = pl_module.cfg + cfg: Config = pl_module.cfg hparams = {key: cfg.get(key) for key in self.hyperparams} diff --git a/luxonis_train/callbacks/test_on_train_end.py b/luxonis_train/callbacks/test_on_train_end.py index 6bd3c324..8cf23e3c 100644 --- a/luxonis_train/callbacks/test_on_train_end.py +++ b/luxonis_train/callbacks/test_on_train_end.py @@ -2,6 +2,7 @@ from luxonis_ml.data import LuxonisDataset, ValAugmentations from torch.utils.data import DataLoader +from luxonis_train.utils.config import Config from luxonis_train.utils.loaders import LuxonisLoaderTorch, collate_fn from luxonis_train.utils.registry import CALLBACKS @@ -11,31 +12,32 @@ class TestOnTrainEnd(pl.Callback): """Callback to perform a test run at the end of the training.""" def on_train_end(self, trainer: pl.Trainer, pl_module: pl.LightningModule) -> None: + cfg: Config = pl_module.cfg + dataset = LuxonisDataset( - dataset_name=pl_module.cfg.dataset.dataset_name, - team_id=pl_module.cfg.dataset.team_id, - dataset_id=pl_module.cfg.dataset.dataset_id, - bucket_type=pl_module.cfg.dataset.bucket_type, - bucket_storage=pl_module.cfg.dataset.bucket_storage, + dataset_name=cfg.dataset.name, + team_id=cfg.dataset.team_id, + dataset_id=cfg.dataset.id, + bucket_type=cfg.dataset.bucket_type, + bucket_storage=cfg.dataset.bucket_storage, ) loader_test = LuxonisLoaderTorch( dataset, - view=pl_module.cfg.dataset.test_view, + view=cfg.dataset.test_view, augmentations=ValAugmentations( - image_size=pl_module.cfg.trainer.preprocessing.train_image_size, + image_size=cfg.trainer.preprocessing.train_image_size, augmentations=[ - i.model_dump() - for i in pl_module.cfg.trainer.preprocessing.augmentations + i.model_dump() for i in cfg.trainer.preprocessing.augmentations ], - train_rgb=pl_module.cfg.trainer.preprocessing.train_rgb, - keep_aspect_ratio=pl_module.cfg.trainer.preprocessing.keep_aspect_ratio, + train_rgb=cfg.trainer.preprocessing.train_rgb, + keep_aspect_ratio=cfg.trainer.preprocessing.keep_aspect_ratio, ), ) pytorch_loader_test = DataLoader( loader_test, - batch_size=pl_module.cfg.trainer.batch_size, - num_workers=pl_module.cfg.trainer.num_workers, + batch_size=cfg.trainer.batch_size, + num_workers=cfg.trainer.num_workers, collate_fn=collate_fn, ) trainer.test(pl_module, pytorch_loader_test) diff --git a/luxonis_train/core/core.py b/luxonis_train/core/core.py index de17be0d..75bd1d2a 100644 --- a/luxonis_train/core/core.py +++ b/luxonis_train/core/core.py @@ -122,9 +122,9 @@ def __init__( callbacks=LuxonisProgressBar() if self.cfg.use_rich_text else None, ) self.dataset = LuxonisDataset( - dataset_name=self.cfg.dataset.dataset_name, + dataset_name=self.cfg.dataset.name, team_id=self.cfg.dataset.team_id, - dataset_id=self.cfg.dataset.dataset_id, + dataset_id=self.cfg.dataset.id, bucket_type=self.cfg.dataset.bucket_type, bucket_storage=self.cfg.dataset.bucket_storage, ) diff --git a/luxonis_train/core/tuner.py b/luxonis_train/core/tuner.py index d86efac4..c9f8e151 100644 --- a/luxonis_train/core/tuner.py +++ b/luxonis_train/core/tuner.py @@ -26,19 +26,22 @@ def __init__(self, cfg: str | dict, args: list[str] | tuple[str, ...] | None): used for config overriding. """ super().__init__(cfg, args) + if self.cfg.tuner is None: + raise ValueError("You have to specify the `tuner` section in config.") + self.tune_cfg = self.cfg.tuner def tune(self) -> None: """Runs Optuna tunning of hyperparameters.""" pruner = ( optuna.pruners.MedianPruner() - if self.cfg.tuner.use_pruner + if self.tune_cfg.use_pruner else optuna.pruners.NopPruner() ) storage = None - if self.cfg.tuner.storage.active: - if self.cfg.tuner.storage.storage_type == "local": + if self.tune_cfg.storage.active: + if self.tune_cfg.storage.storage_type == "local": storage = "sqlite:///study_local.db" else: storage = "postgresql://{}:{}@{}:{}/{}".format( @@ -50,7 +53,7 @@ def tune(self) -> None: ) study = optuna.create_study( - study_name=self.cfg.tuner.study_name, + study_name=self.tune_cfg.study_name, storage=storage, direction="minimize", pruner=pruner, @@ -59,8 +62,8 @@ def tune(self) -> None: study.optimize( self._objective, - n_trials=self.cfg.tuner.n_trials, - timeout=self.cfg.tuner.timeout, + n_trials=self.tune_cfg.n_trials, + timeout=self.tune_cfg.timeout, ) def _objective(self, trial: optuna.trial.Trial) -> float: @@ -128,7 +131,7 @@ def _objective(self, trial: optuna.trial.Trial) -> float: def _get_trial_params(self, trial: optuna.trial.Trial) -> dict[str, Any]: """Get trial params based on specified config.""" - cfg_tuner = self.cfg.tuner.params + cfg_tuner = self.tune_cfg.params new_params = {} for key, value in cfg_tuner.items(): key_info = key.split("_") diff --git a/luxonis_train/models/luxonis_model.py b/luxonis_train/models/luxonis_model.py index 77dc3dca..80a57d99 100644 --- a/luxonis_train/models/luxonis_model.py +++ b/luxonis_train/models/luxonis_model.py @@ -142,7 +142,7 @@ def __init__( for node_cfg in self.cfg.model.nodes: node_name = node_cfg.name Node = BaseNode.REGISTRY.get(node_name) - node_name = node_cfg.override_name or node_name + node_name = node_cfg.alias or node_name if node_cfg.freezing.active: epochs = self.cfg.trainer.epochs if node_cfg.freezing.unfreeze_after is None: @@ -714,7 +714,7 @@ def _init_attached_module( storage: Mapping[str, Mapping[str, BaseAttachedModule]], ) -> tuple[str, str]: Module = registry.get(cfg.name) - module_name = cfg.override_name or cfg.name + module_name = cfg.alias or cfg.name node_name = cfg.attached_to module = Module(**cfg.params, node=self.nodes[node_name]) storage[node_name][module_name] = module # type: ignore diff --git a/luxonis_train/models/predefined_models/classification_model.py b/luxonis_train/models/predefined_models/classification_model.py index 2a88a393..33e56242 100644 --- a/luxonis_train/models/predefined_models/classification_model.py +++ b/luxonis_train/models/predefined_models/classification_model.py @@ -26,13 +26,13 @@ def nodes(self) -> list[ModelNodeConfig]: return [ ModelNodeConfig( name=self.backbone, - override_name="classification_backbone", + alias="classification_backbone", freezing=self.backbone_params.pop("freezing", {}), params=self.backbone_params, ), ModelNodeConfig( name="ClassificationHead", - override_name="classification_head", + alias="classification_head", inputs=["classification_backbone"], freezing=self.head_params.pop("freezing", {}), params=self.head_params, @@ -44,7 +44,7 @@ def losses(self) -> list[LossModuleConfig]: return [ LossModuleConfig( name="CrossEntropyLoss", - override_name="classification_loss", + alias="classification_loss", attached_to="classification_head", params=self.loss_params, weight=1.0, @@ -56,20 +56,20 @@ def metrics(self) -> list[MetricModuleConfig]: return [ MetricModuleConfig( name="F1Score", - override_name="classification_f1_score", + alias="classification_f1_score", is_main_metric=True, attached_to="classification_head", params={"task": self.task}, ), MetricModuleConfig( name="Accuracy", - override_name="classification_accuracy", + alias="classification_accuracy", attached_to="classification_head", params={"task": self.task}, ), MetricModuleConfig( name="Recall", - override_name="classification_recall", + alias="classification_recall", attached_to="classification_head", params={"task": self.task}, ), diff --git a/luxonis_train/models/predefined_models/detection_model.py b/luxonis_train/models/predefined_models/detection_model.py index 3be80082..41a7dfdc 100644 --- a/luxonis_train/models/predefined_models/detection_model.py +++ b/luxonis_train/models/predefined_models/detection_model.py @@ -25,7 +25,7 @@ def nodes(self) -> list[ModelNodeConfig]: nodes = [ ModelNodeConfig( name="EfficientRep", - override_name="detection_backbone", + alias="detection_backbone", freezing=self.backbone_params.pop("freezing", {}), params=self.backbone_params, ), @@ -34,7 +34,7 @@ def nodes(self) -> list[ModelNodeConfig]: nodes.append( ModelNodeConfig( name="RepPANNeck", - override_name="detection_neck", + alias="detection_neck", inputs=["detection_backbone"], freezing=self.neck_params.pop("freezing", {}), params=self.neck_params, @@ -44,7 +44,7 @@ def nodes(self) -> list[ModelNodeConfig]: nodes.append( ModelNodeConfig( name="EfficientBBoxHead", - override_name="detection_head", + alias="detection_head", freezing=self.head_params.pop("freezing", {}), inputs=["detection_neck"] if self.use_neck else ["detection_backbone"], params=self.head_params, @@ -57,7 +57,7 @@ def losses(self) -> list[LossModuleConfig]: return [ LossModuleConfig( name="AdaptiveDetectionLoss", - override_name="detection_loss", + alias="detection_loss", attached_to="detection_head", params=self.loss_params, weight=1.0, @@ -69,7 +69,7 @@ def metrics(self) -> list[MetricModuleConfig]: return [ MetricModuleConfig( name="MeanAveragePrecision", - override_name="detection_map", + alias="detection_map", attached_to="detection_head", is_main_metric=True, ), @@ -80,7 +80,7 @@ def visualizers(self) -> list[AttachedModuleConfig]: return [ AttachedModuleConfig( name="BBoxVisualizer", - override_name="detection_visualizer", + alias="detection_visualizer", attached_to="detection_head", params=self.visualizer_params, ) diff --git a/luxonis_train/models/predefined_models/keypoint_detection_model.py b/luxonis_train/models/predefined_models/keypoint_detection_model.py index a123f51b..96bef885 100644 --- a/luxonis_train/models/predefined_models/keypoint_detection_model.py +++ b/luxonis_train/models/predefined_models/keypoint_detection_model.py @@ -26,7 +26,7 @@ def nodes(self) -> list[ModelNodeConfig]: nodes = [ ModelNodeConfig( name="EfficientRep", - override_name="kpt_detection_backbone", + alias="kpt_detection_backbone", freezing=self.backbone_params.pop("freezing", {}), params=self.backbone_params, ), @@ -35,7 +35,7 @@ def nodes(self) -> list[ModelNodeConfig]: nodes.append( ModelNodeConfig( name="RepPANNeck", - override_name="kpt_detection_neck", + alias="kpt_detection_neck", inputs=["kpt_detection_backbone"], freezing=self.neck_params.pop("freezing", {}), params=self.neck_params, @@ -45,7 +45,7 @@ def nodes(self) -> list[ModelNodeConfig]: nodes.append( ModelNodeConfig( name="ImplicitKeypointBBoxHead", - override_name="kpt_detection_head", + alias="kpt_detection_head", inputs=["kpt_detection_neck"] if self.use_neck else ["kpt_detection_backbone"], @@ -71,13 +71,13 @@ def metrics(self) -> list[MetricModuleConfig]: return [ MetricModuleConfig( name="ObjectKeypointSimilarity", - override_name="kpt_detection_oks", + alias="kpt_detection_oks", attached_to="kpt_detection_head", is_main_metric=True, ), MetricModuleConfig( name="MeanAveragePrecisionKeypoints", - override_name="kpt_detection_map", + alias="kpt_detection_map", attached_to="kpt_detection_head", ), ] @@ -87,7 +87,7 @@ def visualizers(self) -> list[AttachedModuleConfig]: return [ AttachedModuleConfig( name="MultiVisualizer", - override_name="kpt_detection_visualizer", + alias="kpt_detection_visualizer", attached_to="kpt_detection_head", params={ "visualizers": [ diff --git a/luxonis_train/models/predefined_models/segmentation_model.py b/luxonis_train/models/predefined_models/segmentation_model.py index 1e1ba0c4..9bc936a7 100644 --- a/luxonis_train/models/predefined_models/segmentation_model.py +++ b/luxonis_train/models/predefined_models/segmentation_model.py @@ -26,13 +26,13 @@ def nodes(self) -> list[ModelNodeConfig]: return [ ModelNodeConfig( name=self.backbone, - override_name="segmentation_backbone", + alias="segmentation_backbone", freezing=self.backbone_params.pop("freezing", {}), params=self.backbone_params, ), ModelNodeConfig( name="SegmentationHead", - override_name="segmentation_head", + alias="segmentation_head", inputs=["segmentation_backbone"], freezing=self.head_params.pop("freezing", {}), params=self.head_params, @@ -46,7 +46,7 @@ def losses(self) -> list[LossModuleConfig]: name="BCEWithLogitsLoss" if self.task == "binary" else "CrossEntropyLoss", - override_name="segmentation_loss", + alias="segmentation_loss", attached_to="segmentation_head", params=self.loss_params, weight=1.0, @@ -58,14 +58,14 @@ def metrics(self) -> list[MetricModuleConfig]: return [ MetricModuleConfig( name="JaccardIndex", - override_name="segmentation_jaccard_index", + alias="segmentation_jaccard_index", attached_to="segmentation_head", is_main_metric=True, params={"task": self.task}, ), MetricModuleConfig( name="F1Score", - override_name="segmentation_f1_score", + alias="segmentation_f1_score", attached_to="segmentation_head", params={"task": self.task}, ), @@ -76,7 +76,7 @@ def visualizers(self) -> list[AttachedModuleConfig]: return [ AttachedModuleConfig( name="SegmentationVisualizer", - override_name="segmentation_visualizer", + alias="segmentation_visualizer", attached_to="segmentation_head", params=self.visualizer_params, ) diff --git a/luxonis_train/nodes/README.md b/luxonis_train/nodes/README.md index 2c3758f9..bd44ac5a 100644 --- a/luxonis_train/nodes/README.md +++ b/luxonis_train/nodes/README.md @@ -172,9 +172,11 @@ Adapted from [here](https://arxiv.org/pdf/2209.02976.pdf). **Params** -| Key | Type | Default value | Description | -| --------- | ---- | ------------- | ---------------------- | -| num_heads | bool | 3 | Number of output heads | +| Key | Type | Default value | Description | +| ---------- | ----- | ------------- | -------------------------------------------------- | +| num_heads | bool | 3 | Number of output heads | +| conf_thres | float | 0.25 | confidence threshold for nms (used for evaluation) | +| iou_thres | float | 0.45 | iou threshold for nms (used for evaluation) | ## ImplicitKeypointBBoxHead diff --git a/luxonis_train/utils/config.py b/luxonis_train/utils/config.py index 9fd83fed..48661f7d 100644 --- a/luxonis_train/utils/config.py +++ b/luxonis_train/utils/config.py @@ -16,7 +16,7 @@ class AttachedModuleConfig(BaseModel): name: str attached_to: str - override_name: str | None = None + alias: str | None = None params: dict[str, Any] = {} @@ -35,7 +35,7 @@ class FreezingConfig(BaseModel): class ModelNodeConfig(BaseModel): name: str - override_name: str | None = None + alias: str | None = None inputs: list[str] = [] params: dict[str, Any] = {} freezing: FreezingConfig = FreezingConfig() @@ -82,14 +82,14 @@ def check_predefined_model(self): @model_validator(mode="after") def check_graph(self): - graph = {node.override_name or node.name: node.inputs for node in self.nodes} + graph = {node.alias or node.name: node.inputs for node in self.nodes} if not is_acyclic(graph): raise ValueError("Model graph is not acyclic.") if not self.outputs: outputs: list[str] = [] # nodes which are not inputs to any nodes inputs = set(node_name for node in self.nodes for node_name in node.inputs) for node in self.nodes: - name = node.override_name or node.name + name = node.alias or node.name if name not in inputs: outputs.append(name) self.outputs = outputs @@ -97,12 +97,21 @@ def check_graph(self): raise ValueError("No outputs specified.") return self - model_config = { - "json_schema_extra": { - "if": {"properties": {"predefined_model": {"type": "null"}}}, - "then": {"properties": {"nodes": {"type": "array"}}}, - } - } + @model_validator(mode="after") + def check_unique_names(self): + for section, objects in [ + ("nodes", self.nodes), + ("losses", self.losses), + ("metrics", self.metrics), + ("visualizers", self.visualizers), + ]: + names = set() + for obj in objects: + name = obj.alias or obj.name + if name in names: + raise ValueError(f"Duplicate name `{name}` in `{section}` section.") + names.add(name) + return self class TrackerConfig(BaseModel): @@ -118,8 +127,8 @@ class TrackerConfig(BaseModel): class DatasetConfig(BaseModel): - dataset_name: str | None = None - dataset_id: str | None = None + name: str | None = None + id: str | None = None team_name: str | None = None team_id: str | None = None bucket_type: BucketType = BucketType.INTERNAL @@ -133,25 +142,6 @@ class DatasetConfig(BaseModel): def get_enum_value(self, v: Enum, _) -> str: return str(v.value) - model_config = { - "json_schema_extra": { - "anyOf": [ - { - "allOf": [ - {"required": ["dataset_name"]}, - {"properties": {"dataset_name": {"type": "string"}}}, - ] - }, - { - "allOf": [ - {"required": ["dataset_id"]}, - {"properties": {"dataset_id": {"type": "string"}}}, - ] - }, - ] - }, - } - class NormalizeAugmentationConfig(BaseModel): active: bool = True @@ -288,9 +278,7 @@ class TunerConfig(BaseModel): storage: StorageConfig = StorageConfig() params: Annotated[ dict[str, list[str | int | float | bool]], Field(default={}, min_length=1) - ] = {} - - model_config = {"json_schema_extra": {"required": ["params"]}} + ] class Config(LuxonisConfig): @@ -300,20 +288,9 @@ class Config(LuxonisConfig): tracker: TrackerConfig = TrackerConfig() trainer: TrainerConfig = TrainerConfig() exporter: ExportConfig = ExportConfig() - tuner: TunerConfig = TunerConfig() + tuner: TunerConfig | None = None ENVIRON: Environ = Field(Environ(), exclude=True) - @model_validator(mode="before") - @classmethod - def check_tuner_init(cls, data: Any) -> Any: - if isinstance(data, dict): - if data.get("tuner") and not data.get("tuner", {}).get("params"): - del data["tuner"] - logger.warning( - "`tuner` block specified but no `tuner.params`. If trying to tune values you have to specify at least one parameter" - ) - return data - @model_validator(mode="before") @classmethod def check_environment(cls, data: Any) -> Any: diff --git a/pyproject.toml b/pyproject.toml index 0ea07add..048c005b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "luxonis-train" -version = "0.1.0" +version = "0.0.1" description = "Luxonis training framework for seamless training of various neural networks." readme = "README.md" requires-python = ">=3.10" diff --git a/tools/main.py b/tools/main.py index f1f7891e..73843593 100644 --- a/tools/main.py +++ b/tools/main.py @@ -126,9 +126,9 @@ def inspect( image_size = cfg.trainer.preprocessing.train_image_size dataset = LuxonisDataset( - dataset_name=cfg.dataset.dataset_name, + dataset_name=cfg.dataset.name, team_id=cfg.dataset.team_id, - dataset_id=cfg.dataset.dataset_id, + dataset_id=cfg.dataset.id, bucket_type=cfg.dataset.bucket_type, bucket_storage=cfg.dataset.bucket_storage, )