DOCS-2494: Create custom ML training script, upload to registry, and submit training job

.github/workflows/get_modular_resources.py

@@ -77,9 +77,12 @@ async def main():
     response : ListRegistryItemsResponse = await cloud._app_client.ListRegistryItems(request)
 
     ml_models_list = []
+    training_scripts_list = []
     for item in response.items:
         if item.type == 2:
             ml_models_list.append(item)
+        if item.type == 5:
+            training_scripts_list.append(item)
 
     for model in ml_models_list:
         if model.visibility == 2:
@@ -96,6 +99,20 @@ async def main():
         json_m)
             print(insert_resp)
 
+    for script in training_scripts_list:
+        if script.visibility == 2:
+            json_m = {
+                "id": script.item_id,
+                "model_id": script.item_id,
+                "name": script.name,
+                "description": script.description,
+                "last_updated": time_now,
+                "url": "https://app.viam.com/ml-training/" + script.public_namespace + "/" + script.name + "/"
+            }
+            insert_resp = typesense_client.collections['trainingscripts'].documents.upsert(
+        json_m)
+            print(insert_resp)
+
     viam_client.close()
 
     # Deleting documents that didn't get updated (presumably deleted)
@@ -106,6 +123,9 @@ async def main():
         res = typesense_client.collections['mlmodels'].documents.delete({'filter_by': 'last_updated: <' + str(time_now)})
         print("ML models deleted")
         print(res)
+        res = typesense_client.collections['trainingscripts'].documents.delete({'filter_by': 'last_updated: <' + time_now})
+        print("Training scripts deleted")
+        print(res)
     except Exception as e:
         print(e)
         pass

assets/js/models.js

@@ -182,3 +182,90 @@ if (mlmodels.length !== 0) {
 
   searchML.start();
 }
+
+const scripts = document.getElementsByClassName("training-scripts");
+
+if (scripts.length !== 0) {
+  const scripts = document.getElementsByClassName("training-scripts")[0].id;
+  const typesenseInstantsearchAdapterScripts = new TypesenseInstantSearchAdapter({
+    server: {
+      apiKey: "2bRwauAYhbrdjti898D6RXSkwgyEihDk", // Be sure to use an API key that only allows search operations
+      nodes: [
+        {
+          host: "cgnvrk0xwyj9576lp-1.a1.typesense.net",
+          port: "443",
+          protocol: "https",
+        },
+      ],
+      cacheSearchResultsForSeconds: 2 * 60, // Cache search results from server. Defaults to 2 minutes. Set to 0 to disable caching.
+    },
+    // The following parameters are directly passed to Typesense's search API endpoint.
+    //  So you can pass any parameters supported by the search endpoint below.
+    //  query_by is required.
+    additionalSearchParameters: {
+      query_by: "model_id,description",
+      sort_by: "last_updated:desc",
+      infix: "always",
+    },
+  });
+  const searchClientScripts = typesenseInstantsearchAdapterScripts.searchClient;
+
+  const searchScripts = instantsearch({
+    indexName: "trainingscripts",
+    searchClient: searchClientScripts,
+  });
+
+  let filtersScripts;
+  let itemtemplateScripts;
+
+  filtersScripts = {
+    hitsPerPage: 5,
+  };
+  itemtemplateScripts = `
+  <div class="name"><p><a href="{{url}}"><code>{{#helpers.highlight}}{ "attribute": "model_id" }{{/helpers.highlight}}</code></a></p></div>
+  <div class="description">{{#helpers.highlight}}{ "attribute": "description" }{{/helpers.highlight}}</div>
+  `;
+
+  searchScripts.addWidgets([
+    instantsearch.widgets.hits({
+      container: "#hitsScripts",
+      templates: {
+        item: itemtemplateScripts,
+      },
+    }),
+    instantsearch.widgets.searchBox({
+      container: "#searchboxScripts",
+      placeholder: "Search for a script...",
+      poweredBy: false,
+      wrapInput: true,
+      showReset: false,
+      showSubmit: false,
+      showLoadingIndicator: false,
+    }),
+    instantsearch.widgets.stats({
+      container: "#searchstatsScripts",
+      templates: {
+        text(data, { html }) {
+          let resultsScripts = "";
+
+          if (data.hasManyResults) {
+            resultsScripts += `${data.nbHits} results:`;
+          } else if (data.hasOneResult) {
+            resultsScripts += `1 result:`;
+          } else {
+            resultsScripts += ``;
+          }
+
+          return `<span>${resultsScripts}</span>`;
+        },
+      },
+    }),
+    instantsearch.widgets.configure(filtersScripts),
+    instantsearch.widgets.pagination({
+      container: "#paginationScripts",
+      scrollTo: false,
+    }),
+  ]);
+
+  searchScripts.start();
+}

assets/scss/_styles_project.scss

@@ -1477,7 +1477,7 @@ ul.sectionlist > li:hover {
 
 // Pagination CSS start
 
-#pagination, #paginationML {
+#pagination, #paginationML, #paginationScripts {
   margin: 2rem auto;
 }
 

docs/cli.md

@@ -392,10 +392,10 @@ For example, the following adds three images specified by their file ids to the
 viam data dataset add ids --dataset-id=abc --location-id=123 --org-id=123 --file-ids=abc,123,def
 ```
 
-To find the dataset ID of a given dataset, go to the [**Datasets** subtab](https://app.viam.com/data/datasets) under the **Data** tab on the Viam app and select a dataset.
-The dataset ID can be found in the URL of the Viam app window when viewing a given dataset, following the `?id=` portion of the URL, resembling `abcdef1234567890abcdef12`.
+To find the dataset ID of a given dataset, go to the [**DATASETS** subtab](https://app.viam.com/data/datasets) of the **DATA** tab on the Viam app and select a dataset.
+Click **...** in the left-hand menu and click **Copy dataset ID**.
 
-To find the file ID of a given image, navigate to the [**Data** tab in the Viam app](https://app.viam.com/data/view) and select your image.
+To find the file ID of a given image, navigate to the [**DATA** tab in the Viam app](https://app.viam.com/data/view) and select your image.
 Its **File ID** is shown under the **Details** subtab that appears on the right.
 
 You cannot use filter arguments, such as `--start` or `--end` when using `ids`.
@@ -411,11 +411,11 @@ For example, the following adds all images captured between January 1 and Octobe
 viam data dataset add filter --dataset-id=abc --org-ids=123 --start=2023-01-01T05:00:00.000Z --end=2023-10-01T04:00:00.000Z --tags=example
 ```
 
-To find the dataset ID of a given dataset, go to the [**Datasets** subtab](https://app.viam.com/data/datasets) under the **Data** tab on the Viam app and select a dataset.
-The dataset ID can be found in the URL of the Viam app window when viewing a given dataset, following the `?id=` portion of the URL, resembling `abcdef1234567890abcdef12`.
+To find the dataset ID of a given dataset, go to the [**DATASETS** subtab](https://app.viam.com/data/datasets) under the **DATA** tab on the Viam app and select a dataset.
+Click **...** in the left-hand menu and click **Copy dataset ID**.
 
-You can also have the filter parameters generated for you using the **Filters** pane of the **Data** tab.
-Navigate to the [**Data** tab in the Viam app](https://app.viam.com/data/view), make your selections from the search parameters under the **Filters** pane (such as robot name, start and end time, or tags), and click the **Copy export command** button.
+You can also have the filter parameters generated for you using the **Filters** pane of the **DATA** tab.
+Navigate to the [**DATA** tab in the Viam app](https://app.viam.com/data/view), make your selections from the search parameters under the **Filters** pane (such as robot name, start and end time, or tags), and click the **Copy export command** button.
 A `viam data export` command string will be copied to your clipboard that includes the search parameters you selected.
 You can use the same filter parameters (such as `--start`, `--end`, etc) with your `viam data database add filter` or `viam data database remove filter` commands, except you would exclude the `--data-type` and `--destination` flags, which are specific to `viam data export`.
 
@@ -1068,6 +1068,79 @@ viam.service.vision.v1.VisionService.GetClassificationsFromCamera
 
 The `--stream` argument, when included in the CLI command prior to the `--data` command, will stream data back at the specified interval.
 
+### `training-script`
+
+Manage training scripts for custom ML training.
+For more information, see [Train a Model with a Custom Python Training Script](/services/ml/upload-training-script/).
+
+#### Command options
+
+<!-- prettier-ignore -->
+| Command option | Description | Positional arguments |
+| -------------- | ----------- | -------------------- |
+| `upload` | Upload ML training script to the registry | - |
+| `update` | Update visibility of ML training script in registry | - |
+
+##### Named arguments
+
+<!-- prettier-ignore -->
+| Argument | Description | Applicable commands | Required? |
+| -------- | ----------- | ------------------- | --------- |
+| `--path` | The path to ML training scripts for upload. | `upload` | **Required** |
+| `--org-id` | The organization ID to host the scripts in. You can find your organization ID by running `viam organizations list` or by visiting your organization's **Settings** page in [the Viam app](https://app.viam.com/). | `upload`, `update` | **Required** |
+| `--script-name` | Name of the ML training script to update or upload. | `upload`, `update` | **Required** |
+| `--version` | Version of the ML training script to upload. | `upload` | Optional |
+| `--framework` | Framework of the ML training script to upload, can be `tflite`, `tensorflow`, `pytorch`, or `onnx`. | `upload` | Optional |
+| `--type` | Task type of the ML training script to upload, can be `single_label_classification`, `multi_label_classification`, or `object_detection`. | `upload` | Optional |
+| `--draft` | Indicate draft mode, drafts are not viewable in the registry. | `upload` | Optional |
+| `--visibility` | Visibility of the registry item, can be `public` or `private`. | `update` | **Required** |
+| `--description` | Description of the ML training script. | `update` | Optional |
+
+### `train`
+
+Use a training script to train an ML model on data.
+
+#### Command options
+
+<!-- prettier-ignore -->
+| Command option | Description | Positional arguments |
+| -------------- | ----------- | -------------------- |
+| `submit` | Submits training job on data in the Viam cloud | `managed`, `custom` |
+
+##### Positional arguments: `submit`
+
+<!-- prettier-ignore -->
+| Argument | Description | Positional Arguments |
+| -------- | ----------- | -------------------- |
+| `managed` | Submits training job on data in the Viam cloud with a Viam-managed training script | - |
+| `custom` | Submits custom training job on data in the Viam cloud | `from-registry`, `with-upload` |
+
+##### Position arguments: `submit custom`
+
+<!-- prettier-ignore -->
+| Argument | Description |
+| -------- | ----------- |
+| `from-registry` | Submit custom training job with an existing training script in the registry on data in the Viam cloud |
+| `with-upload` | Upload a draft training script and submit a custom training job on data in the Viam cloud |
+
+##### Named arguments
+
+<!-- prettier-ignore -->
+| Argument | Description | Applicable commands | Required? |
+| -------- | ----------- | ------------------- | --------- |
+| `--dataset-id` | The ID of the dataset to train on. To find the dataset ID of a given dataset, go to the [**DATASETS** subtab](https://app.viam.com/data/datasets) of the **DATA** tab on the Viam app and select a dataset. Click **...** in the left-hand menu and click **Copy dataset ID**. | `submit managed`, `submit custom from-registry`, `submit custom with-upload` | **Required** |
+| `--model-org-id` | The organization ID to train and save the ML model in. You can find your organization ID by running `viam organizations list` or by visiting your organization's **Settings** page in [the Viam app](https://app.viam.com/). | `submit managed`, `submit custom with-upload` | **Required** |
+| `--org-id` | The organization ID to train and save the ML model in. You can find your organization ID by running `viam organizations list` or by visiting your organization's **Settings** page in [the Viam app](https://app.viam.com/). | `submit custom from-registry` | **Required** |
+| `--model-name` | The name of the ML model. | `submit managed`, `submit custom from-registry`, `submit custom with-upload` | **Required** |
+| `--model-type` | Type of model to train. can be one of `single_label_classification`, `multi_label_classification`, or `object_detection`. | `submit managed`, `submit custom with-upload` | **Required**, Optional |
+| `--model-labels` | Labels to train on. These will either be classification or object detection labels. | `submit managed` | **Required** |
+| `--model-version` | Set the version of the submitted model. Defaults to current timestamp if unspecified. | `submit managed`, `submit custom from-registry`, `submit custom with-upload` | **Required** |
+| `--script-name` | The registry name of the ML training script to use for training. | `submit custom from-registry` | **Required** |
+| `--version` | The version of the ML training script to use for training. | `submit custom from-registry`, `submit custom with-upload` | **Required** |
+| `--path` | The path to the ML training script to upload. | `submit custom with-upload` | **Required** |
+| `--script-name` | The script name of the ML training script to upload. | `submit custom with-upload` | **Required** |
+| `--framework` | Framework of the ML training script to upload, can be `tflite`, `tensorflow`, `pytorch`, or `onnx`. | `submit custom with-upload` | Optional |
+
 ### `version`
 
 The `version` command returns the version of the Viam CLI.

docs/registry/_index.md

@@ -29,48 +29,22 @@ menuindent: true
 The [Viam registry](https://app.viam.com/registry) is the central place where you can browse:
 
 {{< cards >}}
-{{% manualcard link="#ml-models" %}}
-
-**ML models** to deploy with machine applications like computer vision
-
-{{% /manualcard %}}
 {{% manualcard link="#modular-resources" %}}
 
 **Modular resources** that add capabilities to your machine beyond what is built into `viam-server`
 
 {{% /manualcard %}}
-{{< /cards >}}
-
-## ML models
-
-Viam provides the ability to train, upload, and deploy machine learning models within the platform.
-See [Machine Learning](/services/ml/) for more information.
+{{% manualcard link="#ml-models" %}}
 
-The Viam registry hosts trained ML models that users have made public, which you can use to deploy classifiers or detectors for your use case onto your robot instead of training your own.
-You can also [upload your own model to the registry](/services/ml/upload-model/).
+**ML models** to deploy with machine applications like computer vision
 
-You can search the available ML models from the Viam registry here:
+{{% /manualcard %}}
+{{% manualcard link="#ml-training-scripts" %}}
 
-<div id="searchboxML"></div>
-<p>
-<div id="searchstatsML"></div></p>
-<div class="mr-model" id="">
-  <div class="modellistheader">
-    <div class="name">Model</div>
-    <div>Description</div>
-  </div>
-<div id="hitsML" class="modellist">
-</div>
-<div id="paginationML"></div>
-</div>
-<noscript>
-    <div class="alert alert-caution" role="alert">
-        <h4 class="alert-heading">Javascript</h4>
-        <p>Please enable javascript to see and search ML models.</p>
-    </div>
-</noscript>
+**Training scripts** to train machine learning models in the Viam cloud from custom Python scripts.
 
-To use an existing model from the registry, [deploy the ML model to your robot](/services/ml/deploy/) and use a [Vision service](/services/vision/) to make detections or classifications on-machine.
+{{% /manualcard %}}
+{{< /cards >}}
 
 ## Modular resources
 
@@ -206,3 +180,61 @@ When implementing a custom {{< glossary_tooltip term_id="model" text="model" >}}
 {{% card link="/tutorials/custom/custom-base-dog/" %}}
 {{% card link="/registry/examples/custom-arm/" %}}
 {{< /cards >}}
+
+## ML models
+
+Viam provides the ability to train, upload, and deploy machine learning models within the platform.
+See [Machine Learning](/services/ml/) for more information.
+
+The Viam registry hosts trained ML models that users have made public, which you can use to deploy classifiers or detectors for your use case onto your robot instead of training your own.
+You can also [upload your own model to the registry](/services/ml/upload-model/).
+
+You can search the available ML models from the Viam registry here:
+
+<div id="searchboxML"></div>
+<p>
+<div id="searchstatsML"></div></p>
+<div class="mr-model" id="">
+  <div class="modellistheader">
+    <div class="name">Model</div>
+    <div>Description</div>
+  </div>
+<div id="hitsML" class="modellist">
+</div>
+<div id="paginationML"></div>
+</div>
+<noscript>
+    <div class="alert alert-caution" role="alert">
+        <h4 class="alert-heading">Javascript</h4>
+        <p>Please enable javascript to see and search ML models.</p>
+    </div>
+</noscript>
+
+To use an existing model from the registry, [deploy the ML model to your robot](/services/ml/deploy/) and use a [Vision service](/services/vision/) to make detections or classifications on-machine.
+
+## ML training scripts
+
+The Viam registry hosts trained ML training scripts that users have made public, which you can use to train custom machine learning models from Python source distribution.
+You can also upload your own training script.
+Follow the guide to [Train a Model with a Custom Python Training Script](/services/ml/upload-training-script/#upload-a-new-training-script-or-new-version).
+
+You can search the available ML training scripts from the Viam registry here:
+
+<div id="searchboxScripts"></div>
+<p>
+<div id="searchstatsScripts"></div></p>
+<div class="training-scripts" id="">
+  <div class="modellistheader">
+    <div class="name">Script</div>
+    <div>Description</div>
+  </div>
+<div id="hitsScripts" class="modellist">
+</div>
+<div id="paginationScripts"></div>
+</div>
+<noscript>
+    <div class="alert alert-caution" role="alert">
+        <h4 class="alert-heading">Javascript</h4>
+        <p>Please enable javascript to see and search ML custom training scripts.</p>
+    </div>
+</noscript>