Merge branch 'interuss:master' into master

.github/workflows/ci.yml

@@ -67,3 +67,31 @@ jobs:
       run: make qualify-locally
     - name: Bring down local DSS instance
       run: make down-locally
+
+  dss-tests-with-yugabyte:
+    name: DSS tests with Yugabyte
+    runs-on: ubuntu-latest
+    env:
+      COMPOSE_PROFILES: with-yugabyte
+    steps:
+      - name: Job information
+        run: |
+          echo "Job information"
+          echo "Trigger: ${{ github.event_name }}"
+          echo "Host: ${{ runner.os }}"
+          echo "Repository: ${{ github.repository }}"
+          echo "Branch: ${{ github.ref }}"
+          docker images
+          go env
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          submodules: true
+      - name: Build dss image
+        run: make build-dss
+      - name: Tear down any pre-existing local DSS instance
+        run: make down-locally
+      - name: Start local DSS instance
+        run: make start-locally
+      - name: Bring down local DSS instance
+        run: make down-locally

.github/workflows/codeql.yml

@@ -0,0 +1,94 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL Advanced"
+
+on:
+  push:
+    branches: [ "master" ]
+  pull_request:
+    branches: [ "master" ]
+  schedule:
+    - cron: '16 14 * * 6'
+
+jobs:
+  analyze:
+    name: Analyze (${{ matrix.language }})
+    # Runner size impacts CodeQL analysis time. To learn more, please see:
+    #   - https://gh.io/recommended-hardware-resources-for-running-codeql
+    #   - https://gh.io/supported-runners-and-hardware-resources
+    #   - https://gh.io/using-larger-runners (GitHub.com only)
+    # Consider using larger runners or machines with greater resources for possible analysis time improvements.
+    runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}
+    permissions:
+      # required for all workflows
+      security-events: write
+
+      # required to fetch internal or private CodeQL packs
+      packages: read
+
+      # only required for workflows in private repositories
+      actions: read
+      contents: read
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+        - language: go
+          build-mode: autobuild
+        - language: python
+          build-mode: none
+        # CodeQL supports the following values keywords for 'language': 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift'
+        # Use `c-cpp` to analyze code written in C, C++ or both
+        # Use 'java-kotlin' to analyze code written in Java, Kotlin or both
+        # Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both
+        # To learn more about changing the languages that are analyzed or customizing the build mode for your analysis,
+        # see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning.
+        # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how
+        # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v3
+      with:
+        languages: ${{ matrix.language }}
+        build-mode: ${{ matrix.build-mode }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+
+        # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        # queries: security-extended,security-and-quality
+
+    # If the analyze step fails for one of the languages you are analyzing with
+    # "We were unable to automatically build your code", modify the matrix above
+    # to set the build mode to "manual" for that language. Then modify this step
+    # to build your code.
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+    - if: matrix.build-mode == 'manual'
+      shell: bash
+      run: |
+        echo 'If you are using a "manual" build mode for one or more of the' \
+          'languages you are analyzing, replace this with the commands to build' \
+          'your code, for example:'
+        echo '  make bootstrap'
+        echo '  make release'
+        exit 1
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v3
+      with:
+        category: "/language:${{matrix.language}}"

Makefile

@@ -114,8 +114,10 @@ test-go-units-crdb: cleanup-test-go-units-crdb
 	@docker run -d --name dss-crdb-for-testing -p 26257:26257 -p 8080:8080  cockroachdb/cockroach:v24.1.3 start-single-node --insecure > /dev/null
 	@until [ -n "`docker logs dss-crdb-for-testing | grep 'nodeID'`" ]; do echo "Waiting for CRDB to be ready"; sleep 3; done;
 	go run ./cmds/db-manager/main.go migrate --schemas_dir ./build/db_schemas/rid --db_version latest --cockroach_host localhost
+	go run ./cmds/db-manager/main.go migrate --schemas_dir ./build/db_schemas/scd --db_version latest --cockroach_host localhost
 	go test -count=1 -v ./pkg/rid/store/cockroach --cockroach_host localhost --cockroach_port 26257 --cockroach_ssl_mode disable --cockroach_user root --cockroach_db_name rid
 	go test -count=1 -v ./pkg/rid/application --cockroach_host localhost --cockroach_port 26257 --cockroach_ssl_mode disable --cockroach_user root --cockroach_db_name rid
+	go test -count=1 -v ./pkg/scd/store/cockroach --cockroach_host localhost --cockroach_port 26257 --cockroach_ssl_mode disable --cockroach_user root --cockroach_db_name scd
 	@docker stop dss-crdb-for-testing > /dev/null
 	@docker rm dss-crdb-for-testing > /dev/null
 
@@ -139,8 +141,8 @@ restart-all: build-dss down-locally start-locally
 
 .PHONY: start-locally
 start-locally:
+	@test ${COMPOSE_PROFILES} || echo "\033[0;96mTo start Yugabyte datastore, set the docker compose profile to with-yugabyte. Example using env variable: export COMPOSE_PROFILES=with-yugabyte\033[0m"
 	build/dev/run_locally.sh up -d
-	build/dev/wait_for_local_dss.sh
 
 .PHONY: probe-locally
 probe-locally:

build/db_schemas/yugabyte/rid/downfrom-v1.0.0-crdb_v4.0.0.sql

@@ -0,0 +1,3 @@
+DROP TABLE IF EXISTS identification_service_areas;
+DROP TABLE IF EXISTS subscriptions;
+DROP TABLE IF EXISTS schema_versions;

build/db_schemas/yugabyte/rid/upto-v1.0.0-crdb_v4.0.0.sql

@@ -0,0 +1,47 @@
+-- This migration is equivalent to rid v4.0.0 schema for CockroachDB with the notable exception
+-- of the inverted index replaced by ybgin, which presents currently some limitations
+-- https://docs.yugabyte.com/preview/explore/ysql-language-features/indexes-constraints/gin/#limitations.
+
+CREATE TABLE subscriptions (
+    id UUID PRIMARY KEY,
+    owner TEXT NOT NULL,
+    url TEXT NOT NULL,
+    notification_index INT4 DEFAULT 0,
+    starts_at TIMESTAMPTZ,
+    ends_at TIMESTAMPTZ,
+    updated_at TIMESTAMPTZ NOT NULL,
+    cells BIGINT[] NOT NULL,
+    writer TEXT,
+    CHECK (starts_at IS NULL OR ends_at IS NULL OR starts_at < ends_at),
+    CHECK (array_length(cells, 1) IS NOT NULL)
+);
+CREATE INDEX s_owner_idx ON subscriptions (owner);
+CREATE INDEX s_starts_at_idx ON subscriptions (starts_at);
+CREATE INDEX s_ends_at_idx ON subscriptions (ends_at);
+CREATE INDEX s_cell_idx ON subscriptions USING ybgin (cells);
+CREATE INDEX subs_by_time_with_owner ON subscriptions (ends_at) INCLUDE (owner);
+
+CREATE TABLE identification_service_areas (
+    id UUID PRIMARY KEY,
+    owner TEXT NOT NULL,
+    url TEXT NOT NULL,
+    starts_at TIMESTAMPTZ,
+    ends_at TIMESTAMPTZ,
+    updated_at TIMESTAMPTZ NOT NULL,
+    cells BIGINT[] NOT NULL,
+    writer TEXT,
+    CHECK (starts_at IS NULL OR ends_at IS NULL OR starts_at < ends_at),
+    CHECK (array_length(cells, 1) IS NOT NULL)
+);
+CREATE INDEX isa_owner_idx ON identification_service_areas (owner);
+CREATE INDEX isa_starts_at_idx ON identification_service_areas (starts_at);
+CREATE INDEX isa_ends_at_idx ON identification_service_areas (ends_at);
+CREATE INDEX isa_updated_at_idx ON identification_service_areas (updated_at);
+CREATE INDEX isa_cell_idx ON identification_service_areas USING ybgin (cells);
+
+CREATE TABLE schema_versions (
+	onerow_enforcer bool PRIMARY KEY DEFAULT TRUE CHECK(onerow_enforcer),
+	schema_version TEXT NOT NULL
+);
+
+INSERT INTO schema_versions (schema_version) VALUES ('v1.0.0');

build/db_schemas/yugabyte/scd/downfrom-v1.0.0-crdb_v3.2.0.sql

@@ -0,0 +1,6 @@
+DROP TABLE IF EXISTS schema_versions;
+DROP TABLE IF EXISTS scd_uss_availability;
+DROP TABLE IF EXISTS scd_constraints;
+DROP TABLE IF EXISTS scd_operations;
+DROP TABLE IF EXISTS scd_subscriptions;
+DROP TYPE IF EXISTS operational_intent_state;

build/db_schemas/yugabyte/scd/upto-v1.0.0-crdb_v3.2.0.sql

@@ -0,0 +1,90 @@
+-- This migration is equivalent to scd v3.2.0 schema for CockroachDB with the notable exception
+-- of the inverted index replaced by ybgin, which presents currently some limitations
+-- https://docs.yugabyte.com/preview/explore/ysql-language-features/indexes-constraints/gin/#limitations.
+
+CREATE TABLE IF NOT EXISTS scd_subscriptions (
+  id UUID PRIMARY KEY,
+  owner TEXT NOT NULL,
+  version INT4 NOT NULL DEFAULT 0,
+  url TEXT NOT NULL,
+  notification_index INT4 DEFAULT 0,
+  notify_for_operations BOOL DEFAULT false,
+  notify_for_constraints BOOL DEFAULT false,
+  implicit BOOL DEFAULT false,
+  starts_at TIMESTAMPTZ,
+  ends_at TIMESTAMPTZ,
+  updated_at TIMESTAMPTZ NOT NULL,
+  cells BIGINT[],
+  CHECK (starts_at IS NULL OR ends_at IS NULL OR starts_at < ends_at),
+  CHECK (notify_for_operations OR notify_for_constraints)
+);
+CREATE INDEX ss_owner_idx ON scd_subscriptions (owner);
+CREATE INDEX ss_starts_at_idx ON scd_subscriptions (starts_at);
+CREATE INDEX ss_ends_at_idx ON scd_subscriptions (ends_at);
+CREATE INDEX ss_cells_idx ON scd_subscriptions USING ybgin (cells);
+
+CREATE TYPE operational_intent_state AS ENUM ('Unknown', 'Accepted', 'Activated', 'Nonconforming', 'Contingent');
+
+CREATE TABLE IF NOT EXISTS scd_operations (
+  id UUID PRIMARY KEY,
+  owner TEXT NOT NULL,
+  version INT4 NOT NULL DEFAULT 0,
+  url TEXT NOT NULL,
+  altitude_lower REAL,
+  altitude_upper REAL,
+  starts_at TIMESTAMPTZ,
+  ends_at TIMESTAMPTZ,
+  subscription_id UUID REFERENCES scd_subscriptions(id) ON DELETE CASCADE,
+  updated_at TIMESTAMPTZ NOT NULL,
+  state operational_intent_state NOT NULL DEFAULT 'Unknown',
+  cells BIGINT[],
+  uss_requested_ovn TEXT,
+  past_ovns TEXT[] NOT NULL DEFAULT ARRAY []::TEXT[],
+  CHECK (starts_at IS NULL OR ends_at IS NULL OR starts_at < ends_at),
+  CHECK (uss_requested_ovn != ''),
+  CHECK (
+      array_position(past_ovns, NULL) IS NULL AND
+      array_position(past_ovns, '') IS NULL AND
+      array_position(past_ovns, uss_requested_ovn) IS NULL
+      )
+);
+CREATE INDEX so_owner_idx ON scd_operations (owner);
+CREATE INDEX so_altitude_lower_idx ON scd_operations (altitude_lower);
+CREATE INDEX so_altitude_upper_idx ON scd_operations (altitude_upper);
+CREATE INDEX so_starts_at_idx ON scd_operations (starts_at);
+CREATE INDEX so_ends_at_idx ON scd_operations (ends_at);
+CREATE INDEX so_updated_at_idx ON scd_operations (updated_at);
+CREATE INDEX so_subscription_id_idx ON scd_operations (subscription_id);
+CREATE INDEX so_cells_idx ON scd_operations USING ybgin (cells);
+
+CREATE TABLE IF NOT EXISTS scd_constraints (
+  id UUID PRIMARY KEY,
+  owner TEXT NOT NULL,
+  version INT4 NOT NULL DEFAULT 0,
+  url TEXT NOT NULL,
+  altitude_lower REAL,
+  altitude_upper REAL,
+  starts_at TIMESTAMPTZ,
+  ends_at TIMESTAMPTZ,
+  updated_at TIMESTAMPTZ NOT NULL,
+  cells BIGINT[] NOT NULL CHECK (array_length(cells, 1) IS NOT NULL),
+  CHECK (starts_at IS NULL OR ends_at IS NULL OR starts_at < ends_at)
+);
+CREATE INDEX sc_owner_idx ON scd_constraints (owner);
+CREATE INDEX sc_starts_at_idx ON scd_constraints (starts_at);
+CREATE INDEX sc_ends_at_idx ON scd_constraints (ends_at);
+CREATE INDEX sc_cells_idx ON scd_constraints USING ybgin (cells);
+
+
+CREATE TABLE IF NOT EXISTS scd_uss_availability (
+  id TEXT PRIMARY KEY,
+  availability TEXT NOT NULL,
+  updated_at TIMESTAMPTZ NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS schema_versions (
+  onerow_enforcer bool PRIMARY KEY DEFAULT TRUE CHECK(onerow_enforcer),
+  schema_version TEXT NOT NULL
+);
+
+INSERT INTO schema_versions (schema_version) VALUES ('v1.0.0');