Merge pull request #904 from cherrycl/issue-903

feat: Use keeper as registry service and set registry service to a selection

TAF/config/global_variables.py

@@ -2,9 +2,6 @@
 
 LOG_LEVEL = "INFO"
 
-# Base endpoint in the Consul path for all services
-CONSUL_CONFIG_BASE_ENDPOINT = "/v1/kv/edgex/v3"
-
 # API Version
 API_VERSION = "v3"
 
@@ -20,25 +17,31 @@
 SECURITY_SERVICE_NEEDED=os.getenv("SECURITY_SERVICE_NEEDED")
 DOCKER_HOST_IP=os.getenv("DOCKER_HOST_IP")
 ARCH=os.getenv("ARCH")
+REGISTRY_SERVICE=os.getenv("REGISTRY_SERVICE")
 
 # Token related variables
 jwt_token = ''
 consul_token = ''
 
 # Service port
+APP_SAMPLE_PORT=59700
 APP_HTTP_EXPORT_PORT = 59704
 APP_MQTT_EXPORT_PORT = 59703
-APP_FUNCTIOAL_TESTS_PORT = 59705
+APP_FUNCTIONAL_TESTS_PORT = 59705
 APP_EXTERNAL_MQTT_TRIGGER_PORT = 59706
-REGISTRY_PORT = 8500
+CONSUL_PORT = 8500
 EX_BROKER_PORT = 1884
 BROKER_PORT = 1883
 
+# Registry Config Version
+CONFIG_VERSION = "v3"
+
 if SECURITY_SERVICE_NEEDED == 'true':
     URI_SCHEME = "https"
     CORE_DATA_PORT = "8443/core-data"
     CORE_METADATA_PORT = "8443/core-metadata"
     CORE_COMMAND_PORT = "8443/core-command"
+    CORE_KEEPER_PORT = "8443/core-keeper"
     SUPPORT_NOTIFICATIONS_PORT = "8443/support-notifications"
     SUPPORT_SCHEDULER_PORT = "8443/support-scheduler"
     RULESENGINE_PORT = "8443/rules-engine"
@@ -48,11 +51,18 @@
     CORE_DATA_PORT = 59880
     CORE_METADATA_PORT = 59881
     CORE_COMMAND_PORT = 59882
+    CORE_KEEPER_PORT = 59890
     SUPPORT_NOTIFICATIONS_PORT = 59860
     SUPPORT_SCHEDULER_PORT = 59861
     RULESENGINE_PORT = 59720
     ONVIF_CAMERA_PORT = 59984
 
+if REGISTRY_SERVICE == 'Consul':
+    REGISTRY_PORT = CONSUL_PORT
+else:
+    REGISTRY_PORT = CORE_KEEPER_PORT
+
+
 # External MQTT Auth
 EX_BROKER_USER = os.getenv("EX_BROKER_USER")
 EX_BROKER_PASSWD = os.getenv("EX_BROKER_PASSWD")

TAF/testCaseModules/keywords/app-service/AppServiceAPI.robot

@@ -22,20 +22,20 @@ Suite Teardown for App Service
     Run Teardown Keywords
 
 Set ${service} Functions ${functions}
-    ${path}=  Set variable  ${CONSUL_CONFIG_BASE_ENDPOINT}/${service}/Writable/Pipeline/ExecutionOrder
-    Update Service Configuration On Consul  ${path}  ${functions}
+    ${path}=  Set variable  /${service}/Writable/Pipeline/ExecutionOrder
+    Update Configuration On Registry Service  ${path}  ${functions}
 
 Set Transform Type ${type}
-    ${path}=  Set variable  ${CONSUL_CONFIG_BASE_ENDPOINT}/app-functional-tests/Writable/Pipeline/Functions/Transform/Parameters/Type
-    Update Service Configuration On Consul  ${path}  ${type}
+    ${path}=  Set variable  /app-functional-tests/Writable/Pipeline/Functions/Transform/Parameters/Type
+    Update Configuration On Registry Service  ${path}  ${type}
 
 Set Compress Algorithm ${algorithm}
-    ${path}=  Set variable  ${CONSUL_CONFIG_BASE_ENDPOINT}/app-functional-tests/Writable/Pipeline/Functions/Compress/Parameters/Algorithm
-    Update Service Configuration On Consul  ${path}  ${algorithm}
+    ${path}=  Set variable  /app-functional-tests/Writable/Pipeline/Functions/Compress/Parameters/Algorithm
+    Update Configuration On Registry Service  ${path}  ${algorithm}
 
 Set Encrypt Algorithm ${algorithm}
-    ${path}=  Set variable  ${CONSUL_CONFIG_BASE_ENDPOINT}/app-functional-tests/Writable/Pipeline/Functions/Encrypt/Parameters/Algorithm
-    Update Service Configuration On Consul  ${path}  ${algorithm}
+    ${path}=  Set variable  /app-functional-tests/Writable/Pipeline/Functions/Encrypt/Parameters/Algorithm
+    Update Configuration On Registry Service  ${path}  ${algorithm}
 
 Trigger Function Pipeline With ${data}
     ${trigger_data}=  Run keyword if  '${data}' != 'Invalid Data'  set variable  Valid Data

TAF/testCaseModules/keywords/common/commonKeywords.robot

@@ -11,6 +11,8 @@ Library   TAF/testCaseModules/keywords/setup/setup_teardown.py
 
 *** Variables ***
 ${default_response_time_threshold}  1200
+${CONSUL_CONFIG_BASE_ENDPOINT}      /v1/kv/edgex/${CONFIG_VERSION}
+${KEEPER_CONFIG_BASE_ENDPOINT}      /api/${API_VERSION}/kvs/key/edgex/${CONFIG_VERSION}
 
 *** Keywords ***
 Setup Suite
@@ -242,7 +244,7 @@ Update Service Configuration On Consul
     ${headers}=  Create Dictionary  X-Consul-Token=${consul_token}
     ${url}  Set Variable  http://${BASE_URL}:${REGISTRY_PORT}
     Create Session  Consul  url=${url}  disable_warnings=true
-    ${resp}=  PUT On Session  Consul  ${path}  data=${value}  headers=${headers}  expected_status=200
+    ${resp}=  PUT On Session  Consul  ${CONSUL_CONFIG_BASE_ENDPOINT}/${path}  data=${value}  headers=${headers}  expected_status=200
     Sleep  1s  # Waiting for the configuration updating
 
 Query Service Configuration On Consul
@@ -251,16 +253,67 @@ Query Service Configuration On Consul
     ${headers}=  Create Dictionary  X-Consul-Token=${consul_token}
     ${url}  Set Variable  http://${BASE_URL}:${REGISTRY_PORT}
     Create Session  Consul  url=${url}  disable_warnings=true
-    ${resp}=  Get On Session  Consul  ${path}  headers=${headers}  expected_status=any
+    ${resp}=  Get On Session  Consul  ${CONSUL_CONFIG_BASE_ENDPOINT}/${path}  headers=${headers}  expected_status=any
     Set Response to Test Variables  ${resp}
 
+Delete Service Configuration On Consul
+    [Arguments]  ${path}
+    ${consul_token}  Run Keyword If  $SECURITY_SERVICE_NEEDED == 'true'  Get Consul Token
+    ${headers}=  Create Dictionary  X-Consul-Token=${consul_token}
+    ${url}  Set Variable  http://${BASE_URL}:${REGISTRY_PORT}
+    Create Session  Consul  url=${url}  disable_warnings=true
+    DELETE On Session  Consul  ${CONSUL_CONFIG_BASE_ENDPOINT}/${path}  params=recurse=true  headers=${headers}
+    ...  expected_status=200
+
 Get Consul Token
     ${command}  Set Variable  cat /tmp/edgex/secrets/consul-acl-token/bootstrap_token.json
     ${result}  Run Process  ${WORK_DIR}/TAF/utils/scripts/${DEPLOY_TYPE}/execute-command-in-docker.sh core-consul "${command}"
     ...     shell=True  stderr=STDOUT  output_encoding=UTF-8  timeout=10s
     ${token}  Evaluate  json.loads('''${result.stdout}''')  json
     [Return]  ${token}[SecretID]
 
+Update Service Configuration On Keeper
+    [Arguments]  ${path}  ${value}
+    ${headers}=  Create Dictionary  Authorization=Bearer ${jwt_token}
+    ${url}  Set Variable  ${URI_SCHEME}://${BASE_URL}:${CORE_KEEPER_PORT}
+    Create Session  Keeper  url=${url}  disable_warnings=true
+    ${body}  Create Dictionary  value=${value}
+    ${resp}=  PUT On Session  Keeper  ${KEEPER_CONFIG_BASE_ENDPOINT}/${path}
+    ...   json=${body}  headers=${headers}  expected_status=200
+    Sleep  1s  # Waiting for the configuration updating
+
+Query Service Configuration On Keeper
+    [Arguments]  ${path}
+    ${headers}=  Create Dictionary  Authorization=Bearer ${jwt_token}
+    ${url}  Set Variable  ${URI_SCHEME}://${BASE_URL}:${CORE_KEEPER_PORT}
+    Create Session  Keeper  url=${url}  disable_warnings=true
+    ${resp}=  GET On Session  Keeper  ${KEEPER_CONFIG_BASE_ENDPOINT}/${path}
+    ...   headers=${headers}  expected_status=any
+    Set Response to Test Variables  ${resp}
+
+Delete Service Configuration On Keeper
+    [Arguments]  ${path}
+    ${headers}=  Create Dictionary  Authorization=Bearer ${jwt_token}
+    ${url}  Set Variable  ${URI_SCHEME}://${BASE_URL}:${CORE_KEEPER_PORT}
+    Create Session  Keeper  url=${url}  disable_warnings=true
+    ${resp}=  DELETE On Session  Keeper  ${KEEPER_CONFIG_BASE_ENDPOINT}/${path}
+    ...   headers=${headers}  expected_status=200
+
+Update Configuration On Registry Service
+    [Arguments]  ${path}  ${value}
+    Run Keyword If  "${REGISTRY_SERVICE}" == "Consul"  Update Service Configuration On Consul  ${path}  ${value}
+    ...    ELSE IF  "${REGISTRY_SERVICE}" == "Keeper"  Update Service Configuration On Keeper  ${path}  ${value}
+
+Query Configuration On Registry Service
+    [Arguments]  ${path}
+    Run Keyword If  "${REGISTRY_SERVICE}" == "Consul"  Query Service Configuration On Consul  ${path}
+    ...    ELSE IF  "${REGISTRY_SERVICE}" == "Keeper"  Query Service Configuration On Keeper  ${path}
+
+Delete Configuration On Registry Service
+    [Arguments]  ${path}
+    Run Keyword If  "${REGISTRY_SERVICE}" == "Consul"  Delete Service Configuration On Consul  ${path}
+    ...    ELSE IF  "${REGISTRY_SERVICE}" == "Keeper"  Delete Service Configuration On Keeper  ${path}
+
 Run Redis Subscriber Progress And Output
     [Arguments]  ${topic}  ${keyword}  ${expected_msg_count}=1  ${duration}=30
     ${current_time}  get current epoch time
@@ -329,18 +382,21 @@ Service ${service} Secrets Should be Stored
     Set Response to Test Variables  ${resp}
     Should Contain  ${content}[data]  ${secrets_key}  ${secrets_value}
 
-Secrets Should be Stored To Consul
+Secrets Should be Stored To Registry Service
     [Arguments]  ${service}
-    # Validate Secret Name
-    ${secrets_name_path}  Set Variable  ${CONSUL_CONFIG_BASE_ENDPOINT}/${service}/Writable/InsecureSecrets/${secrets_name}/SecretName
-    Query Service Configuration On Consul  ${secrets_name_path}
-    ${secrets_name_consul_value}  Evaluate  base64.b64decode('${content}[0][Value]').decode('utf-8')  modules=base64
-    Should Be Equal  ${secrets_name_consul_value}  ${secrets_name}
-    # Validate Secret Data
-    ${secrets_data_path}  Set Variable  ${CONSUL_CONFIG_BASE_ENDPOINT}/${service}/Writable/InsecureSecrets/${secrets_name}/SecretData/${secrets_key}
-    Query Service Configuration On Consul  ${secrets_data_path}
-    ${secrets_key_consul_value}  Evaluate  base64.b64decode('${content}[0][Value]').decode('utf-8')  modules=base64
-    Should Be Equal  ${secrets_key_consul_value}  ${secrets_value}
+    # Validate Secret Name and Secret Data
+    FOR  ${value}  IN  name  data
+        ${path}  Run Keyword If  "${value}" == "name"  Set Variable  SecretName
+                 ...    ELSE IF  "${value}" == "data"  Set Variable  SecretData/${secrets_key}
+        ${secrets_path}  Set Variable  ${service}/Writable/InsecureSecrets/${secrets_name}/${path}
+        Query Configuration On Registry Service  ${secrets_path}
+        ${name_base64_value}  Run Keyword If  "${REGISTRY_SERVICE}" == "Consul"  Set Variable  ${content}[0][Value]
+                         ...    ELSE IF  "${REGISTRY_SERVICE}" == "Keeper"  Set Variable  ${content}[response][0][value]
+        ${stored_value}  Evaluate  base64.b64decode('${name_base64_value}').decode('utf-8')  modules=base64
+        ${set_value}  Run Keyword If  "${value}" == "name"  Set Variable  ${secrets_name}
+                 ...    ELSE IF  "${value}" == "data"  Set Variable  ${secrets_value}
+        Should Be Equal  ${stored_value}  ${set_value}
+    END
 
 Should Return Status Code "${statusCode_0}" or "${statusCode_1}"
     Should Match Regexp  "${response}"  (${statusCode_0}|${statusCode_1})

TAF/testCaseModules/keywords/common/metrics.robot

@@ -6,18 +6,21 @@ Library      String
 Resource     TAF/testCaseModules/keywords/common/commonKeywords.robot
 Resource     TAF/testCaseModules/keywords/app-service/AppServiceAPI.robot
 
+*** Variables ***
+${APP_SERVICE_NAME}  app-sample
+
 *** Keywords ***
-Set Telemetry ${config}=${value} For ${service_name} On Consul
-    ${telemetry_path}  Set Variable  ${CONSUL_CONFIG_BASE_ENDPOINT}/${service_name}/Writable/Telemetry
+Set Telemetry ${config}=${value} For ${service_name} On Registry Service
+    ${telemetry_path}  Set Variable  /${service_name}/Writable/Telemetry
     ${path}  Set Variable   ${telemetry_path}/${config}
-    Update Service Configuration On Consul  ${path}  ${value}
+    Update Configuration On Registry Service  ${path}  ${value}
 
-Set Topics For app-samle PerTopicPipelines On Consul
+Set Topics For ${APP_SERVICE_NAME} PerTopicPipelines On Registry Service
     ${perTopics}  Create List  float  int8-16
-    ${path}  Set Variable  ${CONSUL_CONFIG_BASE_ENDPOINT}/app-sample/Writable/Pipeline/PerTopicPipelines
+    ${path}  Set Variable  /${APP_SERVICE_NAME}/Writable/Pipeline/PerTopicPipelines
     FOR  ${ITEM}  IN  @{perTopics}
         ${topics_path}  Set Variable  ${path}/${ITEM}/Topics
-        Update Service Configuration On Consul  ${topics_path}  edgex/events/device/+/+/${device_name}/#
+        Update Configuration On Registry Service  ${topics_path}  events/device/+/+/${device_name}/#
     END
 
 Metrics ${metrics_name} With ${field_name} Should Be Received
@@ -37,7 +40,7 @@ Metrics ${metrics_name} With ${field_name} Should Be Received
         ...        AND  Exit For Loop
     END
 
-Recieved Metrics ${metrics_name} For All Pipelines And ${field_name} Should Not Be 0
+Received Metrics ${metrics_name} For All Pipelines And ${field_name} Should Not Be 0
     Wait Until Keyword Succeeds  10x  1s  File Should Not Be Empty  ${WORK_DIR}/TAF/testArtifacts/logs/${subscriber_file}
     @{pipeline_ids}  Create List  default-pipeline  float-pipeline  int8-16-pipeline
     @{message_ids}  Create List
@@ -101,5 +104,5 @@ Get First Lines
     [Return]  ${list}
 
 Set PerTopicPipelines ${perTopicPipeline} ExecutionOrder ${functions}
-    ${path}=  Set variable  ${CONSUL_CONFIG_BASE_ENDPOINT}/app-sample/Writable/Pipeline/PerTopicPipelines/${perTopicPipeline}/ExecutionOrder
-    Update Service Configuration On Consul  ${path}  ${functions}
+    ${path}=  Set variable  /${APP_SERVICE_NAME}/Writable/Pipeline/PerTopicPipelines/${perTopicPipeline}/ExecutionOrder
+    Update Configuration On Registry Service  ${path}  ${functions}

TAF/testCaseModules/keywords/core-metadata/coreMetadataAPI.robot

@@ -18,7 +18,7 @@ ${deviceResourceUri}  /api/${API_VERSION}/deviceresource
 ${deviceUri}         /api/${API_VERSION}/device
 ${provisionWatcherUri}  /api/${API_VERSION}/provisionwatcher
 ${LOG_FILE_PATH}     ${WORK_DIR}/TAF/testArtifacts/logs/coreMetadataAPI.log
-${uomValidationPath}   ${CONSUL_CONFIG_BASE_ENDPOINT}/core-metadata/Writable/UoM/Validation
+${uomValidationPath}   /core-metadata/Writable/UoM/Validation
 @{device_service_list}  device-virtual  device-modbus  device-rest
 
 *** Keywords ***
@@ -762,9 +762,9 @@ Create A Provision Watcher Sample With Associated Test-Device-Service And ${devi
     Generate Provision Watchers  ${provisionwatcher}
     Create provision watcher ${provisionwatcher}
 
-Set ProfileChange.${config}=${value} For Core-Metadata On Consul
-   ${path}=  Set Variable  ${CONSUL_CONFIG_BASE_ENDPOINT}/core-metadata/Writable/ProfileChange/${config}
-   Update Service Configuration On Consul  ${path}  ${value}
+Set ProfileChange.${config}=${value} For Core-Metadata On Registry Service
+   ${path}=  Set Variable  /core-metadata/Writable/ProfileChange/${config}
+   Update Configuration On Registry Service  ${path}  ${value}
 
 ## UoM
 Query UoM

TAF/testCaseModules/keywords/setup/edgex.py

@@ -50,6 +50,7 @@ def restart_services(*args):
             SettingsInfo().constant.DEPLOY_TYPE)
     cmd = ["sh", script_path, *args]
     run_command(cmd)
+    checker.check_services_startup(list(args))
 
 
 def remove_services(*args):

TAF/testCaseModules/keywords/setup/startup_checker.py

@@ -45,7 +45,13 @@
                                     "pingUrl": "/api/{}/ping".format(SettingsInfo().constant.API_VERSION)},
         "app-service-mqtt-export": {"composeName": "app-service-mqtt-export",
                                     "port": SettingsInfo().constant.APP_MQTT_EXPORT_PORT,
-                                    "pingUrl": "/api/{}/ping".format(SettingsInfo().constant.API_VERSION)}
+                                    "pingUrl": "/api/{}/ping".format(SettingsInfo().constant.API_VERSION)},
+        "app-functional-tests": {"composeName": "app-functional-tests",
+                                 "port": SettingsInfo().constant.APP_FUNCTIONAL_TESTS_PORT,
+                                 "pingUrl": "/api/{}/ping".format(SettingsInfo().constant.API_VERSION)},
+        "app-sample": {"composeName": "app-sample",
+                       "port": SettingsInfo().constant.APP_SAMPLE_PORT,
+                       "pingUrl": "/api/{}/ping".format(SettingsInfo().constant.API_VERSION)}
     }
 
 httpConnTimeout = 5

TAF/testScenarios/functionalTest/API/app-service/info/GET.robot

@@ -7,7 +7,7 @@ Suite Teardown   Suite Teardown for App Service
 *** Variables ***
 ${SUITE}          App-Service GET Testcases
 ${LOG_FILE_PATH}  ${WORK_DIR}/TAF/testArtifacts/logs/app-service-get.log
-${AppServiceUrl_functional}  http://${BASE_URL}:${APP_FUNCTIOAL_TESTS_PORT}
+${AppServiceUrl_functional}  http://${BASE_URL}:${APP_FUNCTIONAL_TESTS_PORT}
 
 *** Test Cases ***
 InfoGET001 - Query ping

TAF/testScenarios/functionalTest/API/app-service/secrets/POST.robot

@@ -13,7 +13,7 @@ SecretsPOST001 - Stores secrets to the secret client with Path
     When Store Secret Data With Path
     Then Should Return Status Code "201"
     And Run Keyword If  $SECURITY_SERVICE_NEEDED == 'true'  Service ${app_service_name} Secrets Should be Stored
-        ...       ELSE  Secrets Should be Stored To Consul  ${app_service_name}
+        ...       ELSE  Secrets Should be Stored To Registry Service  ${app_service_name}
     And Response Time Should Be Less Than "${default_response_time_threshold}"ms
 
 ErrSecretsPOST001 - Stores secrets to the secret client fails (empty path)

TAF/testScenarios/functionalTest/API/app-service/trigger/POST-negative.robot

@@ -7,25 +7,28 @@ Suite Teardown   Suite Teardown for App Service
 *** Variables ***
 ${SUITE}          App-Service Trigger POST Negative Testcases
 ${LOG_FILE_PATH}  ${WORK_DIR}/TAF/testArtifacts/logs/app-service-trigger-negative.log
-${AppServiceUrl_functional}  http://${BASE_URL}:${APP_FUNCTIOAL_TESTS_PORT}
+${AppServiceUrl_functional}  http://${BASE_URL}:${APP_FUNCTIONAL_TESTS_PORT}
+${APP_SERVICE_NAME}  app-functional-tests
 
 *** Test Cases ***
 ErrTriggerPOST001 - Trigger pipeline fails (Invalid Data)
-    Given Set app-functional-tests Functions FilterByDeviceName, Transform, SetResponseData
+    Given Set ${APP_SERVICE_NAME} Functions FilterByDeviceName, Transform, SetResponseData
+    And Run Keyword If  "${REGISTRY_SERVICE}" == "Keeper"  Restart Services  ${APP_SERVICE_NAME}
     When Trigger Function Pipeline With Invalid Data
     Then Should Return Status Code "400"
     And Response Time Should Be Less Than "${default_response_time_threshold}"ms
 
 ErrTriggerPOST002 - Trigger pipeline fails (Unprocessable Entity)
-    Given Set app-functional-tests Functions FilterByDeviceName, Transform, SetResponseData
+    Given Set ${APP_SERVICE_NAME} Functions FilterByDeviceName, Transform, SetResponseData
     And Update Target Type To raw
+    And Run Keyword If  "${REGISTRY_SERVICE}" == "Keeper"  Restart Services  ${APP_SERVICE_NAME}
     When Trigger Function Pipeline With Invalid Data
     Then Should Return Status Code "422"
     And Response Time Should Be Less Than "${default_response_time_threshold}"ms
     [Teardown]  Update Target Type To event
 
 *** Keywords ***
 Update Target Type To ${value}
-    ${path}=  Set variable  ${CONSUL_CONFIG_BASE_ENDPOINT}/app-functional-tests/Writable/Pipeline/TargetType
-    Update Service Configuration On Consul  ${path}  ${value}
+    ${path}=  Set variable  /app-functional-tests/Writable/Pipeline/TargetType
+    Update Configuration On Registry Service  ${path}  ${value}