From 8eee9b8ed8cdaaa616902000db548a87856a8298 Mon Sep 17 00:00:00 2001 From: Igor Abdrakhimov Date: Wed, 5 Jun 2024 10:56:15 -0700 Subject: [PATCH 01/11] Working IPC --- .../ci_run_greengrass_discovery_cfg.json | 32 ++- samples/ipc_greengrass.py | 56 ++--- test/greengrass/ipc/copy_files.sh | 3 + test/greengrass/ipc/gdk-config.json | 22 ++ test/greengrass/ipc/gg-e2e-tests/pom.xml | 63 ++++++ .../greengrass/features/component.feature | 11 + test/greengrass/ipc/recipe.yaml | 28 +++ utils/run_in_ci.py | 196 +++++++++++------- 8 files changed, 296 insertions(+), 115 deletions(-) create mode 100644 test/greengrass/ipc/copy_files.sh create mode 100644 test/greengrass/ipc/gdk-config.json create mode 100644 test/greengrass/ipc/gg-e2e-tests/pom.xml create mode 100644 test/greengrass/ipc/gg-e2e-tests/src/main/resources/greengrass/features/component.feature create mode 100644 test/greengrass/ipc/recipe.yaml diff --git a/.github/workflows/ci_run_greengrass_discovery_cfg.json b/.github/workflows/ci_run_greengrass_discovery_cfg.json index 5e67b26c..b89c1240 100644 --- a/.github/workflows/ci_run_greengrass_discovery_cfg.json +++ b/.github/workflows/ci_run_greengrass_discovery_cfg.json @@ -1,39 +1,35 @@ { - "language": "Python", - "sample_file": "./aws-iot-device-sdk-python-v2/samples/basic_discovery.py", - "sample_region": "us-east-1", - "sample_main_class": "", + "language": "CPP", + "runnable_file": "basic-discovery", + "runnable_region": "us-east-1", + "runnable_main_class": "", "arguments": [ { "name": "--cert", - "secret": "ci/Greengrass/cert", + "secret": "ci/GreengrassDiscovery/cert", "filename": "tmp_certificate.pem" }, { "name": "--key", - "secret": "ci/Greengrass/key", + "secret": "ci/GreengrassDiscovery/key", "filename": "tmp_key.pem" }, { - "name": "--ca_file", - "secret": "ci/Greengrass/ca", - "filename": "tmp_ca.pem" + "name": "--thing_name", + "data": "CI_Greengrass_Discovery_Thing" }, { "name": "--region", "data": "us-east-1" }, { - "name": "--thing_name", - "data": "CI_GreenGrass_Thing" - }, - { - "name": "--is_ci", - "data": "true" + "name": "--topic", + "data": "clients/CI_Greengrass_Discovery_Thing/hello/world/$INPUT_UUID" }, { - "name": "--print_discover_resp_only", - "data": "" + "name": "--mode", + "data": "publish" } - ] + ], + "stdin_file": "messages.txt" } diff --git a/samples/ipc_greengrass.py b/samples/ipc_greengrass.py index a8aeaa4e..92db3bd0 100644 --- a/samples/ipc_greengrass.py +++ b/samples/ipc_greengrass.py @@ -22,33 +22,39 @@ import json import time import os +import sys import awsiot.greengrasscoreipc import awsiot.greengrasscoreipc.model as model if __name__ == '__main__': - ipc_client = awsiot.greengrasscoreipc.connect() - - while True: - telemetry_data = { - "timestamp": int(round(time.time() * 1000)), - "battery_state_of_charge": 42.5, - "location": { - "longitude": 48.15743, - "latitude": 11.57549, - }, - } - - op = ipc_client.new_publish_to_iot_core() - op.activate(model.PublishToIoTCoreRequest( - topic_name="my/iot/{}/telemetry".format(os.getenv("AWS_IOT_THING_NAME")), - qos=model.QOS.AT_LEAST_ONCE, - payload=json.dumps(telemetry_data).encode(), - )) - try: - result = op.get_response().result(timeout=5.0) - print("successfully published message:", result) - except Exception as e: - print("failed to publish message:", e) - - time.sleep(5) + try: + print("Connecting to Greengrass...") + ipc_client = awsiot.greengrasscoreipc.connect() + + for i in range(0, 10): + telemetry_data = { + "timestamp": int(round(time.time() * 1000)), + "battery_state_of_charge": 42.5, + "location": { + "longitude": 48.15743, + "latitude": 11.57549, + }, + } + + op = ipc_client.new_publish_to_iot_core() + op.activate(model.PublishToIoTCoreRequest( + topic_name="my/iot/{}/telemetry".format(os.getenv("AWS_IOT_THING_NAME")), + qos=model.QOS.AT_LEAST_ONCE, + payload=json.dumps(telemetry_data).encode(), + )) + try: + result = op.get_response().result(timeout=5.0) + print("Successfully published message: {}".format(result)) + except Exception as e: + print("Failed to publish message: {}".format(e)) + sys.exit(1) + + except Exception as e: + print("Sample failed: {}".format(e)) + sys.exit(1) diff --git a/test/greengrass/ipc/copy_files.sh b/test/greengrass/ipc/copy_files.sh new file mode 100644 index 00000000..eb0a8ef6 --- /dev/null +++ b/test/greengrass/ipc/copy_files.sh @@ -0,0 +1,3 @@ +cp ../../../samples/ipc_greengrass.py . +cp ../../../utils/run_in_ci.py . +cp ../../../.github/workflows/ci_run_greengrass_ipc_cfg.json . diff --git a/test/greengrass/ipc/gdk-config.json b/test/greengrass/ipc/gdk-config.json new file mode 100644 index 00000000..92222370 --- /dev/null +++ b/test/greengrass/ipc/gdk-config.json @@ -0,0 +1,22 @@ +{ + "component": { + "software.amazon.awssdk.sdk-gg-ipc": { + "author": "iot-device-sdk", + "version": "NEXT_PATCH", + "build": { + "build_system": "custom", + "custom_build_command": ["bash", "copy_files.sh"] + }, + "publish": { + "bucket": "", + "region": "" + } + } + }, + "gdk_version": "1.3.0", + "test-e2e": { + "gtf_options": { + "tags": "testgg" + } + } +} diff --git a/test/greengrass/ipc/gg-e2e-tests/pom.xml b/test/greengrass/ipc/gg-e2e-tests/pom.xml new file mode 100644 index 00000000..460832fc --- /dev/null +++ b/test/greengrass/ipc/gg-e2e-tests/pom.xml @@ -0,0 +1,63 @@ + + + 4.0.0 + + com.aws.greengrass + uat-features + jar + 1.0.0 + OTF + + + 1.2.0-SNAPSHOT + 1.8 + 1.8 + + + + + greengrass-common + greengrass common + + https://d2jrmugq4soldf.cloudfront.net/snapshots + + + + + + + com.aws.greengrass + aws-greengrass-testing-standalone + ${otf.version} + compile + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.2.2 + + + package + + shade + + + + + + + com.aws.greengrass.testing.launcher.TestLauncher + + + + + + + + + diff --git a/test/greengrass/ipc/gg-e2e-tests/src/main/resources/greengrass/features/component.feature b/test/greengrass/ipc/gg-e2e-tests/src/main/resources/greengrass/features/component.feature new file mode 100644 index 00000000..655f9880 --- /dev/null +++ b/test/greengrass/ipc/gg-e2e-tests/src/main/resources/greengrass/features/component.feature @@ -0,0 +1,11 @@ +Feature: Testing features of Greengrassv2 IPC sample + + @testgg + Scenario: As a developer, I can create a component and deploy it on my device + Given my device is registered as a Thing + And my device is running Greengrass + When I create a Greengrass deployment with components + | software.amazon.awssdk.sdk-gg-ipc | file:recipe.yaml | + And I deploy the Greengrass deployment configuration + Then the Greengrass deployment is COMPLETED on the device after 180 seconds + And the software.amazon.awssdk.sdk-gg-ipc log on the device contains the line "Successfully published message" within 20 seconds diff --git a/test/greengrass/ipc/recipe.yaml b/test/greengrass/ipc/recipe.yaml new file mode 100644 index 00000000..7aee24fe --- /dev/null +++ b/test/greengrass/ipc/recipe.yaml @@ -0,0 +1,28 @@ +--- +RecipeFormatVersion: "2020-01-25" +ComponentName: software.amazon.awssdk.sdk-gg-ipc +ComponentVersion: "1.0.0" +ComponentDescription: "This is test for the Greengrass IPC sample" +ComponentPublisher: "iot-device-sdk" +ComponentConfiguration: + DefaultConfiguration: + accessControl: + aws.greengrass.ipc.mqttproxy: + software.amazon.awssdk.sdk-gg-ipc:mqttproxy:1: + policyDescription: "Allows access to publish and subscribe to a Greengrass IPC test topic" + operations: + - aws.greengrass#PublishToIoTCore + - aws.greengrass#SubscribeToIoTCore + resources: + - "my/iot/*/telemetry" +Manifests: + - Platform: + os: all + Artifacts: + - URI: "file:ipc_greengrass.py" + - URI: "file:run_in_ci.py" + - URI: "file:ci_run_greengrass_ipc_cfg.json" + Lifecycle: + Run: | + echo "GG core:" {iot:thingName} + python3 {artifacts:path}/run_in_ci.py --runnable_dir {artifacts:path} --file {artifacts:path}/ci_run_greengrass_ipc_cfg.json diff --git a/utils/run_in_ci.py b/utils/run_in_ci.py index a999d2de..d2eea2ea 100644 --- a/utils/run_in_ci.py +++ b/utils/run_in_ci.py @@ -33,7 +33,8 @@ def setup_json_arguments_list(file, input_uuid=None): for argument in config_json['arguments']: # Add the name of the argument - config_json_arguments_list.append(argument['name']) + if( 'name' in argument): + config_json_arguments_list.append(argument['name']) # Based on the data present, we need to process and add the data differently try: @@ -147,7 +148,18 @@ def make_windows_pfx_file(certificate_file_path, private_key_path, pfx_file_path # Import the PFX into the Windows Certificate Store # (Passing '$mypwd' is required even though it is empty and our certificate has no password. It fails CI otherwise) - import_pfx_arguments = ["powershell.exe", "Import-PfxCertificate", "-FilePath", pfx_file_path, "-CertStoreLocation", "Cert:\\" + pfx_certificate_store_location, "-Password", "$mypwd"] + import_pfx_arguments = [ + "powershell.exe", + # Powershell 7.3 introduced an issue where launching powershell from cmd would not set PSModulePath correctly. + # As a workaround, we set `PSModulePath` to empty so powershell would automatically reset the PSModulePath to default. + # More details: https://github.com/PowerShell/PowerShell/issues/18530 + "$env:PSModulePath = '';", + "Import-PfxCertificate", + "-FilePath", pfx_file_path, + "-CertStoreLocation", + "Cert:\\" + pfx_certificate_store_location, + "-Password", + "$mypwd"] import_pfx_run = subprocess.run(args=import_pfx_arguments, shell=True, stdout=subprocess.PIPE) if (import_pfx_run.returncode != 0): print ("ERROR: Could not import PFX certificate into Windows store!") @@ -216,7 +228,8 @@ def cleanup_runnable(): global config_json_arguments_list for argument in config_json['arguments']: - config_json_arguments_list.append(argument['name']) + if( 'name' in argument): + config_json_arguments_list.append(argument['name']) # Based on the data present, we need to process and add the data differently try: @@ -242,7 +255,7 @@ def cleanup_runnable(): return -1 -def launch_runnable(): +def launch_runnable(runnable_dir): global config_json global config_json_arguments_list @@ -250,92 +263,128 @@ def launch_runnable(): print("No configuration JSON file data found!") return -1 + # Prepare data for runnable's STDIN + subprocess_stdin = None + if "stdin_file" in config_json: + stdin_file = os.path.join(runnable_dir, config_json['stdin_file']) + with open(stdin_file, "rb") as file: + subprocess_stdin = file.read() + exit_code = 0 + runnable_timeout = None + if ('timeout' in config_json): + runnable_timeout = config_json['timeout'] + print("Launching runnable...") - # Java - if (config_json['language'] == "Java"): - - # Flatten arguments down into a single string - arguments_as_string = "" - for i in range(0, len(config_json_arguments_list)): - arguments_as_string += str(config_json_arguments_list[i]) - if (i+1 < len(config_json_arguments_list)): - arguments_as_string += " " - - arguments = ["mvn", "compile", "exec:java"] - arguments.append("-pl") - arguments.append(config_json['runnable_file']) - arguments.append("-Dexec.mainClass=" + config_json['runnable_main_class']) - arguments.append("-Daws.crt.ci=True") - - # We have to do this as a string, unfortunately, due to how -Dexec.args= works... - argument_string = subprocess.list2cmdline(arguments) + " -Dexec.args=\"" + arguments_as_string + "\"" - print(f"Running cmd: {argument_string}") - runnable_return = subprocess.run(argument_string, shell=True) - exit_code = runnable_return.returncode - - # C++ - elif (config_json['language'] == "CPP"): - runnable_return = subprocess.run( - args=config_json_arguments_list, executable=config_json['runnable_file']) - exit_code = runnable_return.returncode - - elif (config_json['language'] == "Python"): - config_json_arguments_list.append("--is_ci") - config_json_arguments_list.append("True") - - runnable_return = subprocess.run( - args=[sys.executable, config_json['runnable_file']] + config_json_arguments_list) - exit_code = runnable_return.returncode - - elif (config_json['language'] == "Javascript"): - os.chdir(config_json['runnable_file']) - - config_json_arguments_list.append("--is_ci") - config_json_arguments_list.append("true") - - runnable_return_one = None - if sys.platform == "win32" or sys.platform == "cygwin": - runnable_return_one = subprocess.run(args=["npm", "install"], shell=True) - else: - runnable_return_one = subprocess.run(args=["npm", "install"]) + try: + # Java + if (config_json['language'] == "Java"): + # Flatten arguments down into a single string + arguments_as_string = "" + for i in range(0, len(config_json_arguments_list)): + arguments_as_string += str(config_json_arguments_list[i]) + if (i+1 < len(config_json_arguments_list)): + arguments_as_string += " " + + arguments = ["mvn", "compile", "exec:java"] + arguments.append("-pl") + arguments.append(config_json['runnable_file']) + arguments.append("-Dexec.mainClass=" + config_json['runnable_main_class']) + arguments.append("-Daws.crt.ci=True") + + # We have to do this as a string, unfortunately, due to how -Dexec.args= works... + argument_string = subprocess.list2cmdline(arguments) + " -Dexec.args=\"" + arguments_as_string + "\"" + print(f"Running cmd: {argument_string}") + runnable_return = subprocess.run(argument_string, input=subprocess_stdin, timeout=runnable_timeout, shell=True) + exit_code = runnable_return.returncode + + elif (config_json['language'] == "Java JAR"): + # Flatten arguments down into a single string + arguments_as_string = "" + for i in range(0, len(config_json_arguments_list)): + arguments_as_string += str(config_json_arguments_list[i]) + if (i+1 < len(config_json_arguments_list)): + arguments_as_string += " " + + runnable_file = os.path.join(runnable_dir, config_json['runnable_file']) + + arguments = ["java"] + arguments.append("-Daws.crt.ci=True") + arguments.append("-jar") + arguments.append(runnable_file) + + argument_string = subprocess.list2cmdline(arguments) + " " + arguments_as_string + print(f"Running cmd: {argument_string}") + runnable_return = subprocess.run(argument_string, input=subprocess_stdin, timeout=runnable_timeout, shell=True) + exit_code = runnable_return.returncode + + # C++ + elif (config_json['language'] == "CPP"): + runnable_file = os.path.join(runnable_dir, config_json['runnable_file']) + runnable_return = subprocess.run(args=config_json_arguments_list, input=subprocess_stdin, timeout=runnable_timeout, executable=runnable_file) + exit_code = runnable_return.returncode + + elif (config_json['language'] == "Python"): + config_json_arguments_list.append("--is_ci") + config_json_arguments_list.append("True") + + runnable_file = os.path.join(runnable_dir, config_json['runnable_file']) + runnable_return = subprocess.run( + args=[sys.executable, runnable_file] + config_json_arguments_list, input=subprocess_stdin, timeout=runnable_timeout) + exit_code = runnable_return.returncode + + elif (config_json['language'] == "Javascript"): + os.chdir(config_json['runnable_file']) + + config_json_arguments_list.append("--is_ci") + config_json_arguments_list.append("true") + + runnable_return_one = None + if not 'skip_install' in config_json: + if sys.platform == "win32" or sys.platform == "cygwin": + runnable_return_one = subprocess.run(args=["npm", "install"], shell=True, timeout=runnable_timeout) + else: + runnable_return_one = subprocess.run(args=["npm", "install"], timeout=runnable_timeout) - if (runnable_return_one == None or runnable_return_one.returncode != 0): - exit_code = runnable_return_one.returncode - else: - runnable_return_two = None - arguments = [] - if 'node_cmd' in config_json: - arguments = config_json['node_cmd'].split(" ") + if not 'skip_install' in config_json and (runnable_return_one == None or runnable_return_one.returncode != 0): + exit_code = runnable_return_one.returncode else: - arguments = ["node", "dist/index.js"] + runnable_return_two = None + arguments = [] + if 'node_cmd' in config_json: + arguments = config_json['node_cmd'].split(" ") + else: + arguments = ["node", "dist/index.js"] - if sys.platform == "win32" or sys.platform == "cygwin": - runnable_return_two = subprocess.run( - args=arguments + config_json_arguments_list, shell=True) - else: - runnable_return_two = subprocess.run( - args=arguments + config_json_arguments_list) + if sys.platform == "win32" or sys.platform == "cygwin": + runnable_return_two = subprocess.run( + args=arguments + config_json_arguments_list, shell=True, check=True, timeout=runnable_timeout) + else: + runnable_return_two = subprocess.run( + args=arguments + config_json_arguments_list, input=subprocess_stdin, timeout=runnable_timeout) - if (runnable_return_two != None): - exit_code = runnable_return_two.returncode - else: - exit_code = 1 + if (runnable_return_two != None): + exit_code = runnable_return_two.returncode + else: + exit_code = 1 + except subprocess.CalledProcessError as e: + print(e.output) + exit_code = 1 cleanup_runnable() return exit_code -def setup_and_launch(file, input_uuid=None): +def setup_and_launch(file, input_uuid=None, runnable_dir=''): setup_result = setup_runnable(file, input_uuid) if setup_result != 0: print("Setting up runnable failed") return setup_result print("About to launch runnable...") - return launch_runnable() + return launch_runnable(runnable_dir) def main(): @@ -344,13 +393,16 @@ def main(): argument_parser.add_argument("--file", required=True, help="Configuration file to pull CI data from") argument_parser.add_argument("--input_uuid", required=False, help="UUID data to replace '$INPUT_UUID' with. Only works in Data field") + argument_parser.add_argument("--runnable_dir", required=False, default='', + help="Directory where runnable_file is located") parsed_commands = argument_parser.parse_args() file = parsed_commands.file input_uuid = parsed_commands.input_uuid + runnable_dir = parsed_commands.runnable_dir print(f"Starting to launch runnable: config {file}; input UUID: {input_uuid}") - test_result = setup_and_launch(file, input_uuid) + test_result = setup_and_launch(file, input_uuid, runnable_dir) sys.exit(test_result) From b83f16b1d1a178a07c72e51a7de2e1323290749f Mon Sep 17 00:00:00 2001 From: Igor Abdrakhimov Date: Wed, 5 Jun 2024 11:00:24 -0700 Subject: [PATCH 02/11] Enable GG in CI --- .github/workflows/ci.yml | 60 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a2729e83..f4de533c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,6 +29,7 @@ env: CI_JOBS_ROLE: arn:aws:iam::180635532705:role/CI_Jobs_Role CI_FLEET_PROVISIONING_ROLE: arn:aws:iam::180635532705:role/service-role/CI_FleetProvisioning_Role CI_GREENGRASS_ROLE: arn:aws:iam::180635532705:role/CI_Greengrass_Role + CI_GREENGRASS_INSTALLER_ROLE: arn:aws:iam::180635532705:role/CI_GreengrassInstaller_Role CI_DEVICE_ADVISOR: arn:aws:iam::180635532705:role/CI_DeviceAdvisor_Role CI_MQTT5_ROLE: arn:aws:iam::180635532705:role/CI_MQTT5_Role CI_BUILD_AND_TEST_ROLE: arn:aws:iam::180635532705:role/V2_SDK_Unit_Testing @@ -402,6 +403,65 @@ jobs: run: | python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --file ${{ env.CI_SAMPLES_CFG_FOLDER }}/ci_run_greengrass_discovery_cfg.json + linux-smoke-tests: + runs-on: ubuntu-22.04 + permissions: + id-token: write # This is required for requesting the JWT + steps: + - name: Setup environment + run: | + python3 -m pip install boto3 + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: ${{ env.CI_BUILD_AND_TEST_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: Build ${{ env.PACKAGE_NAME }} + run: | + python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" + chmod a+x builder + ./builder build -p ${{ env.PACKAGE_NAME }} + - name: Install Greengrass Development Kit + run: | + python3 -m pip install awsiotsdk + python3 -m pip install -U git+https://github.com/aws-greengrass/aws-greengrass-gdk-cli.git@v1.6.2 + # - name: Configure AWS credentials (Greengrass) + # uses: aws-actions/configure-aws-credentials@v2 + # with: + # role-to-assume: ${{ env.CI_GREENGRASS_INSTALLER_ROLE }} + # aws-region: ${{ env.AWS_DEFAULT_REGION }} + # - name: Build and run Greengrass basic discovery sample + # working-directory: ./aws-iot-device-sdk-python-v2/tests/greengrass/basic_discovery + # run: | + # gdk component build + # gdk test-e2e build + # gdk test-e2e run + # - name: Show logs + # working-directory: ./aws-iot-device-sdk-python-v2/tests/greengrass/basic_discovery + # # Print logs unconditionally to provide more details on Greengrass run even if the test failed. + # if: always() + # run: | + # echo "=== greengrass.log" + # cat testResults/gg*/greengrass.log + # echo "=== software.amazon.awssdk.sdk-gg-test-discovery.log" + # cat testResults/gg*/software.amazon.awssdk.sdk-gg-test-discovery.log + - name: Build and run Greengrass IPC sample + working-directory: ./aws-iot-device-sdk-python-v2/tests/greengrass/ipc + run: | + gdk component build + gdk test-e2e build + gdk test-e2e run + - name: Show logs + working-directory: ./aws-iot-device-sdk-python-v2/tests/greengrass/ipc + # Print logs unconditionally to provide more details on Greengrass run even if the test failed. + if: always() + run: | + echo "=== greengrass.log" + cat testResults/gg*/greengrass.log + echo "=== software.amazon.awssdk.sdk-gg-ipc.log" + cat testResults/gg*/software.amazon.awssdk.sdk-gg-ipc.log + + # check that docs can still build check-docs: runs-on: ubuntu-22.04 # latest From 9022f3d04bda43fc1ad068bfa485e166c31d764d Mon Sep 17 00:00:00 2001 From: Igor Abdrakhimov Date: Wed, 5 Jun 2024 11:01:47 -0700 Subject: [PATCH 03/11] Fix CI job name --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f4de533c..acecf0d1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -403,7 +403,7 @@ jobs: run: | python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --file ${{ env.CI_SAMPLES_CFG_FOLDER }}/ci_run_greengrass_discovery_cfg.json - linux-smoke-tests: + linux-greengrass-tests: runs-on: ubuntu-22.04 permissions: id-token: write # This is required for requesting the JWT From 92083f5b5f9cef51aac1ab10afff9fc819040f44 Mon Sep 17 00:00:00 2001 From: Igor Abdrakhimov Date: Wed, 5 Jun 2024 11:03:41 -0700 Subject: [PATCH 04/11] Fix working dir --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index acecf0d1..7813d5c5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -431,13 +431,13 @@ jobs: # role-to-assume: ${{ env.CI_GREENGRASS_INSTALLER_ROLE }} # aws-region: ${{ env.AWS_DEFAULT_REGION }} # - name: Build and run Greengrass basic discovery sample - # working-directory: ./aws-iot-device-sdk-python-v2/tests/greengrass/basic_discovery + # working-directory: ./aws-iot-device-sdk-python-v2/test/greengrass/basic_discovery # run: | # gdk component build # gdk test-e2e build # gdk test-e2e run # - name: Show logs - # working-directory: ./aws-iot-device-sdk-python-v2/tests/greengrass/basic_discovery + # working-directory: ./aws-iot-device-sdk-python-v2/test/greengrass/basic_discovery # # Print logs unconditionally to provide more details on Greengrass run even if the test failed. # if: always() # run: | @@ -446,13 +446,13 @@ jobs: # echo "=== software.amazon.awssdk.sdk-gg-test-discovery.log" # cat testResults/gg*/software.amazon.awssdk.sdk-gg-test-discovery.log - name: Build and run Greengrass IPC sample - working-directory: ./aws-iot-device-sdk-python-v2/tests/greengrass/ipc + working-directory: ./aws-iot-device-sdk-python-v2/test/greengrass/ipc run: | gdk component build gdk test-e2e build gdk test-e2e run - name: Show logs - working-directory: ./aws-iot-device-sdk-python-v2/tests/greengrass/ipc + working-directory: ./aws-iot-device-sdk-python-v2/test/greengrass/ipc # Print logs unconditionally to provide more details on Greengrass run even if the test failed. if: always() run: | From 71fe27eb70536f178b1899572da2c448b49f92f6 Mon Sep 17 00:00:00 2001 From: Igor Abdrakhimov Date: Wed, 5 Jun 2024 11:05:35 -0700 Subject: [PATCH 05/11] Add missing config file --- .../workflows/ci_run_greengrass_ipc_cfg.json | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 .github/workflows/ci_run_greengrass_ipc_cfg.json diff --git a/.github/workflows/ci_run_greengrass_ipc_cfg.json b/.github/workflows/ci_run_greengrass_ipc_cfg.json new file mode 100644 index 00000000..7680faba --- /dev/null +++ b/.github/workflows/ci_run_greengrass_ipc_cfg.json @@ -0,0 +1,21 @@ +{ + "language": "Python", + "runnable_file": "ipc_greengrass.py", + "runnable_region": "us-east-1", + "runnable_main_class": "", + "arguments": [ + { + "name": "--topic", + "data": "test/gg-ipc-topic" + }, + { + "name": "--message", + "data": "hello" + }, + { + "name": "--is_ci", + "data": "true" + } + + ] +} From 6a1e7c7663344d5ad17c8b273d708829d18282ce Mon Sep 17 00:00:00 2001 From: Igor Abdrakhimov Date: Wed, 5 Jun 2024 11:08:52 -0700 Subject: [PATCH 06/11] Fix creds --- .github/workflows/ci.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7813d5c5..fe94509c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -425,11 +425,11 @@ jobs: run: | python3 -m pip install awsiotsdk python3 -m pip install -U git+https://github.com/aws-greengrass/aws-greengrass-gdk-cli.git@v1.6.2 - # - name: Configure AWS credentials (Greengrass) - # uses: aws-actions/configure-aws-credentials@v2 - # with: - # role-to-assume: ${{ env.CI_GREENGRASS_INSTALLER_ROLE }} - # aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: Configure AWS credentials (Greengrass) + uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: ${{ env.CI_GREENGRASS_INSTALLER_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} # - name: Build and run Greengrass basic discovery sample # working-directory: ./aws-iot-device-sdk-python-v2/test/greengrass/basic_discovery # run: | From f1b5d08d126c0d534cd6304e9ae0cf519b717c2c Mon Sep 17 00:00:00 2001 From: Igor Abdrakhimov Date: Wed, 5 Jun 2024 12:53:11 -0700 Subject: [PATCH 07/11] Add discovery --- .github/workflows/ci.yml | 31 ++++---- .../workflows/ci_run_basic_connect_cfg.json | 4 + .../workflows/ci_run_cognito_connect_cfg.json | 4 + .../ci_run_custom_authorizer_connect_cfg.json | 4 + .../ci_run_fleet_provisioning_cfg.json | 4 + .../ci_run_greengrass_discovery_cfg.json | 7 +- .../workflows/ci_run_greengrass_ipc_cfg.json | 1 - .github/workflows/ci_run_jobs_cfg.json | 4 + .../ci_run_mqtt5_custom_authorizer_cfg.json | 4 + ...qtt5_custom_authorizer_websockets_cfg.json | 4 + .../ci_run_mqtt5_fleet_provisioning_cfg.json | 4 + .github/workflows/ci_run_mqtt5_jobs_cfg.json | 4 + .../ci_run_mqtt5_pkcs11_connect_cfg.json | 4 + .../workflows/ci_run_pkcs11_connect_cfg.json | 4 + .../workflows/ci_run_pkcs12_connect_cfg.json | 4 + .../ci_run_websocket_connect_cfg.json | 4 + .../ci_run_windows_cert_connect_cfg.json | 4 + .../workflows/ci_run_x509_connect_cfg.json | 4 + samples/basic_discovery.py | 6 +- test/greengrass/basic_discovery/copy_files.sh | 3 + .../basic_discovery/gdk-config.json | 22 ++++++ .../basic_discovery/gg-e2e-tests/pom.xml | 63 ++++++++++++++++ .../greengrass/features/component.feature | 74 +++++++++++++++++++ .../basic_discovery/hello_world_subscriber.py | 55 ++++++++++++++ test/greengrass/basic_discovery/recipe.yaml | 45 +++++++++++ utils/run_in_ci.py | 3 - 26 files changed, 344 insertions(+), 26 deletions(-) create mode 100644 test/greengrass/basic_discovery/copy_files.sh create mode 100644 test/greengrass/basic_discovery/gdk-config.json create mode 100644 test/greengrass/basic_discovery/gg-e2e-tests/pom.xml create mode 100644 test/greengrass/basic_discovery/gg-e2e-tests/src/main/resources/greengrass/features/component.feature create mode 100644 test/greengrass/basic_discovery/hello_world_subscriber.py create mode 100644 test/greengrass/basic_discovery/recipe.yaml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fe94509c..c98181bf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -430,21 +430,22 @@ jobs: with: role-to-assume: ${{ env.CI_GREENGRASS_INSTALLER_ROLE }} aws-region: ${{ env.AWS_DEFAULT_REGION }} - # - name: Build and run Greengrass basic discovery sample - # working-directory: ./aws-iot-device-sdk-python-v2/test/greengrass/basic_discovery - # run: | - # gdk component build - # gdk test-e2e build - # gdk test-e2e run - # - name: Show logs - # working-directory: ./aws-iot-device-sdk-python-v2/test/greengrass/basic_discovery - # # Print logs unconditionally to provide more details on Greengrass run even if the test failed. - # if: always() - # run: | - # echo "=== greengrass.log" - # cat testResults/gg*/greengrass.log - # echo "=== software.amazon.awssdk.sdk-gg-test-discovery.log" - # cat testResults/gg*/software.amazon.awssdk.sdk-gg-test-discovery.log + - name: Build and run Greengrass basic discovery sample + working-directory: ./aws-iot-device-sdk-python-v2/test/greengrass/basic_discovery + run: | + export PYTHONPATH=${{ github.workspace }}/aws-iot-device-sdk-python-v2/samples + gdk component build + gdk test-e2e build + gdk test-e2e run + - name: Show logs + working-directory: ./aws-iot-device-sdk-python-v2/test/greengrass/basic_discovery + # Print logs unconditionally to provide more details on Greengrass run even if the test failed. + if: always() + run: | + echo "=== greengrass.log" + cat testResults/gg*/greengrass.log + echo "=== software.amazon.awssdk.sdk-gg-test-discovery.log" + cat testResults/gg*/software.amazon.awssdk.sdk-gg-test-discovery.log - name: Build and run Greengrass IPC sample working-directory: ./aws-iot-device-sdk-python-v2/test/greengrass/ipc run: | diff --git a/.github/workflows/ci_run_basic_connect_cfg.json b/.github/workflows/ci_run_basic_connect_cfg.json index fae564c2..8b3beccc 100644 --- a/.github/workflows/ci_run_basic_connect_cfg.json +++ b/.github/workflows/ci_run_basic_connect_cfg.json @@ -17,6 +17,10 @@ "name": "--key", "secret": "ci/PubSub/key", "filename": "tmp_key.pem" + }, + { + "name": "--is_ci", + "data": "true" } ] } diff --git a/.github/workflows/ci_run_cognito_connect_cfg.json b/.github/workflows/ci_run_cognito_connect_cfg.json index 1f6b6cd1..06294750 100644 --- a/.github/workflows/ci_run_cognito_connect_cfg.json +++ b/.github/workflows/ci_run_cognito_connect_cfg.json @@ -15,6 +15,10 @@ { "name": "--cognito_identity", "secret": "ci/Cognito/identity_id" + }, + { + "name": "--is_ci", + "data": "true" } ] } diff --git a/.github/workflows/ci_run_custom_authorizer_connect_cfg.json b/.github/workflows/ci_run_custom_authorizer_connect_cfg.json index cbd9afa9..3921e89f 100644 --- a/.github/workflows/ci_run_custom_authorizer_connect_cfg.json +++ b/.github/workflows/ci_run_custom_authorizer_connect_cfg.json @@ -15,6 +15,10 @@ { "name": "--custom_auth_password", "secret": "ci/CustomAuthorizer/password" + }, + { + "name": "--is_ci", + "data": "true" } ] } diff --git a/.github/workflows/ci_run_fleet_provisioning_cfg.json b/.github/workflows/ci_run_fleet_provisioning_cfg.json index 0370102f..7068e745 100644 --- a/.github/workflows/ci_run_fleet_provisioning_cfg.json +++ b/.github/workflows/ci_run_fleet_provisioning_cfg.json @@ -25,6 +25,10 @@ { "name": "--template_parameters", "data": "{\"SerialNumber\":\"$INPUT_UUID\"}" + }, + { + "name": "--is_ci", + "data": "true" } ] } diff --git a/.github/workflows/ci_run_greengrass_discovery_cfg.json b/.github/workflows/ci_run_greengrass_discovery_cfg.json index b89c1240..34c389ce 100644 --- a/.github/workflows/ci_run_greengrass_discovery_cfg.json +++ b/.github/workflows/ci_run_greengrass_discovery_cfg.json @@ -1,6 +1,6 @@ { - "language": "CPP", - "runnable_file": "basic-discovery", + "language": "Python", + "runnable_file": "basic_discovery.py", "runnable_region": "us-east-1", "runnable_main_class": "", "arguments": [ @@ -30,6 +30,5 @@ "name": "--mode", "data": "publish" } - ], - "stdin_file": "messages.txt" + ] } diff --git a/.github/workflows/ci_run_greengrass_ipc_cfg.json b/.github/workflows/ci_run_greengrass_ipc_cfg.json index 7680faba..767b98df 100644 --- a/.github/workflows/ci_run_greengrass_ipc_cfg.json +++ b/.github/workflows/ci_run_greengrass_ipc_cfg.json @@ -16,6 +16,5 @@ "name": "--is_ci", "data": "true" } - ] } diff --git a/.github/workflows/ci_run_jobs_cfg.json b/.github/workflows/ci_run_jobs_cfg.json index bb7300d2..8df922b2 100644 --- a/.github/workflows/ci_run_jobs_cfg.json +++ b/.github/workflows/ci_run_jobs_cfg.json @@ -21,6 +21,10 @@ { "name": "--thing_name", "data": "CI_Jobs_Thing" + }, + { + "name": "--is_ci", + "data": "true" } ] } diff --git a/.github/workflows/ci_run_mqtt5_custom_authorizer_cfg.json b/.github/workflows/ci_run_mqtt5_custom_authorizer_cfg.json index f3608180..1325b5a6 100644 --- a/.github/workflows/ci_run_mqtt5_custom_authorizer_cfg.json +++ b/.github/workflows/ci_run_mqtt5_custom_authorizer_cfg.json @@ -15,6 +15,10 @@ { "name": "--custom_auth_password", "secret": "ci/CustomAuthorizer/password" + }, + { + "name": "--is_ci", + "data": "true" } ] } diff --git a/.github/workflows/ci_run_mqtt5_custom_authorizer_websockets_cfg.json b/.github/workflows/ci_run_mqtt5_custom_authorizer_websockets_cfg.json index c77cbc12..90bd08e8 100644 --- a/.github/workflows/ci_run_mqtt5_custom_authorizer_websockets_cfg.json +++ b/.github/workflows/ci_run_mqtt5_custom_authorizer_websockets_cfg.json @@ -19,6 +19,10 @@ { "name": "--use_websockets", "data": "true" + }, + { + "name": "--is_ci", + "data": "true" } ] } diff --git a/.github/workflows/ci_run_mqtt5_fleet_provisioning_cfg.json b/.github/workflows/ci_run_mqtt5_fleet_provisioning_cfg.json index 58b03e7f..9919f29f 100644 --- a/.github/workflows/ci_run_mqtt5_fleet_provisioning_cfg.json +++ b/.github/workflows/ci_run_mqtt5_fleet_provisioning_cfg.json @@ -25,6 +25,10 @@ { "name": "--template_parameters", "data": "{\"SerialNumber\":\"$INPUT_UUID\"}" + }, + { + "name": "--is_ci", + "data": "true" } ] } diff --git a/.github/workflows/ci_run_mqtt5_jobs_cfg.json b/.github/workflows/ci_run_mqtt5_jobs_cfg.json index 118af85a..6c0249e0 100644 --- a/.github/workflows/ci_run_mqtt5_jobs_cfg.json +++ b/.github/workflows/ci_run_mqtt5_jobs_cfg.json @@ -21,6 +21,10 @@ { "name": "--thing_name", "data": "CI_Jobs_Thing" + }, + { + "name": "--is_ci", + "data": "true" } ] } diff --git a/.github/workflows/ci_run_mqtt5_pkcs11_connect_cfg.json b/.github/workflows/ci_run_mqtt5_pkcs11_connect_cfg.json index a8aaa3d9..4470facc 100644 --- a/.github/workflows/ci_run_mqtt5_pkcs11_connect_cfg.json +++ b/.github/workflows/ci_run_mqtt5_pkcs11_connect_cfg.json @@ -34,6 +34,10 @@ { "name": "--key_label", "data": "my-key" + }, + { + "name": "--is_ci", + "data": "true" } ] } diff --git a/.github/workflows/ci_run_pkcs11_connect_cfg.json b/.github/workflows/ci_run_pkcs11_connect_cfg.json index dfcf9cdc..4b1a2660 100644 --- a/.github/workflows/ci_run_pkcs11_connect_cfg.json +++ b/.github/workflows/ci_run_pkcs11_connect_cfg.json @@ -34,6 +34,10 @@ { "name": "--key_label", "data": "my-key" + }, + { + "name": "--is_ci", + "data": "true" } ] } diff --git a/.github/workflows/ci_run_pkcs12_connect_cfg.json b/.github/workflows/ci_run_pkcs12_connect_cfg.json index 3b633187..f2f96224 100644 --- a/.github/workflows/ci_run_pkcs12_connect_cfg.json +++ b/.github/workflows/ci_run_pkcs12_connect_cfg.json @@ -15,6 +15,10 @@ { "name": "--pkcs12_password", "secret": "ci/PubSub/key_pkcs12_password" + }, + { + "name": "--is_ci", + "data": "true" } ] } diff --git a/.github/workflows/ci_run_websocket_connect_cfg.json b/.github/workflows/ci_run_websocket_connect_cfg.json index d21980cd..378123f2 100644 --- a/.github/workflows/ci_run_websocket_connect_cfg.json +++ b/.github/workflows/ci_run_websocket_connect_cfg.json @@ -11,6 +11,10 @@ { "name": "--signing_region", "data": "us-east-1" + }, + { + "name": "--is_ci", + "data": "true" } ] } diff --git a/.github/workflows/ci_run_windows_cert_connect_cfg.json b/.github/workflows/ci_run_windows_cert_connect_cfg.json index 2e16ae15..e441e832 100644 --- a/.github/workflows/ci_run_windows_cert_connect_cfg.json +++ b/.github/workflows/ci_run_windows_cert_connect_cfg.json @@ -15,6 +15,10 @@ "windows_cert_key": "ci/PubSub/key", "windows_cert_key_path": "tmp_key.pem", "windows_cert_pfx_key_path": "tmp_pfx.pem" + }, + { + "name": "--is_ci", + "data": "true" } ] } diff --git a/.github/workflows/ci_run_x509_connect_cfg.json b/.github/workflows/ci_run_x509_connect_cfg.json index ac64d384..24f43488 100644 --- a/.github/workflows/ci_run_x509_connect_cfg.json +++ b/.github/workflows/ci_run_x509_connect_cfg.json @@ -33,6 +33,10 @@ { "name": "--x509_thing_name", "data": "CI_PubSub_Thing" + }, + { + "name": "--is_ci", + "data": "true" } ] } diff --git a/samples/basic_discovery.py b/samples/basic_discovery.py index 63a69d72..8998a977 100644 --- a/samples/basic_discovery.py +++ b/samples/basic_discovery.py @@ -102,9 +102,9 @@ def on_publish(topic, payload, dup, qos, retain, **kwargs): message['message'] = cmdData.input_message message['sequence'] = loop_count messageJson = json.dumps(message) - pub_future, _ = mqtt_connection.publish(cmdData.input_topic, messageJson, QoS.AT_MOST_ONCE) - pub_future.result() - print('Published topic {}: {}\n'.format(cmdData.input_topic, messageJson)) + pub_future, _ = mqtt_connection.publish(cmdData.input_topic, messageJson, QoS.AT_LEAST_ONCE) + publish_completion_data = pub_future.result() + print('Published topic {}: {} (puback reason: {})\n'.format(cmdData.input_topic, messageJson, repr(publish_completion_data.puback.reason_code))) loop_count += 1 time.sleep(1) diff --git a/test/greengrass/basic_discovery/copy_files.sh b/test/greengrass/basic_discovery/copy_files.sh new file mode 100644 index 00000000..ed8e31c3 --- /dev/null +++ b/test/greengrass/basic_discovery/copy_files.sh @@ -0,0 +1,3 @@ +cp ../../../samples/basic_discovery.py . +cp ../../../utils/run_in_ci.py . +cp ../../../.github/workflows/ci_run_greengrass_discovery_cfg.json . diff --git a/test/greengrass/basic_discovery/gdk-config.json b/test/greengrass/basic_discovery/gdk-config.json new file mode 100644 index 00000000..468e794e --- /dev/null +++ b/test/greengrass/basic_discovery/gdk-config.json @@ -0,0 +1,22 @@ +{ + "component": { + "software.amazon.awssdk.sdk-gg-test-discovery": { + "author": "iot-device-sdk", + "version": "NEXT_PATCH", + "build": { + "build_system": "custom", + "custom_build_command": ["bash", "copy_files.sh"] + }, + "publish": { + "bucket": "iot-sdk-ci-bucket-us-east-1", + "region": "us-east-1" + } + } + }, + "gdk_version": "1.3.0", + "test-e2e": { + "gtf_options": { + "tags": "testgg" + } + } +} diff --git a/test/greengrass/basic_discovery/gg-e2e-tests/pom.xml b/test/greengrass/basic_discovery/gg-e2e-tests/pom.xml new file mode 100644 index 00000000..460832fc --- /dev/null +++ b/test/greengrass/basic_discovery/gg-e2e-tests/pom.xml @@ -0,0 +1,63 @@ + + + 4.0.0 + + com.aws.greengrass + uat-features + jar + 1.0.0 + OTF + + + 1.2.0-SNAPSHOT + 1.8 + 1.8 + + + + + greengrass-common + greengrass common + + https://d2jrmugq4soldf.cloudfront.net/snapshots + + + + + + + com.aws.greengrass + aws-greengrass-testing-standalone + ${otf.version} + compile + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.2.2 + + + package + + shade + + + + + + + com.aws.greengrass.testing.launcher.TestLauncher + + + + + + + + + diff --git a/test/greengrass/basic_discovery/gg-e2e-tests/src/main/resources/greengrass/features/component.feature b/test/greengrass/basic_discovery/gg-e2e-tests/src/main/resources/greengrass/features/component.feature new file mode 100644 index 00000000..9f4c0453 --- /dev/null +++ b/test/greengrass/basic_discovery/gg-e2e-tests/src/main/resources/greengrass/features/component.feature @@ -0,0 +1,74 @@ +Feature: Testing features of Greengrassv2 basic discovery sample + + @testgg + Scenario: As a developer, I can create a component and deploy it on my device + Given my device is registered as a Thing + And my device is running Greengrass + When I create a Greengrass deployment with components + | aws.greengrass.clientdevices.Auth | LATEST | + | aws.greengrass.clientdevices.mqtt.Moquette | LATEST | + | aws.greengrass.clientdevices.mqtt.Bridge | LATEST | + | aws.greengrass.clientdevices.IPDetector | LATEST | + | software.amazon.awssdk.sdk-gg-test-discovery | file:recipe.yaml | + When I update my Greengrass deployment configuration, setting the component aws.greengrass.clientdevices.Auth configuration to: + """ + { + "MERGE": { + "deviceGroups": { + "formatVersion": "2021-03-05", + "definitions": { + "MyDeviceGroup": { + "selectionRule": "thingName: CI_Greengrass_Discovery_Thing", + "policyName": "MyRestrictivePolicy" + } + }, + "policies": { + "MyRestrictivePolicy": { + "AllowConnect": { + "statementDescription": "Allow client devices to connect.", + "operations": [ + "mqtt:connect" + ], + "resources": [ + "*" + ] + }, + "AllowPublish": { + "statementDescription": "Allow client devices to publish on topic.", + "operations": [ + "mqtt:publish" + ], + "resources": [ + "*clients/*/hello/world/*" + ] + } + } + } + } + } + } + """ + When I update my Greengrass deployment configuration, setting the component aws.greengrass.clientdevices.mqtt.Bridge configuration to: + """ + { + "MERGE": { + "mqttTopicMapping": { + "HelloWorldCoreMapping": { + "topic": "clients/+/hello/world/+", + "source": "LocalMqtt", + "target": "IotCore" + }, + "HelloWorldPubsubMapping": { + "topic": "clients/+/hello/world/+", + "source": "LocalMqtt", + "target": "Pubsub" + } + } + } + } + """ + And I deploy the Greengrass deployment configuration + Then the Greengrass deployment is COMPLETED on the device after 190 seconds + And the software.amazon.awssdk.sdk-gg-test-discovery log on the device contains the line "Successfully subscribed to topic" within 180 seconds + And the software.amazon.awssdk.sdk-gg-test-discovery log on the device contains the line "Received new message" within 240 seconds + And the software.amazon.awssdk.sdk-gg-test-discovery log on the device contains the line "disassociated CI_Greengrass_Discovery_Thing" within 260 seconds diff --git a/test/greengrass/basic_discovery/hello_world_subscriber.py b/test/greengrass/basic_discovery/hello_world_subscriber.py new file mode 100644 index 00000000..44fd99b7 --- /dev/null +++ b/test/greengrass/basic_discovery/hello_world_subscriber.py @@ -0,0 +1,55 @@ +import argparse +import sys +import time +import traceback +import uuid + +from awsiot.greengrasscoreipc.clientv2 import GreengrassCoreIPCClientV2 + + +def on_message(event): + try: + print('Topic: {}'.format(event.binary_message.context.topic)) + message = str(event.binary_message.message, 'utf-8') + print('Received new message: {}'.format(message)) + except: + traceback.print_exc() + + +def main(): + argument_parser = argparse.ArgumentParser( + description="Run Greengrass subscriber component") + argument_parser.add_argument( + "--input_uuid", required=False, help="UUID for unique topic name. UUID will be generated if this option is omit") + parsed_commands = argument_parser.parse_args() + + input_uuid = parsed_commands.input_uuid if parsed_commands.input_uuid else str(uuid.uuid4()) + + try: + ipc_client = GreengrassCoreIPCClientV2() + + client_device_hello_world_topic = 'clients/+/hello/world/{}'.format(input_uuid) + + # SubscribeToTopic returns a tuple with the response and the operation. + _, operation = ipc_client.subscribe_to_topic( + topic=client_device_hello_world_topic, on_stream_event=on_message) + print('Successfully subscribed to topic: {}'.format(client_device_hello_world_topic)) + + # Keep the main thread alive, or the process will exit. + try: + while True: + time.sleep(10) + except InterruptedError: + print('Subscribe interrupted.') + + operation.close() + except Exception: + print('Exception occurred when using IPC.', file=sys.stderr) + traceback.print_exc() + exit(1) + + print("Subscriber done") + + +if __name__ == "__main__": + main() diff --git a/test/greengrass/basic_discovery/recipe.yaml b/test/greengrass/basic_discovery/recipe.yaml new file mode 100644 index 00000000..d39af2b8 --- /dev/null +++ b/test/greengrass/basic_discovery/recipe.yaml @@ -0,0 +1,45 @@ +--- +RecipeFormatVersion: "2020-01-25" +ComponentName: software.amazon.awssdk.sdk-gg-test-discovery +ComponentVersion: "1.0.0" +ComponentDescription: "This is test for the Greengrass basic discovery sample" +ComponentPublisher: "iot-device-sdk" +ComponentConfiguration: + DefaultConfiguration: + accessControl: + aws.greengrass.ipc.pubsub: + software.amazon.awssdk.sdk-gg-test-discovery:pubsub:1: + policyDescription: "Allows access to subscribe to a Greengrass IPC test topic" + operations: + - aws.greengrass#SubscribeToTopic + - aws.greengrass#PublishToTopic + resources: + - "clients/*/hello/world/*" +Manifests: + - Platform: + os: all + Artifacts: + - URI: "file:hello_world_subscriber.py" + - URI: "file:run_in_ci.py" + - URI: "file:ci_run_greengrass_discovery_cfg.json" + - URI: "file:basic_discovery.py" + Lifecycle: + Install: | + echo "GG core:" {iot:thingName} + aws greengrassv2 batch-associate-client-device-with-core-device --core-device-thing-name {iot:thingName} --entries thingName=CI_Greengrass_Discovery_Thing + aws greengrassv2 list-client-devices-associated-with-core-device --core-device-thing-name {iot:thingName} + Run: | + UUID=$(python3 -c "import uuid; print (uuid.uuid4())") + echo "Starting subscriber" + python3 -u {artifacts:path}/hello_world_subscriber.py --input_uuid ${UUID} & + sleep 10 + echo "Starting discovery" + python3 {artifacts:path}/run_in_ci.py --runnable_dir {artifacts:path} --input_uuid ${UUID} --file {artifacts:path}/ci_run_greengrass_discovery_cfg.json + aws greengrassv2 batch-disassociate-client-device-from-core-device --core-device-thing-name {iot:thingName} --entries thingName=CI_Greengrass_Discovery_Thing + echo "Run: disassociated CI_Greengrass_Discovery_Thing" + Shutdown: | + echo "Shutdown: disassociating CI_Greengrass_Discovery_Thing" + aws greengrassv2 batch-disassociate-client-device-from-core-device --core-device-thing-name {iot:thingName} --entries thingName=CI_Greengrass_Discovery_Thing + Recover: | + echo "Recover: disassociating CI_Greengrass_Discovery_Thing" + aws greengrassv2 batch-disassociate-client-device-from-core-device --core-device-thing-name {iot:thingName} --entries thingName=CI_Greengrass_Discovery_Thing diff --git a/utils/run_in_ci.py b/utils/run_in_ci.py index d2eea2ea..d8d284a6 100644 --- a/utils/run_in_ci.py +++ b/utils/run_in_ci.py @@ -327,9 +327,6 @@ def launch_runnable(runnable_dir): exit_code = runnable_return.returncode elif (config_json['language'] == "Python"): - config_json_arguments_list.append("--is_ci") - config_json_arguments_list.append("True") - runnable_file = os.path.join(runnable_dir, config_json['runnable_file']) runnable_return = subprocess.run( args=[sys.executable, runnable_file] + config_json_arguments_list, input=subprocess_stdin, timeout=runnable_timeout) From 0ee4c4486b10d8abe40d1bee67a716f68e7d16c8 Mon Sep 17 00:00:00 2001 From: Igor Abdrakhimov Date: Wed, 5 Jun 2024 12:54:32 -0700 Subject: [PATCH 08/11] Add readmes --- test/greengrass/basic_discovery/README.md | 44 +++++++++++++++++++++++ test/greengrass/ipc/README.md | 43 ++++++++++++++++++++++ 2 files changed, 87 insertions(+) create mode 100644 test/greengrass/basic_discovery/README.md create mode 100644 test/greengrass/ipc/README.md diff --git a/test/greengrass/basic_discovery/README.md b/test/greengrass/basic_discovery/README.md new file mode 100644 index 00000000..77f1c575 --- /dev/null +++ b/test/greengrass/basic_discovery/README.md @@ -0,0 +1,44 @@ +# Test for Greengrass Discovery Sample + +Greengrass discovery test runs using [Greengrass Development Kit Command-Line Interface](https://docs.aws.amazon.com/greengrass/v2/developerguide/greengrass-development-kit-cli.html) (GDK CLI). + +### Greengrass discovery component + +For Greengrass discovery sample to work, a Greengrass component subscribed to a particular topic is required. +The following files defines this custom component: + +- [gdk-config.json](./gdk-config.json) - `gdk` reads this file to build and publish component. +- [copy_files.sh](./copy_files.sh) - utility to copy all required files for `gdk` to be able to build the component. +- [recipe.yaml](./recipe.yaml) - defines a component's details, dependencies, artifacts, and lifecycles. +- [hello_world_subscriber.py](./hello_world_subscriber.py) - a simple Greengrass client that subscribes to a given topic using Greengrass IPC. + +### How the test runs + +The first step is to build GreengrassV2 component artifacts and recipes from its source code: + +```shell +gdk component build +``` + +Then the following command builds the testing module: + +```shell +gdk test-e2e build +``` + +Finally, the test can run: + +```shell +gdk test-e2e run +``` + +The test behavior is defined in the [component.feature](./gg-e2e-tests/src/main/resources/greengrass/features/component.feature) +config file using a domain-specific language called [Gherkin](https://docs.aws.amazon.com/greengrass/v2/developerguide/gg-testing-framework.html). + +The test spins up Greengrass core, installs and configures Greengrass component dependencies (including the custom +Greengrass component described in the previous section). After everything is set up, it performs checks. They are defined +at the very bottom of the file and basically grep a log file for specific messages. + +On completion, the test creates log files in `testResult` directory with the run details. The component's logs are stored +in `testResult/gg-/software.amazon.awssdk.sdk-gg-test-discovery.log` file. Though, if error occurred before +the component started its execution, this file might be absent. diff --git a/test/greengrass/ipc/README.md b/test/greengrass/ipc/README.md new file mode 100644 index 00000000..415195d6 --- /dev/null +++ b/test/greengrass/ipc/README.md @@ -0,0 +1,43 @@ +# Test for Greengrass IPC Sample + +Greengrass IPC test runs using [Greengrass Development Kit Command-Line Interface](https://docs.aws.amazon.com/greengrass/v2/developerguide/greengrass-development-kit-cli.html) (GDK CLI). + +### Greengrass IPC component + +For Greengrass IPC sample to work, it should be deployed as a Greengrass component. +The following files defines this component: + +- [gdk-config.json](./gdk-config.json) - `gdk` reads this file to build and publish component. +- [copy_files.sh](./copy_files.sh) - utility to copy all required files for `gdk` to be able to build the component. +- [recipe.yaml](./recipe.yaml) - defines a component's details, dependencies, artifacts, and lifecycles. + +### How the test runs + +The first step is to build GreengrassV2 component artifacts and recipes from its source code: + +```shell +gdk component build +``` + +Then the following command builds the testing module: + +```shell +gdk test-e2e build +``` + +Finally, the test can run: + +```shell +gdk test-e2e run +``` + +The test behavior is defined in the [component.feature](./gg-e2e-tests/src/main/resources/greengrass/features/component.feature) +config file using a domain-specific language called [Gherkin](https://docs.aws.amazon.com/greengrass/v2/developerguide/gg-testing-framework.html). + +The test spins up Greengrass core, installs and configures Greengrass component dependencies (including the custom +Greengrass component described in the previous section). After everything is set up, it performs checks. They are defined +at the very bottom of the file and basically grep a log file for specific messages. + +On completion, the test creates log files in `testResult` directory with the run details. The component's logs are stored +in `testResult/gg-/software.amazon.awssdk.sdk-gg-ipc.log` file. Though, if error occurred before the +component started its execution, this file might be absent. From 674079d3f76230b2fd8630cc09e0a2e2bb772066 Mon Sep 17 00:00:00 2001 From: Igor Abdrakhimov Date: Wed, 5 Jun 2024 12:55:42 -0700 Subject: [PATCH 09/11] Remove old GG CI job --- .github/workflows/ci.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c98181bf..5929c138 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -394,14 +394,6 @@ jobs: Sample_UUID=$(python3 -c "import uuid; print (uuid.uuid4())") python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --file ${{ env.CI_SAMPLES_CFG_FOLDER }}/ci_run_mqtt5_fleet_provisioning_cfg.json --input_uuid ${Sample_UUID} python3 ${{ env.CI_UTILS_FOLDER }}/delete_iot_thing_ci.py --thing_name "Fleet_Thing_${Sample_UUID}" --region "us-east-1" - - name: configure AWS credentials (Greengrass) - uses: aws-actions/configure-aws-credentials@v2 - with: - role-to-assume: ${{ env.CI_GREENGRASS_ROLE }} - aws-region: ${{ env.AWS_DEFAULT_REGION }} - - name: run Greengrass Discovery sample - run: | - python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --file ${{ env.CI_SAMPLES_CFG_FOLDER }}/ci_run_greengrass_discovery_cfg.json linux-greengrass-tests: runs-on: ubuntu-22.04 From ee9c5be190caa797278876cc5549c7b2201e5199 Mon Sep 17 00:00:00 2001 From: Igor Abdrakhimov Date: Wed, 5 Jun 2024 12:56:18 -0700 Subject: [PATCH 10/11] Fix timeout --- .../src/main/resources/greengrass/features/component.feature | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/greengrass/basic_discovery/gg-e2e-tests/src/main/resources/greengrass/features/component.feature b/test/greengrass/basic_discovery/gg-e2e-tests/src/main/resources/greengrass/features/component.feature index 9f4c0453..fb6d21d4 100644 --- a/test/greengrass/basic_discovery/gg-e2e-tests/src/main/resources/greengrass/features/component.feature +++ b/test/greengrass/basic_discovery/gg-e2e-tests/src/main/resources/greengrass/features/component.feature @@ -68,7 +68,7 @@ Feature: Testing features of Greengrassv2 basic discovery sample } """ And I deploy the Greengrass deployment configuration - Then the Greengrass deployment is COMPLETED on the device after 190 seconds + Then the Greengrass deployment is COMPLETED on the device after 300 seconds And the software.amazon.awssdk.sdk-gg-test-discovery log on the device contains the line "Successfully subscribed to topic" within 180 seconds And the software.amazon.awssdk.sdk-gg-test-discovery log on the device contains the line "Received new message" within 240 seconds And the software.amazon.awssdk.sdk-gg-test-discovery log on the device contains the line "disassociated CI_Greengrass_Discovery_Thing" within 260 seconds From 77e14c2f3080fa03264d26d30b8f9285d6f6afb6 Mon Sep 17 00:00:00 2001 From: Igor Abdrakhimov Date: Tue, 11 Jun 2024 14:59:55 -0700 Subject: [PATCH 11/11] Add copyright header --- test/greengrass/basic_discovery/hello_world_subscriber.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/greengrass/basic_discovery/hello_world_subscriber.py b/test/greengrass/basic_discovery/hello_world_subscriber.py index 44fd99b7..ed817a6e 100644 --- a/test/greengrass/basic_discovery/hello_world_subscriber.py +++ b/test/greengrass/basic_discovery/hello_world_subscriber.py @@ -1,3 +1,6 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0. + import argparse import sys import time