diff --git a/README.md b/README.md index 85a8d766c..6d70f1293 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,5 @@ # AWS Greengrass Nucleus Lite -🚧 🚧 🚧 WORK IN PROGRESS 🚧 🚧 🚧 - -> This software does not yet have an official release. Feel free to test and -> report issues you find, but some functionality may be buggy and there may be -> breaking changes before the initial release. - AWS IoT Greengrass runtime for constrained devices. The Greengrass Nucleus Lite provides a smaller alternative to the Classic @@ -67,6 +61,13 @@ corebus. This table identifies the corebus component that does the work. | GetClientDeviceAuthToken | _Future_ | | AuthorizeClientDeviceAction | _Future_ | +## Additional Details + +Known issues are documented [here](./docs/KNOWN_ISSUES.md) with some potential +workarounds. Additionally only basic recipe types are supported, more +information on missing features can be found +[here](./docs/RECIPE_SUPPORT_CHANGES.md). + ## Security See [CONTRIBUTING](docs/CONTRIBUTING.md#security-issue-notifications) for more diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md deleted file mode 100644 index 335ee6910..000000000 --- a/RELEASE_NOTES.md +++ /dev/null @@ -1,174 +0,0 @@ -## 2024-11-26 - -Breaking changes: - -- `docs/INSTALL.md` and `docs/SETUP.md` have been updated to account for the - below. Please follow the new instructions. -- Greengrass Lite will now need to be installed on the target system or in a - container. -- The installation and running process for Greengrass Lite has been updated. The - old `run_nucleus` script has been replaced with systemd service files that - start the nucleus services in an appropriate order under systemd. Running the - install target with make will install both the binaries and service files to - the system. The `run_nucleus` script now enables and starts the service files. -- The services (other than `ggdeploymentd` run as a `ggcore` user and `ggcore` - group by default; the user/group to use can be configured with cmake, and must - be created on the system (provided container image has the user/groups). -- `ggdeploymentd` requires running as root user and nucleus service group - instead of using sudo and requiring sudo permissions for nucleus service user. -- Greengrass Lite now uses `/var/lib/greengrass` for the Greengrass rootDir - (where it will store data that needs to be persisted). Core bus sockets are - now created in `/run/greengrass`. -- It is now recommended to place your initial config in - `/etc/greengrass/config.yaml` or files in `/etc/greengrass/config.d` instead - of using the `ggl-config-init` utility. - -Features: - -- Greengrass Lite is now installed as systemd services. -- Most services don't require root or sudo access. Unless configured otherwise, - services run as `ggcore` user and `ggcore` group, other than `ggdeploymentd`, - which runs as `root` user and `ggcore` group. - -Bug Fixes: - -- Fixed bug with generic components failing to start. -- Fix issue with local deployment copying of CLI passed artifact and recipe dirs - (bug introduced in 2024-11-15). Local deployments will now place artifacts and - recipes in the correct location. -- Local deployment CLI now handles relative paths for artifacts/recipes. -- `-fstrict-flex-arrays=3` is made optional. -- Unused config file locations no longer result in error logs. -- Removed other unintended error logs. -- Nucleus service core-bus sockets now allow group access. -- Nucleus launch Fleet Status reports now correctly wait until connection - established. -- Generic components are started in their proper working directories. -- Configuration files override recipe default configuration. - -## 2024-11-15 - -Breaking changes: - -- The config file loading workflow has changed. `ggconfigd` can now read from a - config file and/or a directory of config files, and use it to set default - values for configuration. - - If config files are available at `/etc/greengrass/config.yaml` or in - `/etc/greengrass/config.d/`, they will be loaded automatically. The - `--config-file` and `--config-dir` args let you override these locations. - -Known Bugs: - -- `ggconfigd` logs error messages if configuration files are not found in - /etc/greengrass; these error messages are harmless and can be ignored. - -Bug Fixes: - -- File permissions for deployment-created files is fixed when running the - Greengrass Lite core services and generic components as different users. -- Greengrass Lite builds when close_range is not available. -- Fixed warning on newer CMake versions. -- Fleet status reports include correct device architecture. -- Greengrass Lite reports runtime field for cloud to identify it as Lite. -- Issue with copying directories during local deployments on some systems is - fixed. - -Features: - -- MQTT is used without clean session, and unacked messages with QoS1 are resent - after disconnects. -- MQTT topic filters are unsubscribed from when all Greengrass clients - subscribed to it close their Greengrass subscriptions for it. - -## 2024-10-22 - -Bug Fixes: - -- `--gc-sections` linker flag was causing misaligned accesses on 32-bit arm. The - flag has been disabled; this will need to be root caused further. - -## 2024-10-21 - -Breaking changes: - -- sqlite autoindex warning fixed. Delete and rebuild your database to get the - fix. -- Database is now case-sensitive. The sample config is also updated (Ensure your - initial config yaml file uses rootPath and not rootpath). Your recipes may - need to be updated to use the correct casing (as in the recipe docs - ) - -Features: - -- Deployment updates the cloud on failed deployments. -- Failing components will be retried up to three times. -- Deployments are marked successful when all components successfully start. -- SubscribeToConfigurationUpdate IPC command is available. -- Cloud deployments now use local dependencies if available. -- ZIP type artifacts are now unarchived. -- Artifact hashes are now verified. -- Stale components are cleaned up. -- A cloud logging generic component is included. -- MQTT connection reconnects with backoff. -- iotcored unsubscribes from unused topic filters. -- Multiple deployments across thing groups is now supported. -- Recipes now support configuration in lifecycle step variable substitution. -- Fleet status service provides updates of Greengrass device state to cloud. -- Components are only restarted if their version is changed. - -Bug Fixes: - -- Absolute file paths are no longer logged on old compiler versions. -- ggipcd uses rootPath for creation of the IPC socket. -- TES environment variables are set for generic components. -- Coverity findings are fixed. -- Many minor bugs were fixed. - -## 2024-10-04 - -Features: - -- Cloud deployments now support dependencies. -- First party component artifacts can now be downloaded. -- IPC Authz policies are now enforced. -- Recipe variables (other than configuration) can be used. -- DefaultConfiguration from recipes is now applied. -- Now builds against Musl and on 32-bit arm platforms. -- Development container is available under `misc/container` for testing. - -Bug Fixes: - -- Now builds for older Linux kernels. -- run_nucleus now runs TES. -- Python is no longer needed. -- Fixed bug with SVCUID validation when building with auth. - -Known Issues: - -- `ggl-config-init` crashes on some platforms (under investigation). -- Errors are logged for sqlite and opening json files; these can be ignored. -- GG user/group in config must be the same user Greengrass is running as. - -## 2024-08-26 - -- Please install fresh, or delete the configuration store config.db file - (located at the ggconfigd service working directory). This is to avoid - conflicts with configuration persisted by older versions - -Features: - -- ggconfigd and GetConfiguration supports reading nested configuration back in a - single read call -- Fleet Provisioning will now manage its own new instance of iotcored - -Bug Fixes: - -- TES_server will now support `http://` prefix in config - -Known Issues: - -- Fleet Provisioning does not terminate even after provisioning is complete will - fix in future -- If using Fleet Provisioning provide the `iotCredEndpoint` within the - `NucleusLite`'s config scope diff --git a/debian/postinst b/debian/postinst index 5300a0d25..4dc75452e 100755 --- a/debian/postinst +++ b/debian/postinst @@ -51,6 +51,7 @@ services: runWithDefault: posixUser: "${gg_user}:${gg_group}" greengrassDataPlanePort: "8443" + platformOverride: {} EOL echo "Configuration file created at $config_file" } diff --git a/docs/Fleet-provisioning.md b/docs/Fleet-provisioning.md index 1c513a33b..1dea500a4 100644 --- a/docs/Fleet-provisioning.md +++ b/docs/Fleet-provisioning.md @@ -9,6 +9,24 @@ can get valid certificates. you can follow the link [here](https://docs.aws.amazon.com/greengrass/v2/developerguide/fleet-provisioning-setup.html) to learn how to create appropriate policies and claim certificate. +``` +Note: +Currently, fleet provisioning can only be run manually. +Hence you will need to follow few important pre-steps + +1. Make sure you are logged in as root +2. Allow read access to all user for your certificates + chmod -R +rx /ggcredentials/ +3. Make sure you do not fill iotCredEndpoint/iotDataEndpoint under + `aws.greengrass.NucleusLite` you should only fill these fields + under `aws.greengrass.fleet_provisioning`'s config +4. If this is your not first run, remove the socket at + /run/greengrass/iotcoredfleet, if it exists +5. Fleet provisioning assumes the your GGL_SYSTEMD_SYSTEM_USER + and GGL_SYSTEMD_SYSTEM_GROUP to be ggcore:ggcore please change + appropriately if you change these values during compile time +``` + Sample Fleet provisioning template: ```json @@ -73,52 +91,41 @@ config should roughly look as below. system: privateKeyPath: "" certificateFilePath: "" - rootCaPath: "/home/ubuntu/repo/fleetClaim/AmazonRootCA1.pem" - rootPath: "/home/ubuntu/aws-greengrass-lite/run_fleet/" - thingName: "" + rootCaPath: "/ggcredentials/fleetClaim/AmazonRootCA1.pem" #[Modify here] + rootPath: "/var/lib/greengrass/" #[Modify here] + thingName: "" #[Must leave blank] services: aws.greengrass.NucleusLite: componentType: "NUCLEUS" configuration: awsRegion: "us-east-1" - iotCredEndpoint: "" - iotDataEndpoint: "" + iotCredEndpoint: "" #[Must leave blank] + iotDataEndpoint: "" #[Must leave blank] iotRoleAlias: "GreengrassV2TokenExchangeRoleAlias" runWithDefault: - posixUser: "ubuntu:ubuntu" + posixUser: "user:group" #[Modify here] greengrassDataPlanePort: "8443" - tesCredUrl: "http://127.0.0.1:8080/" aws.greengrass.fleet_provisioning: configuration: - iotDataEndpoint: "dddddddddddddd-ats.iot.us-east-1.amazonaws.com" - iotCredEndpoint: "aaaaaaaaaaaaaa.credentials.iot.us-east-1.amazonaws.com" - claimKeyPath: "/home/ubuntu/fleetClaim/private.pem.key" - claimCertPath: "/home/ubuntu/fleetClaim/certificate.pem.crt" - templateName: "FleetTestNew" - templateParams: '{"SerialNumber": "14ALES55UFA"}' -``` - -With all this setup for IoT core now let's begin provisioning the device. First -we will start an instance of ggconfigd - -```sh -cd ./run -../build/bin/ggconfigd + iotDataEndpoint: "aaaaaaaaaaaaaa-ats.iot.us-east-1.amazonaws.com" #[Modify here] + iotCredEndpoint: "cccccccccccccc.credentials.iot.us-east-1.amazonaws.com" #[Modify here] + claimKeyPath: "/ggcredentials/fleetClaim/private.pem.key" #[Modify here] + claimCertPath: "/ggcredentials/fleetClaim/certificate.pem.crt" #[Modify here] + templateName: "FleetTestNew" #[Modify here] + templateParams: '{"SerialNumber": "AAA55555"}' #[Modify here] ``` -In another shell, run the config script and the fleet provisioning +In root user shell, run fleet provisioning ```sh cd ./run -../build/bin/ggl-config-init --config ./init_config.yml ../build/bin/fleet-provisioning ``` Now this will trigger the fleet provisioning script which will take a few -minutes to complete, the shell doesn't automatically exits so look for a Info -level log: `Process Complete, Your device is now provisioned`. then you can kill -the process or wait for auto terminate of `300 seconds`. +minutes to complete. -You can then kill the config daemon as well. +> Note: Device will reboot in case of successful run -Now you can return to `## Running the nucleus` step in [SETUP.md](SETUP.md) +If you are storing the standard output then look for log: +`Process Complete, Your device is now provisioned`. diff --git a/docs/INSTALL.md b/docs/INSTALL.md index 78aac94b9..93f45a814 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -7,20 +7,8 @@ init system. ## Dependencies -This project uses the following third party library dependencies: - -- openssl -- libcurl >= 7.86 -- sqlite -- libyaml -- libsystemd -- liburiparser -- libuuid -- libevent -- libzip -- libcgroup-tools - -On Ubuntu, these can be installed with: +This project uses third party library dependencies on Ubuntu > 24.04, these can +be installed with: ```sh sudo apt install libssl-dev libcurl4-openssl-dev libsqlite3-dev libyaml-dev \ @@ -79,6 +67,10 @@ adduser -g ggcore Then run: ``` +sudo mkdir /ggcredentials +//cp your aws credentials(device certificates, private key, root ca) to this folder +chown -R ggcore:ggcore /ggcredentials + mkdir /var/lib/greengrass chown ggcore:ggcore /var/lib/greengrass ``` diff --git a/docs/KNOWN_ISSUES.md b/docs/KNOWN_ISSUES.md new file mode 100644 index 000000000..b41d86931 --- /dev/null +++ b/docs/KNOWN_ISSUES.md @@ -0,0 +1,74 @@ +# Greengrass Lite Known Issues List + +The following are some known issues in the Greengrass Lite software where +behavior may be different from the classic version of Greengrass. Upcoming or +missing features are not included in this list. This list will be updated as +issues are discovered or fixed. + +### Stale phase cleanup does not occur + +In Greengrass Lite, phases that become stale when updating a component version +are not cleaned up. For example, a component that includes both an install and a +run lifecycle phase is updated to no longer have an install phase. This install +phase will not be cleaned up and will still be processed even after a component +update has removed it from the recipe. Customers should avoid removing lifecycle +phases when possible, but adding new phases is okay and behaves as expected. + +Workaround: The deployment should be revised to remove the component first, and +then can be revised to include it again. + +### platformOverride NucleusLite configuration key exists but is not fully supported; architecture.detail can only be set via this configuration + +The platformOverride key in Greengrass classic is used to specify a platform +override which can include any platform attributes such as “os” or +“architecture”. The platformOverride key in Greengrass Lite only supports the +“architecture.detail” platform attribute, to be used to set the expected +architecture.detail attribute. Greengrass Lite will not check the system for its +actual architecture detail and solely rely on this configuration value to set +the architecture.detail platform attribute. + +Workaround: Only use platformOverride to set architecture.detail. If you want to +deploy a component that specifies an architecture.detail platform attribute, +then the correct value must be set for /platformOverride/architecture.detail in +the NucleusLite configuration. + +### Components with a lifecycle step that varies depending on recipe variable substitution will not be restarted if the recipe variable changes + +In Greengrass classic, components will be restarted if the “run” lifecycle phase +changes after a deployment. This may include run lifecycle phases that use +variable interpolation to include something from the component configuration in +the lifecycle script. Greengrass Lite will not detect that a component lifecycle +phase has changed in the event of a configuration change changing the recipe +variable used in the lifecycle script, so these components will not restart if +the version has not changed. + +Workaround: Components should use the SubscribeToConfigurationUpdate IPC command +to subscribe to configuration changes and update their process to use new +values. If this is not possible, an easier workaround may be to simply bump the +component version in order to force a component restart. + +### Greengrass deployments will fail after the same component is deployed locally over 100 times + +This is due to a memory limitation in our current design. + +Workaround: Avoid deploying the same component name over 100 times locally. If +needed, the component can be renamed which will be under a different counter. + +### Certain complex accessControl policies may not properly grant authorization + +accessControl policies for MQTT will not support the `#` or `+` MQTT wildcard +substitutions in the same policy including a normal `*` wildcard (the MQTT +wildcards can still be used if the policy does not include a `*`). accessControl +policies will not support the `${*}`, `${?}`, and `${$}` escape sequences +either. They also do not support recipe variable interpolations in policies due +to Greengrass Lite not supporting recipe variable interpolation in configuration +yet and accessControl policies are part of the component configuration. + +Workaround: Use simpler authorization policies that do not make use of these +features, referencing the docs here: +https://docs.aws.amazon.com/greengrass/v2/developerguide/interprocess-communication.html + +### The root path must be /var/lib/greengrass + +Please ensure that Greengrass Lite is installed to the /var/lib/greengrass root +path. This is a limitation with no workaround at the moment. diff --git a/docs/RECIPE_SUPPORT_CHANGES.md b/docs/RECIPE_SUPPORT_CHANGES.md new file mode 100644 index 000000000..5d41c183e --- /dev/null +++ b/docs/RECIPE_SUPPORT_CHANGES.md @@ -0,0 +1,65 @@ +# Types of Recipes Supported by GG nucleus lite + +For GG nucleus lite we only support basic recipe format and support for more +complex recipe will be delivered with future release. Below is the summary of +what's not supported. If it's not mentioned in the list then that case is +supported as mentioned in the +[aws docs](https://docs.aws.amazon.com/greengrass/v2/developerguide/component-recipe-reference.html). + +## Major differences + +### Recipes + +- All the keys in a recipe are now case sensitive, please visit our aws recipe + docs reference + [link](https://docs.aws.amazon.com/greengrass/v2/developerguide/component-recipe-reference.html) + to know aboout the correct casing. + +- Only linux lifecycles are supported with the current release. + +- Only generic component (`aws.greengrass.generic`) recipe types are supported + with lite. + +- Some lifecycle steps are not currently supported: + + - shutdown + - recover + - bootstrap + +- `Skipif` section for a given lifecycle step is also not supported. + +- Refering to global lifecycle requires mentioning `all` field for it to work. + Refer to [sample recipe 3](./examples/supported_lifecyle_types/3.yaml). + +- "runtime": "\*"(for classic and lite) or "runtime": "aws_nucleus_lite" is + required new field that needs to be added for it to work with lite. See + [sample recipe 1](./examples/supported_lifecyle_types/1.json). + + ```yaml + Manifests: + - Platform: + os: "linux" + runtime: "aws_nucleus_lite" + ``` + +- Regex support is not available within recipe. + +- Docker artifacts are not supported. + +- GG nucleus lite only support variable replacement for following cases: + + - artifacts:path + - artifacts:decompressedPath + - kernel:rootPath + - iot:thingName + - work:path + - Limited configuration:json_pointer support + +- Component_dependency_name prefixes are not supported for recipe variable + replacement. +- Recipe variable interpolation for component configuration is not supported. + +### Nucleus Configuration + +- Platform Override only supports `architecture.detail`, please refer known + issues docs [here](./KNOWN_ISSUES.md). diff --git a/docs/SETUP.md b/docs/SETUP.md index 2002ae78f..281694720 100644 --- a/docs/SETUP.md +++ b/docs/SETUP.md @@ -13,6 +13,23 @@ directory you intend to use as the Greengrass root path. The following examples assume you are using `./build` as the build directory, and `/var/lib/greengrass` as the Greengrass root path. +## Dependencies + +This project uses the following third party library dependencies: + +| Library | Minimum Version Required | +| --------------- | ------------------------ | +| openssl | 3.0 | +| libcurl | 7.86 | +| sqlite | 3.46.0 | +| libyaml | 0.2.2 | +| libsystemd | 249 | +| liburiparser | 0.9.6 | +| libuuid | 2.37.2 | +| libevent | 2.1.12 | +| libzip | 1.7.3 | +| libcgroup-tools | 2.0 | + ## Configuring Greengrass You may configure a single device with the instruction below or a fleet of diff --git a/docs/design/executable/recipe-runner.md b/docs/design/executable/recipe-runner.md index b4f92bea0..8305ca4a7 100644 --- a/docs/design/executable/recipe-runner.md +++ b/docs/design/executable/recipe-runner.md @@ -1,27 +1 @@ # `recipe-runner` design - -See [`recipe-runnerd` spec](../../spec/executable/recipe-runner.md) for the -public interface for `recipe-runnerd`. - -`recipe-runner` is designed to run as a wrapper on the recipe's lifecycle script -section. It will take the script section as is and will replace the recipe -variables with appropriate values during runtime. The executable also -understands the gg-global config and how to interact with it. - -Once a recipe is translated to a unit file, the selected lifecycle will be -converted to a json file with its different lifecycle section. Each lifecycle -section will generate a unit file that is suffixed with it's phase. For an -example a recipe names `sampleComponent-0.1.0` will have unit files named -`sampleComponent-0.1.0_install` and `sampleComponent-0.1.0_run` to represent -install and run phase of lifecycle. As per the recipe2unit's design. - -Once a unit file is created `ggdeploymentd` will use `recipe2unit`'s functions -to execute the specific unit file that will run provided lifecycle phase as a -first time installation process. - -`recipe-runner` will use the provided selected lifecycle section and use -`execvp` to execute the argument provided lifecycle section as a bash script. It -will also forward any environment variables set during runtime. As a side effect -it will create a temporary bash script file with all the gg-recipe variables -replaced with appropriate actual values from the global config and then use will -provide the newly created script file to `execvp`. diff --git a/docs/examples/sample-ggLitePython/artifacts/sample-ggLitePython/1.0.0/ggLitePython.py b/docs/examples/sample-ggLitePython/artifacts/sample-ggLitePython/1.0.0/ggLitePython.py new file mode 100644 index 000000000..a30347738 --- /dev/null +++ b/docs/examples/sample-ggLitePython/artifacts/sample-ggLitePython/1.0.0/ggLitePython.py @@ -0,0 +1,21 @@ +import boto3 + + +def fetch_s3_bucket_list(): + # Create an S3 client + s3 = boto3.client('s3') + # List S3 buckets + response = s3.list_buckets() + # Print the names of all buckets + print("S3 Bucket Names:") + for bucket in response['Buckets']: + print(bucket['Name']) + + +def main(): + print("HELLO WORLD") + fetch_s3_bucket_list() + + +if __name__ == "__main__": + main() diff --git a/docs/examples/sample-ggLitePython/recipes/sample-ggLitePython-1.0.0.yaml b/docs/examples/sample-ggLitePython/recipes/sample-ggLitePython-1.0.0.yaml new file mode 100644 index 000000000..abec97ca3 --- /dev/null +++ b/docs/examples/sample-ggLitePython/recipes/sample-ggLitePython-1.0.0.yaml @@ -0,0 +1,28 @@ +--- +RecipeFormatVersion: "2020-01-25" +ComponentName: sample-ggLitePython +ComponentVersion: 1.0.0 +ComponentType: "aws.greengrass.generic" +ComponentDescription: + This example Python component for GGLite that lists all your s3 bucket names +ComponentPublisher: AWS +ComponentDependencies: + aws.greengrass.TokenExchangeService: + VersionRequirement: ">=0.0.0" + DependencyType: "HARD" +Manifests: + - Platform: + os: linux + runtime: "*" + Lifecycle: + install: + RequiresPrivilege: false + Script: "python3 -m venv ./venv && . ./venv/bin/activate && python3 -m + pip install\ + \ boto3" + run: + RequiresPrivilege: false + Script: + ". ./venv/bin/activate && python3 {artifacts:path}/ggLitePython.py" + # Artifacts: + # - Uri: "---" diff --git a/docs/examples/sample_nucleus_config.yml b/docs/examples/sample_nucleus_config.yml index 172c39aa5..0380d3dab 100644 --- a/docs/examples/sample_nucleus_config.yml +++ b/docs/examples/sample_nucleus_config.yml @@ -16,6 +16,7 @@ services: runWithDefault: posixUser: "gg_component:gg_component" greengrassDataPlanePort: "8443" + platformOverride: {} # aws.greengrass.fleet_provisioning: # configuration: # iotDataEndpoint: "" diff --git a/docs/examples/supported_lifecyle_types/1.json b/docs/examples/supported_lifecyle_types/1.json new file mode 100644 index 000000000..6553d6c62 --- /dev/null +++ b/docs/examples/supported_lifecyle_types/1.json @@ -0,0 +1,49 @@ +{ + // Notice: All Key's are case sensitive + "RecipeFormatVersion": "2020-01-25", + "ComponentName": "s3.list.bucket.Python", + "ComponentVersion": "1.0.4", + "ComponentType": "aws.greengrass.generic", + "ComponentDescription": "This example Python component", + "ComponentPublisher": "TestUser", + "ComponentDependencies": { + "aws.greengrass.TokenExchangeService": { + "VersionRequirement": ">=0.0.0", + "DependencyType": "HARD" + } + }, + "Manifests": [ + { + "Platform": { + "os": "linux", + "runtime": "*", // Notice this is a required new field for GGLite + "architecture": "amd64" + }, + "Lifecycle": { + "run": { + "RequiresPrivilege": true, + "Script": "python3 {artifacts:path}/python_list_S3_bucket.py" + }, + "install": { + "RequiresPrivilege": true, + "Script": "pip install boto3" + } + // Notice bootstrap, shutdown and recover aren't currently supported + }, + "Artifacts": [ + { + // Notice URI needs to Uri(Camel case) + "Uri": "s3://gglite-artifact/python_list_S3_bucket.py", + "Digest": "---", + "Algorithm": "SHA-256", + "Unarchive": "NONE", + "Permission": { + "Read": "OWNER", + "Execute": "NONE" + } + } + ] + } + ], + "Lifecycle": {} +} diff --git a/docs/examples/supported_lifecyle_types/2.json b/docs/examples/supported_lifecyle_types/2.json new file mode 100644 index 000000000..78fdd674b --- /dev/null +++ b/docs/examples/supported_lifecyle_types/2.json @@ -0,0 +1,147 @@ +{ + // Notice: All Key's are case sensitive + "RecipeFormatVersion": "2020-01-25", + "ComponentName": "aws.greengrass.SecureTunneling", + "ComponentVersion": "1.0.100", + "ComponentType": "aws.greengrass.generic", + "ComponentDescription": "Enables AWS IoT Secure Tunneling connections that you can use to establish secure bidirectional communications with Greengrass core devices that are behind restricted firewalls.", + "ComponentPublisher": "AWS", + "ComponentConfiguration": { + "DefaultConfiguration": { + "accessControl": { + "aws.greengrass.ipc.mqttproxy": { + "aws.greengrass.SecureTunneling:mqttproxy:1": { + "policyDescription": "Access to tunnel notification pubsub topic", + "operations": ["aws.greengrass#SubscribeToIoTCore"], + "resources": ["$aws/things/+/tunnels/notify"] + } + } + }, + "OS_DIST_INFO": "auto" + } + }, + "Manifests": [ + { + "Platform": { + "os": "linux", + "runtime": "*", + "architecture": "amd64" // Notice regex isn't currently supported + }, + "Lifecycle": { + "run": { + "Script": "java -jar {artifacts:path}/GreengrassV2SecureTunnelingComponent-1.0-all.jar linux x86_64" + } + }, + "Artifacts": [ + { + "Uri": "s3://gglite-artifact/GreengrassV2SecureTunnelingComponent-1.0-all.jar", + "Digest": "----", + "Algorithm": "SHA-256", + "Unarchive": "NONE", + "Permission": { + "Read": "OWNER", + "Execute": "NONE" + } + } + ] + }, + { + "Platform": { + "os": "linux", + "runtime": "*", + "architecture": "x86_64" // Repeated Lifecycle with just architecture change + }, + "Lifecycle": { + "run": { + "Script": "java -jar {artifacts:path}/GreengrassV2SecureTunnelingComponent-1.0-all.jar linux x86_64" + } + }, + "Artifacts": [ + { + "Uri": "s3://gglite-artifact/GreengrassV2SecureTunnelingComponent-1.0-all.jar", + "Digest": "----", + "Algorithm": "SHA-256", + "Unarchive": "NONE", + "Permission": { + "Read": "OWNER", + "Execute": "NONE" + } + } + ] + }, + { + "Platform": { + "os": "linux", + "runtime": "*", + "architecture": "aarch64" + }, + "Lifecycle": { + "run": { + "Script": "java -jar {artifacts:path}/GreengrassV2SecureTunnelingComponent-1.0-all.jar linux aarch64" + } + }, + "Artifacts": [ + { + "Uri": "s3://gglite-artifact/GreengrassV2SecureTunnelingComponent-1.0-all.jar", + "Digest": "----", + "Algorithm": "SHA-256", + "Unarchive": "NONE", + "Permission": { + "Read": "OWNER", + "Execute": "NONE" + } + } + ] + }, + { + "Platform": { + "os": "linux", + "runtime": "*", + "architecture": "armv8" + }, + "Lifecycle": { + "run": { + "Script": "java -jar {artifacts:path}/GreengrassV2SecureTunnelingComponent-1.0-all.jar linux aarch64" + } + }, + "Artifacts": [ + { + "Uri": "s3://gglite-artifact/GreengrassV2SecureTunnelingComponent-1.0-all.jar", + "Digest": "----", + "Algorithm": "SHA-256", + "Unarchive": "NONE", + "Permission": { + "Read": "OWNER", + "Execute": "NONE" + } + } + ] + }, + { + "Platform": { + "os": "linux", + "architecture.detail": "armv7l", + "runtime": "*", + "architecture": "arm" + }, + "Lifecycle": { + "run": { + "Script": "java -jar {artifacts:path}/GreengrassV2SecureTunnelingComponent-1.0-all.jar linux armv7l" + } + }, + "Artifacts": [ + { + "Uri": "s3://gglite-artifact/GreengrassV2SecureTunnelingComponent-1.0-all.jar", + "Digest": "----", + "Algorithm": "SHA-256", + "Unarchive": "NONE", + "Permission": { + "Read": "OWNER", + "Execute": "NONE" + } + } + ] + } + ], + "Lifecycle": {} +} diff --git a/docs/examples/supported_lifecyle_types/3.yaml b/docs/examples/supported_lifecyle_types/3.yaml new file mode 100644 index 000000000..6d144a798 --- /dev/null +++ b/docs/examples/supported_lifecyle_types/3.yaml @@ -0,0 +1,40 @@ +--- +# Notice: All Key's are case sensitive +RecipeFormatVersion: "2020-01-25" +ComponentName: "s3.list.bucket.Python" +ComponentVersion: "1.0.4" +ComponentType: "aws.greengrass.generic" +ComponentDescription: "This example Python component" +ComponentPublisher: "Amazon" +ComponentDependencies: + aws.greengrass.TokenExchangeService: + VersionRequirement: ">=0.0.0" + DependencyType: "HARD" +ComponentConfiguration: + DefaultConfiguration: + containerMode: "auto" +Manifests: + - Platform: + os: "linux" + runtime: "aws_nucleus_lite" # Notice this is a required new field for GGLite + Lifecycle: {} #Notice No Lifecycle or Selection so Global lifecycle is used + Artifacts: + - Uri: "s3://gglite-artifact/python_list_S3_bucket.py" + Digest: "----" + Algorithm: "SHA-256" + Unarchive: "NONE" + Permission: + Read: "OWNER" + Execute: "NONE" +Lifecycle: + # Notice all is case sensitive and is required when no selection is provided + - all: + Setenv: + TEST_VALUE: "{configuration:/containerMode}" + run: + # Notice Skipif is not supported + RequiresPrivilege: true + Script: "python3 {artifacts:path}/python_list_S3_bucket.py" + Timeout: 200 #Seconds + Setenv: + TEST_VALUE: sample diff --git a/docs/examples/supported_lifecyle_types/4.yaml b/docs/examples/supported_lifecyle_types/4.yaml new file mode 100644 index 000000000..70a971563 --- /dev/null +++ b/docs/examples/supported_lifecyle_types/4.yaml @@ -0,0 +1,32 @@ +--- +# Notice: All Key's are case sensitive +RecipeFormatVersion: "2020-01-25" +ComponentName: "s3.list.bucket.Python" +ComponentVersion: "1.0.4" +ComponentType: "aws.greengrass.generic" +ComponentDescription: "This example Python component" +ComponentPublisher: "Amazon" +ComponentDependencies: + aws.greengrass.TokenExchangeService: + VersionRequirement: ">=0.0.0" + DependencyType: "HARD" +Manifests: + - Platform: + os: "linux" + runtime: "*" # Notice this is a required new field for GGLite + Lifecycle: {} + Selections: + - linux + Artifacts: + - Uri: "s3://gglite-artifact/python_list_S3_bucket.py" + Digest: "----" + Algorithm: "SHA-256" + Unarchive: "NONE" + Permission: + Read: "OWNER" + Execute: "NONE" +Lifecycle: + - linux: + run: + RequiresPrivilege: true + Script: "python3 {artifacts:path}/python_list_S3_bucket.py" diff --git a/docs/spec/executable/ggdeploymentd.md b/docs/spec/executable/ggdeploymentd.md index 777d2c48e..2ef3e3f20 100644 --- a/docs/spec/executable/ggdeploymentd.md +++ b/docs/spec/executable/ggdeploymentd.md @@ -168,6 +168,29 @@ device following a deployment. - Note: Currently excludes local deployments and might result to removal of all those components +## NucleusLite Bootstrap + +GG-Lite supports bootstrap deployments for the NucleusLite component. + +- Upon receiving a deployment, all deployment info will be stored in the config + database under services -> DeploymentService -> deploymentState +- Bootstrap scripts of all bootstrap components will be processed and run +- Device will reboot after bootstrap scripts successfully complete +- On reboot, ggdeploymentd will check the config for a previously in progress + deployment. +- If a deployment is found, it will be resumed and completed. Bootstrap steps + will be skipped and the remaining lifecycle stages will be processed. +- If a deployment is not found on startup, ggdeploymentd will continue + functioning as normal and await the next deployment +- At the end of each deployment, all deployment info in the config database will + be deleted + +- Note: + - Bootstrap is NOT guaranteed to work for components other than NucleusLite. + - Exit codes in bootstrap scripts are not currently supported. Each script + will result in the device being rebooted. + - BootstrapOnRollback is NOT supported + ### samples The expected format of the input map will look as below diff --git a/docs/spec/executable/recipe-runner.md b/docs/spec/executable/recipe-runner.md index 625a91766..d74da0605 100644 --- a/docs/spec/executable/recipe-runner.md +++ b/docs/spec/executable/recipe-runner.md @@ -1,42 +1 @@ # `recipe-runnerd` spec - -`recipe-runnerd` will act like a wrapper around the generic component to -dynamically update the GG-recipe variables - -- [recipe-runnerd-1] The executable will execute all the commands within a - selected phase as a bash script. -- [recipe-runnerd-2] The executable will also forward its environment variables - to the running script using global config. -- [recipe-runnerd-3] On execution failure it prints the error message to stderr -- [recipe-runnerd-4] The executable will take only 1 file as an argument and - phase - -## CLI parameters - -## phase - -- [recipe-runnerd-param-phase-1] The argument will dectate which phase needs to - be executed. -- [recipe-runnerd-param-phase-2] The phase argument can be provided by `--phase` - or `-p`. -- [recipe-runnerd-param-phase-3] The phase argument is required. - -### filePath - -- [recipe-runnerd-param-filePath-1] The argument will provide the path to - selected lifecycle json. -- [recipe-runnerd-param-filePath-2] The filePath argument can be provided by - `--filepath` or `-f`. -- [recipe-runnerd-param-filePath-3] The filePath argument is required. - -### timeout - -- [recipe-runnerd-param-timeout-1] The argument will allow user to edit the - timeout seting for the given script in seconds. -- [recipe-runnerd-param-timeout-2] The deafult value for the parmeter is 30 - seconds. -- [recipe-runnerd-param-timeout-3] The timeout argument can be provided by - `--timeout` or `-t`. -- [recipe-runnerd-param-timeout-4] The timeout argument is optional. - -## Environment Variables diff --git a/docs/spec/executable/recipe2unit.md b/docs/spec/executable/recipe2unit.md index 0118e30e3..37a609a34 100644 --- a/docs/spec/executable/recipe2unit.md +++ b/docs/spec/executable/recipe2unit.md @@ -1,22 +1 @@ # `recipe2unit` spec - -`recipe2unit` converts a GG recipe into a linux specific systemd unit file so -that it can be deployed within the edge device. - -- [recipe2unit-1] The executable intakes a GG recipe file and spits out a .unit - file with all the features represented within the recipe. -- [recipe2unit-2] In case of an error it will output `Error Parsing Recipe` - along with appropriate error message. - -## CLI parameters - -### recipe-path - -- [recipe2unit-param-path-1] This argument will allow user to specify the - recipe's location within the disk. -- [recipe2unit-param-path-2] The argument must be provided by `--recipe-path`. -- [recipe2unit-param-path-3] The argument is a required field - -## Environment Variables - -## Core Bus API diff --git a/eventstream/src/decode.c b/eventstream/src/decode.c index 2db28689d..6693f77ea 100644 --- a/eventstream/src/decode.c +++ b/eventstream/src/decode.c @@ -203,11 +203,9 @@ GglError eventstream_decode( break; case EVENTSTREAM_STRING: GGL_LOGT( - "Header: \"%.*s\" => \"%.*s\"", + "Header: \"%.*s\" => (data not shown)", (int) header.name.len, - header.name.data, - (int) header.value.string.len, - header.value.string.data + header.name.data ); break; } diff --git a/fleet-provisioning/bin/fleet-provisioning.c b/fleet-provisioning/bin/fleet-provisioning.c index 98358b020..e044bb7c7 100644 --- a/fleet-provisioning/bin/fleet-provisioning.c +++ b/fleet-provisioning/bin/fleet-provisioning.c @@ -3,8 +3,10 @@ // SPDX-License-Identifier: Apache-2.0 #include "fleet-provisioning.h" +#include #include #include +#include #include #include #include @@ -83,7 +85,7 @@ static void parse_path(char **argv, char *path) { // worry about null termination // NOLINTNEXTLINE(bugprone-not-null-terminated-result) memcpy(path, argv[0], strlen(argv[0]) - strlen(COMPONENT_NAME)); - strncat(path, "iotcored", strlen("iotcored")); + strncat(path, "iotcored", sizeof("iotcored") - 1U); GGL_LOGD("iotcored path: %.*s", (int) strlen(path), path); } @@ -98,8 +100,13 @@ int main(int argc, char **argv) { argp_parse(&argp, argc, argv, 0, 0, &args); args.iotcored_path = iotcored_path; - GglError ret = run_fleet_prov(&args); + pid_t pid = -1; + GglError ret = run_fleet_prov(&args, &pid); if (ret != GGL_ERR_OK) { + if (pid != -1) { + GGL_LOGE("Something went wrong. Killing iotcored"); + ggl_exec_kill_process(pid); + } return 1; } } diff --git a/fleet-provisioning/include/fleet-provisioning.h b/fleet-provisioning/include/fleet-provisioning.h index 52fcc2415..8d29795e1 100644 --- a/fleet-provisioning/include/fleet-provisioning.h +++ b/fleet-provisioning/include/fleet-provisioning.h @@ -5,6 +5,7 @@ #ifndef FLEET_PROVISIONING_H #define FLEET_PROVISIONING_H +#include #include typedef struct { @@ -17,5 +18,5 @@ typedef struct { char *iotcored_path; } FleetProvArgs; -GglError run_fleet_prov(FleetProvArgs *args); +GglError run_fleet_prov(FleetProvArgs *args, pid_t *pid); #endif diff --git a/fleet-provisioning/src/entry.c b/fleet-provisioning/src/entry.c index 87e19f3b3..32a23f3c1 100644 --- a/fleet-provisioning/src/entry.c +++ b/fleet-provisioning/src/entry.c @@ -6,23 +6,27 @@ #include "generate_certificate.h" #include "ggl/exec.h" #include "provisioner.h" +#include "stdbool.h" #include +#include #include #include #include +#include #include #include +#include #include #include #include #include -#include #include #include #define MAX_TEMPLATE_LEN 129 #define MAX_ENDPOINT_LENGTH 129 #define MAX_TEMPLATE_PARAM_LEN 4096 +#define MAX_PATH_LEN 4096 static GglError start_iotcored(FleetProvArgs *args, pid_t *iotcored_pid) { char *iotcore_d_args[] @@ -132,7 +136,7 @@ static GglError fetch_from_db(FleetProvArgs *args) { GGL_STR("iotDataEndpoint") ), GGL_OBJ_BUF(data_endpoint), - &(int64_t) { 0 } + &(int64_t) { 3 } ); if (ret != GGL_ERR_OK) { return ret; @@ -187,8 +191,98 @@ static GglError fetch_from_db(FleetProvArgs *args) { return GGL_ERR_OK; } -GglError run_fleet_prov(FleetProvArgs *args) { - GglError ret = fetch_from_db(args); +static GglError update_cred_access(void) { + char *args[] = { "chown", "-R", "ggcore:ggcore", "/ggcredentials/", NULL }; + + GglError ret = ggl_exec_command(args); + if (ret != GGL_ERR_OK) { + GGL_LOGE("Failed to change ownership of certificates"); + return ret; + } + + char *args_reboot[] = { "systemctl", "reboot", NULL }; + ret = ggl_exec_command(args_reboot); + if (ret != GGL_ERR_OK) { + GGL_LOGE("Failed to reboot the device"); + return ret; + } + + return GGL_ERR_OK; +} + +static GglError update_iot_endpoints(void) { + static uint8_t endpoint_mem[2048] = { 0 }; + GglBuffer data_endpoint = GGL_BUF(endpoint_mem); + GglError ret = ggl_gg_config_read_str( + GGL_BUF_LIST( + GGL_STR("services"), + GGL_STR("aws.greengrass.fleet_provisioning"), + GGL_STR("configuration"), + GGL_STR("iotDataEndpoint") + ), + &data_endpoint + ); + if (ret != GGL_ERR_OK) { + return ret; + } + + ret = ggl_gg_config_write( + GGL_BUF_LIST( + GGL_STR("system"), + GGL_STR("aws.greengrass.NucleusLite"), + GGL_STR("configuration"), + GGL_STR("iotDataEndpoint") + ), + GGL_OBJ_BUF(data_endpoint), + &(int64_t) { 3 } + ); + if (ret != GGL_ERR_OK) { + return ret; + } + + GglBuffer cred_endpoint = GGL_BUF(endpoint_mem); + ret = ggl_gg_config_read_str( + GGL_BUF_LIST( + GGL_STR("services"), + GGL_STR("aws.greengrass.fleet_provisioning"), + GGL_STR("configuration"), + GGL_STR("iotCredEndpoint") + ), + &cred_endpoint + ); + if (ret != GGL_ERR_OK) { + return ret; + } + + ret = ggl_gg_config_write( + GGL_BUF_LIST( + GGL_STR("system"), + GGL_STR("aws.greengrass.NucleusLite"), + GGL_STR("configuration"), + GGL_STR("iotCredEndpoint") + ), + GGL_OBJ_BUF(cred_endpoint), + &(int64_t) { 3 } + ); + if (ret != GGL_ERR_OK) { + return ret; + } + + return GGL_ERR_OK; +} + +GglError run_fleet_prov(FleetProvArgs *args, pid_t *pid) { + GglBuffer ggcredentials_path = GGL_STR("/ggcredentials"); + + int config_dir; + GglError ret + = ggl_dir_open(ggcredentials_path, O_RDONLY, false, &config_dir); + if (ret != GGL_ERR_OK) { + GGL_LOGI("Could not open ggcredentials directory."); + return GGL_ERR_FAILURE; + } + + ret = fetch_from_db(args); if (ret != GGL_ERR_OK) { return ret; } @@ -211,28 +305,60 @@ GglError run_fleet_prov(FleetProvArgs *args) { if (ret != GGL_ERR_OK) { return ret; } + *pid = iotcored_pid; - static char private_file_path[4096] = { 0 }; - static char public_file_path[4096] = { 0 }; - static char csr_file_path[4096] = { 0 }; - static char cert_file_path[4096] = { 0 }; - - strncat(private_file_path, (char *) root_dir.data, root_dir.len); - strncat(public_file_path, (char *) root_dir.data, root_dir.len); - strncat(csr_file_path, (char *) root_dir.data, root_dir.len); - strncat(cert_file_path, (char *) root_dir.data, root_dir.len); + static uint8_t private_file_path_mem[MAX_PATH_LEN] = { 0 }; + GglByteVec private_file_path_vec = GGL_BYTE_VEC(private_file_path_mem); + ret = ggl_byte_vec_append(&private_file_path_vec, ggcredentials_path); + ggl_byte_vec_chain_append( + &ret, &private_file_path_vec, GGL_STR("/private_key.pem.key") + ); + ggl_byte_vec_chain_push(&ret, &private_file_path_vec, '\0'); + if (ret != GGL_ERR_OK) { + GGL_LOGE("Error to append private key path"); + return ret; + } - strncat( - private_file_path, "/private_key.pem", strlen("/private_key.pem.key") + static uint8_t public_file_path_mem[MAX_PATH_LEN] = { 0 }; + GglByteVec public_file_path_vec = GGL_BYTE_VEC(public_file_path_mem); + ret = ggl_byte_vec_append(&public_file_path_vec, ggcredentials_path); + ggl_byte_vec_chain_append( + &ret, &public_file_path_vec, GGL_STR("/public_key.pem.key") ); - strncat(public_file_path, "/public_key.pem", strlen("/public_key.pem")); - strncat(csr_file_path, "/csr.pem", strlen("/csr.pem")); - strncat( - cert_file_path, "/certificate.pem.crt", strlen("/certificate.pem.crt") + ggl_byte_vec_chain_push(&ret, &public_file_path_vec, '\0'); + if (ret != GGL_ERR_OK) { + GGL_LOGE("Error to append public key path"); + return ret; + } + + static uint8_t cert_file_path_mem[MAX_PATH_LEN] = { 0 }; + GglByteVec cert_file_path_vec = GGL_BYTE_VEC(cert_file_path_mem); + ret = ggl_byte_vec_append(&cert_file_path_vec, ggcredentials_path); + ggl_byte_vec_chain_append( + &ret, &cert_file_path_vec, GGL_STR("/certificate.pem.crt") ); + ggl_byte_vec_chain_push(&ret, &cert_file_path_vec, '\0'); + if (ret != GGL_ERR_OK) { + GGL_LOGE("Error to append certificate key path"); + return ret; + } + + static uint8_t csr_file_path_mem[MAX_PATH_LEN] = { 0 }; + GglByteVec csr_file_path_vec = GGL_BYTE_VEC(csr_file_path_mem); + ret = ggl_byte_vec_append(&csr_file_path_vec, ggcredentials_path); + ggl_byte_vec_chain_append(&ret, &csr_file_path_vec, GGL_STR("/csr.pem")); + ggl_byte_vec_chain_push(&ret, &csr_file_path_vec, '\0'); + if (ret != GGL_ERR_OK) { + GGL_LOGE("Error to append csr path"); + return ret; + } generate_key_files( - pkey, csr_req, private_file_path, public_file_path, csr_file_path + pkey, + csr_req, + (char *) private_file_path_vec.buf.data, + (char *) public_file_path_vec.buf.data, + (char *) csr_file_path_vec.buf.data ); EVP_PKEY_free(pkey); @@ -240,48 +366,43 @@ GglError run_fleet_prov(FleetProvArgs *args) { ret = ggl_gg_config_write( GGL_BUF_LIST(GGL_STR("system"), GGL_STR("privateKeyPath")), - GGL_OBJ_BUF((GglBuffer) { .data = (uint8_t *) private_file_path, - .len = strlen(private_file_path) }), - &(int64_t) { 0 } + GGL_OBJ_BUF(private_file_path_vec.buf), + &(int64_t) { 3 } ); if (ret != GGL_ERR_OK) { - ggl_exec_kill_process(iotcored_pid); return ret; } - static char csr_buf[2048] = { 0 }; - FILE *fp; - ulong file_size; + static uint8_t csr_mem[2048] = { 0 }; - // Open the file in binary mode - fp = fopen("./csr.pem", "rb"); - if (fp == NULL) { - perror("Error opening file"); + // Try to read csr into memory + int fd = -1; + ret = ggl_file_open(csr_file_path_vec.buf, O_RDONLY, 0, &fd); + if (ret != GGL_ERR_OK) { + GGL_LOGE("Error opening csr file %d", ret); return 1; } - // Get the file size - fseek(fp, 0, SEEK_END); - file_size = (ulong) ftell(fp); - fseek(fp, 0, SEEK_SET); - - // Read the file into the buffer - size_t read_size = fread(csr_buf, 1, file_size, fp); - - // Close the file - fclose(fp); - - if (read_size != file_size) { - GGL_LOGE("Failed to read th whole file."); + GglBuffer csr_buf = GGL_BUF(csr_mem); + ret = ggl_file_read(fd, &csr_buf); + if (ret != GGL_ERR_OK) { + GGL_LOGE("Failed to read csr file."); return GGL_ERR_FAILURE; } + GGL_LOGD("CSR successfully read.."); - ret = make_request(csr_buf, cert_file_path, iotcored_pid); + ret = make_request(csr_buf, cert_file_path_vec.buf, iotcored_pid); + if (ret != GGL_ERR_OK) { + return ret; + } + ret = update_iot_endpoints(); if (ret != GGL_ERR_OK) { - GGL_LOGE("Something went wrong. Killing iotcored"); - ggl_exec_kill_process(iotcored_pid); + return ret; + } + ret = update_cred_access(); + if (ret != GGL_ERR_OK) { return ret; } diff --git a/fleet-provisioning/src/provisioner.c b/fleet-provisioning/src/provisioner.c index 7eaf84443..dfc652753 100644 --- a/fleet-provisioning/src/provisioner.c +++ b/fleet-provisioning/src/provisioner.c @@ -20,6 +20,8 @@ #include #include #include +#include +#include #include #define TEMPLATE_PARAM_BUFFER_SIZE 10000 @@ -37,6 +39,8 @@ static uint8_t big_buffer_for_bump[4096]; GglObject csr_payload_json_obj; char *global_cert_file_path; +atomic_bool complete_status = false; + static GglBuffer iotcored = GGL_STR("iotcoredfleet"); static const char *certificate_response_url @@ -47,7 +51,7 @@ static const char *certificate_response_reject_url static const char *cert_request_url = "$aws/certificates/create-from-csr/json"; -static int request_thing_name(GglObject *cert_owner_gg_obj) { +static GglError request_thing_name(GglObject *cert_owner_gg_obj) { static uint8_t temp_payload_alloc2[2000] = { 0 }; GglBuffer thing_request_buf = GGL_BUF(temp_payload_alloc2); @@ -104,7 +108,7 @@ static int request_thing_name(GglObject *cert_owner_gg_obj) { (int) iotcored.len, iotcored.data ); - return EPROTO; + return GGL_ERR_FAILURE; } GGL_LOGI("Sent MQTT thing Register publish."); @@ -143,7 +147,9 @@ static GglError set_global_values(pid_t iotcored_pid) { template_name.len ); strncat( - global_register_thing_url, "/provision/json", strlen("/provision/json") + global_register_thing_url, + "/provision/json", + sizeof("/provision/json") - 1U ); // Copy the prefix over to both buffer @@ -153,14 +159,18 @@ static GglError set_global_values(pid_t iotcored_pid) { global_register_thing_url, strlen(global_register_thing_url) ); - strncat(global_register_thing_accept_url, "/accepted", strlen("/accepted")); + strncat( + global_register_thing_accept_url, "/accepted", sizeof("/accepted") - 1U + ); // Add failure suffix strncat( global_register_thing_reject_url, global_register_thing_url, strlen(global_register_thing_url) ); - strncat(global_register_thing_reject_url, "/rejected", strlen("/accepted")); + strncat( + global_register_thing_reject_url, "/rejected", sizeof("/rejected") - 1U + ); // Fetch Template Parameters // TODO: Use args passed from entry.c @@ -195,6 +205,14 @@ static GglError subscribe_callback(void *ctx, uint32_t handle, GglObject data) { return ret; } + GGL_LOGI( + "Got message from IoT Core; topic: %.*s, payload: %.*s.", + (int) topic->len, + topic->data, + (int) payload->len, + payload->data + ); + if (strncmp((char *) topic->data, certificate_response_url, topic->len) == 0) { GglBumpAlloc balloc = ggl_bump_alloc_init(GGL_BUF(big_buffer_for_bump)); @@ -241,7 +259,7 @@ static GglError subscribe_callback(void *ctx, uint32_t handle, GglObject data) { GGL_OBJ_BUF((GglBuffer ) { .data = (uint8_t *) global_cert_file_path, .len = strlen(global_cert_file_path) }), - &(int64_t) { 0 } + &(int64_t) { 3 } ); if (ret != GGL_ERR_OK) { return ret; @@ -266,7 +284,11 @@ static GglError subscribe_callback(void *ctx, uint32_t handle, GglObject data) { // Now that we have a certificate make a call to register a // thing based on that certificate - request_thing_name(val); + ret = request_thing_name(val); + if (ret != GGL_ERR_OK) { + GGL_LOGE("Requesting thing name failed"); + return ret; + } } } } else if (strncmp( @@ -298,7 +320,7 @@ static GglError subscribe_callback(void *ctx, uint32_t handle, GglObject data) { ret = ggl_gg_config_write( GGL_BUF_LIST(GGL_STR("system"), GGL_STR("thingName")), *val, - &(int64_t) { 0 } + &(int64_t) { 3 } ); if (ret != GGL_ERR_OK) { return ret; @@ -309,6 +331,7 @@ static GglError subscribe_callback(void *ctx, uint32_t handle, GglObject data) { ggl_exec_kill_process(global_iotcored_pid); // TODO: Find a way to terminate cleanly with iotcored + atomic_store(&complete_status, true); } } else { GGL_LOGI( @@ -324,9 +347,9 @@ static GglError subscribe_callback(void *ctx, uint32_t handle, GglObject data) { } GglError make_request( - char *csr_as_string, char *cert_file_path, pid_t iotcored_pid + GglBuffer csr_as_ggl_buffer, GglBuffer cert_file_path, pid_t iotcored_pid ) { - global_cert_file_path = cert_file_path; + global_cert_file_path = (char *) cert_file_path.data; GglError ret = set_global_values(iotcored_pid); if (ret != GGL_ERR_OK) { @@ -366,7 +389,6 @@ GglError make_request( } GGL_LOGI("Successfully set csr accepted subscription."); - // NOLINTNEXTLINE(concurrency-mt-unsafe) ggl_sleep(2); // Subscribe to csr reject topic @@ -398,7 +420,6 @@ GglError make_request( } GGL_LOGI("Successfully set csr rejected subscription."); - // NOLINTNEXTLINE(concurrency-mt-unsafe) ggl_sleep(2); // Subscribe to register thing success topic @@ -430,15 +451,41 @@ GglError make_request( } GGL_LOGI("Successfully set thing accepted subscription."); - // NOLINTNEXTLINE(concurrency-mt-unsafe) + // Subscribe to register thing success topic + GglMap subscribe_thing_reject_args = GGL_MAP( + { GGL_STR("topic_filter"), + GGL_OBJ_BUF((GglBuffer + ) { .len = strlen(global_register_thing_reject_url), + .data = (uint8_t *) global_register_thing_reject_url }) }, + ); + + GglError return_thing_sub_reject = ggl_subscribe( + iotcored, + GGL_STR("subscribe"), + subscribe_thing_reject_args, + subscribe_callback, + NULL, + NULL, + NULL, + NULL + ); + if (return_thing_sub_reject != GGL_ERR_OK) { + GGL_LOGE( + "Failed to send thing accepted notify message to %.*s, Error: %d", + (int) iotcored.len, + iotcored.data, + EPROTO + ); + return GGL_ERR_FAILURE; + } + GGL_LOGI("Successfully set thing rejected subscription."); + ggl_sleep(2); // Create a json payload object - GglObject csr_payload_obj = GGL_OBJ_MAP( - GGL_MAP({ GGL_STR("certificateSigningRequest"), - GGL_OBJ_BUF((GglBuffer) { .data = (uint8_t *) csr_as_string, - .len = strlen(csr_as_string) }) }) - ); + GglObject csr_payload_obj + = GGL_OBJ_MAP(GGL_MAP({ GGL_STR("certificateSigningRequest"), + GGL_OBJ_BUF(csr_as_ggl_buffer) })); GglError ret_err_json = ggl_json_encode(csr_payload_obj, &csr_buf); if (ret_err_json != GGL_ERR_OK) { return GGL_ERR_PARSE; @@ -455,8 +502,7 @@ GglError make_request( { GGL_STR("payload"), GGL_OBJ_BUF(csr_buf) }, ); - // NOLINTNEXTLINE(concurrency-mt-unsafe) - ggl_sleep(5); + ggl_sleep(2); // Make Publish request to get the new certificate GglError ret_publish = ggl_notify(iotcored, GGL_STR("publish"), args); @@ -470,7 +516,10 @@ GglError make_request( return GGL_ERR_FAILURE; } - // NOLINTNEXTLINE(concurrency-mt-unsafe) - ggl_sleep(300); + while (!atomic_load(&complete_status)) { // Continuously check the flag + GGL_LOGI("Wating for thing to register"); + ggl_sleep(5); + } + return GGL_ERR_OK; } diff --git a/fleet-provisioning/src/provisioner.h b/fleet-provisioning/src/provisioner.h index a190b4853..493bfe737 100644 --- a/fleet-provisioning/src/provisioner.h +++ b/fleet-provisioning/src/provisioner.h @@ -6,10 +6,11 @@ #define PROVISIONER_H #include +#include #include GglError make_request( - char *csr_as_string, char *cert_file_path, pid_t iotcored_pid + GglBuffer csr_as_ggl_buffer, GglBuffer cert_file_path, pid_t iotcored_pid ); #endif diff --git a/ggdeploymentd/src/bootstrap_manager.c b/ggdeploymentd/src/bootstrap_manager.c index 2e9437d3c..9a047706f 100644 --- a/ggdeploymentd/src/bootstrap_manager.c +++ b/ggdeploymentd/src/bootstrap_manager.c @@ -5,22 +5,51 @@ #include "bootstrap_manager.h" #include "deployment_model.h" #include "deployment_queue.h" +#include "stale_component.h" +#include #include #include #include #include #include -#include #include #include #include #include #include +#include #include +#include #include #include #include +bool component_bootstrap_phase_completed(GglBuffer component_name) { + // check config to see if component bootstrap steps have already been + // completed + uint8_t resp_mem[128] = { 0 }; + GglBuffer resp = GGL_BUF(resp_mem); + GglError ret = ggl_gg_config_read_str( + GGL_BUF_LIST( + GGL_STR("services"), + GGL_STR("DeploymentService"), + GGL_STR("deploymentState"), + GGL_STR("bootstrapComponents"), + component_name + ), + &resp + ); + if (ret == GGL_ERR_OK) { + GGL_LOGD( + "Bootstrap steps have already been run for %.*s.", + (int) component_name.len, + component_name.data + ); + return true; + } + return false; +} + GglError save_component_info( GglBuffer component_name, GglBuffer component_version, GglBuffer type ) { @@ -44,7 +73,7 @@ GglError save_component_info( component_name ), GGL_OBJ_BUF(component_version), - &(int64_t) { 0 } + &(int64_t) { 3 } ); if (ret != GGL_ERR_OK) { GGL_LOGE( @@ -64,7 +93,7 @@ GglError save_component_info( component_name ), GGL_OBJ_BUF(component_version), - &(int64_t) { 0 } + &(int64_t) { 3 } ); if (ret != GGL_ERR_OK) { GGL_LOGE( @@ -102,7 +131,7 @@ GglError save_iot_jobs_id(GglBuffer jobs_id) { GGL_STR("jobsID") ), GGL_OBJ_BUF(jobs_id), - &(int64_t) { 0 } + &(int64_t) { 3 } ); if (ret != GGL_ERR_OK) { GGL_LOGE("Failed to write IoT Jobs ID to config."); @@ -111,6 +140,29 @@ GglError save_iot_jobs_id(GglBuffer jobs_id) { return GGL_ERR_OK; } +GglError save_iot_jobs_version(int64_t jobs_version) { + GGL_LOGD( + "Saving IoT Jobs version %" PRIi64 " in case of bootstrap.", + jobs_version + ); + + GglError ret = ggl_gg_config_write( + GGL_BUF_LIST( + GGL_STR("services"), + GGL_STR("DeploymentService"), + GGL_STR("deploymentState"), + GGL_STR("jobsVersion") + ), + GGL_OBJ_I64(jobs_version), + &(int64_t) { 3 } + ); + if (ret != GGL_ERR_OK) { + GGL_LOGE("Failed to write IoT Jobs Version to config."); + return ret; + } + return GGL_ERR_OK; +} + GglError save_deployment_info(GglDeployment *deployment) { GGL_LOGD("Encountered component requiring bootstrap. Saving deployment " "state to config."); @@ -135,7 +187,7 @@ GglError save_deployment_info(GglDeployment *deployment) { GGL_STR("deploymentDoc") ), deployment_doc, - &(int64_t) { 0 } + &(int64_t) { 3 } ); if (ret != GGL_ERR_OK) { @@ -159,7 +211,7 @@ GglError save_deployment_info(GglDeployment *deployment) { GGL_STR("deploymentType") ), GGL_OBJ_BUF(deployment_type), - &(int64_t) { 0 } + &(int64_t) { 3 } ); if (ret != GGL_ERR_OK) { @@ -171,29 +223,15 @@ GglError save_deployment_info(GglDeployment *deployment) { } GglError retrieve_in_progress_deployment( - GglDeployment *deployment, GglBuffer *jobs_id + GglDeployment *deployment, GglBuffer *jobs_id, int64_t *jobs_version ) { GGL_LOGD("Searching config for any in progress deployment."); - GglError ret = ggl_gg_config_read_str( - GGL_BUF_LIST( - GGL_STR("services"), - GGL_STR("DeploymentService"), - GGL_STR("deploymentState"), - GGL_STR("jobsID") - ), - jobs_id - ); - if (ret != GGL_ERR_OK) { - GGL_LOGW("Failed to retrieve IoT Jobs ID from config."); - return ret; - } - GglBuffer config_mem = GGL_BUF((uint8_t[2500]) { 0 }); GglBumpAlloc balloc = ggl_bump_alloc_init(config_mem); GglObject deployment_config; - ret = ggl_gg_config_read( + GglError ret = ggl_gg_config_read( GGL_BUF_LIST( GGL_STR("services"), GGL_STR("DeploymentService"), @@ -210,11 +248,34 @@ GglError retrieve_in_progress_deployment( return GGL_ERR_INVALID; } + GglObject *jobs_id_obj; + ret = ggl_map_validate( + deployment_config.map, + GGL_MAP_SCHEMA({ GGL_STR("jobsID"), true, GGL_TYPE_BUF, &jobs_id_obj }) + ); + if (ret != GGL_ERR_OK) { + return ret; + } + assert(jobs_id_obj->buf.len < 64); + memcpy(jobs_id->data, jobs_id_obj->buf.data, jobs_id_obj->buf.len); + + GglObject *jobs_version_obj; + ret = ggl_map_validate( + deployment_config.map, + GGL_MAP_SCHEMA( + { GGL_STR("jobsVersion"), true, GGL_TYPE_I64, &jobs_version_obj } + ) + ); + if (ret != GGL_ERR_OK) { + return ret; + } + *jobs_version = jobs_version_obj->i64; + GglObject *deployment_type; ret = ggl_map_validate( deployment_config.map, GGL_MAP_SCHEMA( - { GGL_STR("deploymentType"), false, GGL_TYPE_BUF, &deployment_type } + { GGL_STR("deploymentType"), true, GGL_TYPE_BUF, &deployment_type } ) ); if (ret != GGL_ERR_OK) { @@ -233,7 +294,7 @@ GglError retrieve_in_progress_deployment( ret = ggl_map_validate( deployment_config.map, GGL_MAP_SCHEMA( - { GGL_STR("deploymentDoc"), false, GGL_TYPE_MAP, &deployment_doc } + { GGL_STR("deploymentDoc"), true, GGL_TYPE_MAP, &deployment_doc } ) ); if (ret != GGL_ERR_OK) { @@ -358,32 +419,16 @@ GglError process_bootstrap_phase( // check config to see if component bootstrap steps have already been // completed - uint8_t resp_mem[128] = { 0 }; - GglBuffer resp = GGL_BUF(resp_mem); - GglError ret = ggl_gg_config_read_str( - GGL_BUF_LIST( - GGL_STR("services"), - GGL_STR("DeploymentService"), - GGL_STR("deploymentState"), - GGL_STR("bootstrapComponents"), - component_name - ), - &resp - ); - if (ret == GGL_ERR_OK) { - GGL_LOGD( - "Bootstrap steps have already been run for %.*s. Skipping " - "component.", - (int) component_name.len, - component_name.data - ); + if (component_bootstrap_phase_completed(component_name)) { + GGL_LOGD("Bootstrap processed. Skipping component."); continue; } static uint8_t bootstrap_service_file_path_buf[PATH_MAX]; GglByteVec bootstrap_service_file_path_vec = GGL_BYTE_VEC(bootstrap_service_file_path_buf); - ret = ggl_byte_vec_append(&bootstrap_service_file_path_vec, root_path); + GglError ret + = ggl_byte_vec_append(&bootstrap_service_file_path_vec, root_path); ggl_byte_vec_append(&bootstrap_service_file_path_vec, GGL_STR("/")); ggl_byte_vec_append(&bootstrap_service_file_path_vec, GGL_STR("ggl.")); ggl_byte_vec_chain_append( @@ -409,6 +454,7 @@ GglError process_bootstrap_phase( component_name.data ); } else { // relevant bootstrap service file exists + disable_and_unlink_service(&component_name, BOOTSTRAP); GGL_LOGI( "Found bootstrap service file for %.*s. Processing.", (int) component_name.len, @@ -554,11 +600,18 @@ GglError process_bootstrap_phase( } GGL_LOGI("Rebooting device for bootstrap."); - char *reboot_args[] = { "reboot", NULL }; - ret = ggl_exec_command_async(reboot_args, NULL); - if (ret != GGL_ERR_OK) { - GGL_LOGE("Failed to reboot system for bootstrap."); - return ret; + // NOLINTNEXTLINE(concurrency-mt-unsafe) + int system_ret = system("systemctl reboot"); + if (WIFEXITED(system_ret)) { + if (WEXITSTATUS(system_ret) != 0) { + GGL_LOGE("systemctl reboot failed"); + } + GGL_LOGI( + "systemctl reboot exited with child status %d\n", + WEXITSTATUS(system_ret) + ); + } else { + GGL_LOGE("systemctl reboot did not exit normally"); } } diff --git a/ggdeploymentd/src/bootstrap_manager.h b/ggdeploymentd/src/bootstrap_manager.h index 78988f110..8c3ccf618 100644 --- a/ggdeploymentd/src/bootstrap_manager.h +++ b/ggdeploymentd/src/bootstrap_manager.h @@ -10,6 +10,8 @@ #include #include #include +#include +#include /* deployment info will be saved to config in the following format: @@ -25,8 +27,11 @@ deploymentType: local/IoT Jobs deploymentDoc: jobsID: + jobsVersion: */ +bool component_bootstrap_phase_completed(GglBuffer component_name); + // type can be "bootstrap" or "completed" // bootstrap type indicates that the component's bootstrap steps have completed // running completed type indicates that the component completed deployment @@ -35,9 +40,10 @@ GglError save_component_info( ); GglError save_iot_jobs_id(GglBuffer jobs_id); +GglError save_iot_jobs_version(int64_t jobs_version); GglError save_deployment_info(GglDeployment *deployment); GglError retrieve_in_progress_deployment( - GglDeployment *deployment, GglBuffer *jobs_id + GglDeployment *deployment, GglBuffer *jobs_id, int64_t *jobs_version ); GglError delete_saved_deployment_from_config(void); GglError process_bootstrap_phase( diff --git a/ggdeploymentd/src/component_manager.c b/ggdeploymentd/src/component_manager.c index 5ad3545ab..3c08d287c 100644 --- a/ggdeploymentd/src/component_manager.c +++ b/ggdeploymentd/src/component_manager.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -34,8 +35,9 @@ static GglError find_active_version( if (ret != GGL_ERR_OK) { GGL_LOGI( - "Unable to retrieve version of %s. Assuming no active version " + "Unable to retrieve version of %.*s. Assuming no active version " "found.", + (int) package_name.len, package_name.data ); return GGL_ERR_NOENTRY; @@ -45,6 +47,35 @@ static GglError find_active_version( if (!is_in_range(version_resp, version_requirement)) { return GGL_ERR_NOENTRY; } + + // Check that the component is actually running (or finished) + uint8_t component_status_buf[NAME_MAX]; + GglBuffer component_status = GGL_BUF(component_status_buf); + ret = ggl_gghealthd_retrieve_component_status( + package_name, &component_status + ); + + if (ret != GGL_ERR_OK) { + GGL_LOGI( + "Component status not found for component %.*s despite finding " + "active version. Not using this version.", + (int) package_name.len, + package_name.data + ); + return GGL_ERR_INVALID; + } + + if (!ggl_buffer_eq(component_status, GGL_STR("RUNNING")) + && !ggl_buffer_eq(component_status, GGL_STR("FINISHED"))) { + GGL_LOGI( + "Component %.*s is not in the RUNNING or FINISHED states. Not " + "using the active version.", + (int) package_name.len, + package_name.data + ); + return GGL_ERR_INVALID; + } + *version = version_resp; return GGL_ERR_OK; } diff --git a/ggdeploymentd/src/component_store.c b/ggdeploymentd/src/component_store.c index e3b8825d8..220a345bb 100644 --- a/ggdeploymentd/src/component_store.c +++ b/ggdeploymentd/src/component_store.c @@ -79,18 +79,29 @@ GglError iterate_over_components( GglBuffer *version, struct dirent **entry ) { + GGL_LOGT("Iterating over component recipes in directory"); // NOLINTNEXTLINE(concurrency-mt-unsafe) while ((*entry = readdir(dir)) != NULL) { GglBuffer entry_buf = ggl_buffer_from_null_term((*entry)->d_name); + GGL_LOGT( + "Found directory entry %.*s", (int) entry_buf.len, entry_buf.data + ); // recipe file names follow this format: // -. - // Split directory entry on the index of the "-" character. + // Split the last "-" character to retrieve the component name GglBuffer recipe_component; GglBuffer rest = GGL_STR(""); - for (size_t i = 0; i < entry_buf.len; ++i) { - if (entry_buf.data[i] == '-') { - recipe_component = ggl_buffer_substr(entry_buf, 0, i); - rest = ggl_buffer_substr(entry_buf, i + 1, SIZE_MAX); + for (size_t i = entry_buf.len; i > 0; --i) { + if (entry_buf.data[i - 1] == '-') { + recipe_component = ggl_buffer_substr(entry_buf, 0, i - 1); + rest = ggl_buffer_substr(entry_buf, i, SIZE_MAX); + GGL_LOGT( + "Split entry on '-': component: %.*s rest: %.*s", + (int) recipe_component.len, + recipe_component.data, + (int) rest.len, + rest.data + ); break; } } @@ -107,6 +118,11 @@ GglError iterate_over_components( for (size_t i = rest.len; i > 0; i--) { if (rest.data[i - 1] == '.') { recipe_version = ggl_buffer_substr(rest, 0, i - 1); + GGL_LOGT( + "Found version: %.*s", + (int) recipe_version.len, + recipe_version.data + ); break; } } @@ -133,6 +149,11 @@ GglError iterate_over_components( GglError find_available_component( GglBuffer component_name, GglBuffer requirement, GglBuffer *version ) { + GGL_LOGT( + "Searching for component %.*s", + (int) component_name.len, + component_name.data + ); int recipe_dir_fd; GglError ret = get_recipe_dir_fd(&recipe_dir_fd); if (ret != GGL_ERR_OK) { diff --git a/ggdeploymentd/src/deployment_handler.c b/ggdeploymentd/src/deployment_handler.c index 7bc1c369b..de95df965 100644 --- a/ggdeploymentd/src/deployment_handler.c +++ b/ggdeploymentd/src/deployment_handler.c @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -46,8 +47,8 @@ #include #include -#define MAX_RECIPE_BUF_SIZE 256000 #define MAX_DECODE_BUF_LEN 4096 +#define MAX_RECIPE_MEM 25000 static struct DeploymentConfiguration { char data_endpoint[128]; @@ -817,33 +818,85 @@ static GglError get_device_thing_groups(GglBuffer *response) { static GglError generate_resolve_component_candidates_body( GglBuffer component_name, GglBuffer component_requirements, - GglByteVec *body_vec + GglByteVec *body_vec, + GglAlloc *alloc ) { - GglError byte_vec_ret = GGL_ERR_OK; - ggl_byte_vec_chain_append( - &byte_vec_ret, body_vec, GGL_STR("{\"componentCandidates\": [") + GglObject architecture_detail_read_value; + GglError ret = ggl_gg_config_read( + GGL_BUF_LIST( + GGL_STR("services"), + GGL_STR("aws.greengrass.NucleusLite"), + GGL_STR("configuration"), + GGL_STR("platformOverride"), + GGL_STR("architecture.detail") + ), + alloc, + &architecture_detail_read_value ); + if (ret != GGL_ERR_OK) { + GGL_LOGD("No architecture.detail found, so not including it in the " + "component candidates search."); + architecture_detail_read_value = GGL_OBJ_BUF(GGL_STR("")); + } - ggl_byte_vec_chain_append( - &byte_vec_ret, body_vec, GGL_STR("{\"componentName\": \"") + if (architecture_detail_read_value.type != GGL_TYPE_BUF) { + GGL_LOGD( + "architecture.detail platformOverride in the config is not a " + "buffer, so not including it in the component candidates search" + ); + architecture_detail_read_value = GGL_OBJ_BUF(GGL_STR("")); + } + + // TODO: Support platform attributes for platformOverride configuration + GglMap platform_attributes = GGL_MAP( + { GGL_STR("runtime"), GGL_OBJ_BUF(GGL_STR("aws_nucleus_lite")) }, + { GGL_STR("os"), GGL_OBJ_BUF(GGL_STR("linux")) }, + { GGL_STR("architecture"), GGL_OBJ_BUF(get_current_architecture()) }, ); - ggl_byte_vec_chain_append(&byte_vec_ret, body_vec, component_name); - ggl_byte_vec_chain_append( - &byte_vec_ret, - body_vec, - GGL_STR("\",\"versionRequirements\": {\"requirements\": \"") + + if (architecture_detail_read_value.buf.len != 0) { + platform_attributes = GGL_MAP( + { GGL_STR("runtime"), GGL_OBJ_BUF(GGL_STR("aws_nucleus_lite")) }, + { GGL_STR("os"), GGL_OBJ_BUF(GGL_STR("linux")) }, + { GGL_STR("architecture"), + GGL_OBJ_BUF(get_current_architecture()) }, + { GGL_STR("architecture.detail"), architecture_detail_read_value } + ); + } + + GglMap platform_info = GGL_MAP( + { GGL_STR("name"), GGL_OBJ_BUF(GGL_STR("linux")) }, + { GGL_STR("attributes"), GGL_OBJ_MAP(platform_attributes) } ); - ggl_byte_vec_chain_append(&byte_vec_ret, body_vec, component_requirements); - ggl_byte_vec_chain_append(&byte_vec_ret, body_vec, GGL_STR("\"}}")); - // TODO: Include architecture requirements if any - ggl_byte_vec_chain_append( - &byte_vec_ret, - body_vec, - GGL_STR("],\"platform\": { \"attributes\": { \"os\" : \"linux\", " - "\"runtime\" : \"aws_nucleus_lite\" " - "},\"name\": \"linux\"}}") + GglMap version_requirements_map + = GGL_MAP({ GGL_STR("requirements"), + GGL_OBJ_BUF(component_requirements) }); + + GglMap component_map = GGL_MAP( + { GGL_STR("componentName"), GGL_OBJ_BUF(component_name) }, + { GGL_STR("versionRequirements"), + GGL_OBJ_MAP(version_requirements_map) } + ); + + GglList candidates_list = GGL_LIST(GGL_OBJ_MAP(component_map)); + + GglMap request_body = GGL_MAP( + { GGL_STR("componentCandidates"), GGL_OBJ_LIST(candidates_list) }, + { GGL_STR("platform"), GGL_OBJ_MAP(platform_info) } ); + + static uint8_t rcc_buf[4096]; + GglBuffer rcc_body = GGL_BUF(rcc_buf); + ret = ggl_json_encode(GGL_OBJ_MAP(request_body), &rcc_body); + if (ret != GGL_ERR_OK) { + GGL_LOGE("Error while encoding body for ResolveComponentCandidates call" + ); + return ret; + } + + GglError byte_vec_ret = GGL_ERR_OK; + ggl_byte_vec_chain_append(&byte_vec_ret, body_vec, rcc_body); ggl_byte_vec_chain_push(&byte_vec_ret, body_vec, '\0'); GGL_LOGD("Body for call: %s", body_vec->buf.data); @@ -858,8 +911,11 @@ static GglError resolve_component_with_cloud( ) { static char resolve_candidates_body_buf[2048]; GglByteVec body_vec = GGL_BYTE_VEC(resolve_candidates_body_buf); + static uint8_t rcc_body_config_read_mem[128]; + GglBumpAlloc rcc_balloc + = ggl_bump_alloc_init(GGL_BUF(rcc_body_config_read_mem)); GglError ret = generate_resolve_component_candidates_body( - component_name, version_requirements, &body_vec + component_name, version_requirements, &body_vec, &rcc_balloc.alloc ); if (ret != GGL_ERR_OK) { GGL_LOGE("Failed to generate body for resolveComponentCandidates call"); @@ -1584,7 +1640,7 @@ static GglError resolve_dependencies( // Get actual recipe read GglObject recipe_obj; - static uint8_t recipe_mem[8192] = { 0 }; + static uint8_t recipe_mem[MAX_RECIPE_MEM] = { 0 }; GglBumpAlloc balloc = ggl_bump_alloc_init(GGL_BUF(recipe_mem)); ret = ggl_recipe_get_from_file( args->root_path_fd, @@ -1790,6 +1846,17 @@ static GglError open_component_artifacts_dir( ); } +static GglBuffer get_unversioned_substring(GglBuffer arn) { + size_t colon_index = SIZE_MAX; + for (size_t i = arn.len; i > 0; i--) { + if (arn.data[i - 1] == ':') { + colon_index = i - 1; + break; + } + } + return ggl_buffer_substr(arn, 0, colon_index); +} + static GglError add_arn_list_to_config( GglBuffer component_name, GglBuffer configuration_arn ) { @@ -1818,7 +1885,7 @@ static GglError add_arn_list_to_config( return GGL_ERR_FAILURE; } - GglObjVec new_arn_list = GGL_OBJ_VEC((GglObject[10]) { 0 }); + GglObjVec new_arn_list = GGL_OBJ_VEC((GglObject[100]) { 0 }); if (ret != GGL_ERR_NOENTRY) { // list exists in config, parse for current config arn and append if it // is not already included @@ -1826,10 +1893,10 @@ static GglError add_arn_list_to_config( GGL_LOGE("Configuration arn list not of expected type."); return GGL_ERR_INVALID; } - if (arn_list.list.len >= 10) { + if (arn_list.list.len >= 100) { GGL_LOGE( "Cannot append configArn: Component is deployed as part of too " - "many thing groups (%zu >= 10).", + "many deployments (%zu >= 100).", arn_list.list.len ); } @@ -1838,8 +1905,29 @@ static GglError add_arn_list_to_config( GGL_LOGE("Configuration arn not of type buffer."); return ret; } - if (ggl_buffer_eq(arn->buf, configuration_arn)) { - // arn already added to config + if (ggl_buffer_eq( + get_unversioned_substring(arn->buf), + get_unversioned_substring(configuration_arn) + )) { + // arn for this group already added to config, replace it + GGL_LOGD("Configuration arn already exists for this thing " + "group, overwriting it."); + *arn = GGL_OBJ_BUF(configuration_arn); + ret = ggl_gg_config_write( + GGL_BUF_LIST( + GGL_STR("services"), + component_name, + GGL_STR("configArn") + ), + GGL_OBJ_LIST(arn_list.list), + &(int64_t) { 3 } + ); + if (ret != GGL_ERR_OK) { + GGL_LOGE( + "Failed to write configuration arn list to the config." + ); + return ret; + } return GGL_ERR_OK; } ret = ggl_obj_vec_push(&new_arn_list, *arn); @@ -1853,7 +1941,7 @@ static GglError add_arn_list_to_config( ret = ggl_gg_config_write( GGL_BUF_LIST(GGL_STR("services"), component_name, GGL_STR("configArn")), GGL_OBJ_LIST(new_arn_list.list), - 0 + &(int64_t) { 3 } ); if (ret != GGL_ERR_OK) { GGL_LOGE("Failed to write configuration arn list to the config."); @@ -1982,6 +2070,11 @@ static GglError wait_for_phase_status( GglError ret = ggl_byte_vec_append( &full_comp_name_vec, component_vec.buf_list.bufs[i] ); + ggl_byte_vec_chain_push(&ret, &full_comp_name_vec, '.'); + if (ret != GGL_ERR_OK) { + GGL_LOGE("Failed to push '.' character to component name vector."); + return ret; + } ggl_byte_vec_append(&full_comp_name_vec, phase); if (ret != GGL_ERR_OK) { GGL_LOGE( @@ -2173,6 +2266,31 @@ static void handle_deployment( continue; } + // check config to see if bootstrap steps have already been run for this + // component + if (component_bootstrap_phase_completed(pair->key)) { + GGL_LOGD( + "Bootstrap component %.*s encountered. Bootstrap phase has " + "already been completed. Adding to list of components to " + "process to complete any other lifecycle stages.", + (int) pair->key.len, + pair->key.data + ); + ret = ggl_kv_vec_push( + &components_to_deploy, (GglKV) { pair->key, pair->val } + ); + if (ret != GGL_ERR_OK) { + GGL_LOGE( + "Failed to add component info for %.*s to deployment " + "vector.", + (int) pair->key.len, + pair->key.data + ); + return; + } + continue; + } + int component_artifacts_fd = -1; ret = open_component_artifacts_dir( artifact_store_fd, pair->key, pair->val.buf, &component_artifacts_fd @@ -2193,7 +2311,7 @@ static void handle_deployment( return; } GglObject recipe_obj; - static uint8_t recipe_mem[8192] = { 0 }; + static uint8_t recipe_mem[MAX_RECIPE_MEM] = { 0 }; static uint8_t component_arn_buffer[256]; GglBumpAlloc balloc = ggl_bump_alloc_init(GGL_BUF(recipe_mem)); ret = ggl_recipe_get_from_file( @@ -2306,7 +2424,7 @@ static void handle_deployment( GglObject recipe_buff_obj; GglObject *component_name; - static uint8_t big_buffer_for_bump[MAX_RECIPE_BUF_SIZE]; + static uint8_t big_buffer_for_bump[MAX_RECIPE_MEM]; GglBumpAlloc bump_alloc = ggl_bump_alloc_init(GGL_BUF(big_buffer_for_bump)); HasPhase phases = { 0 }; @@ -2322,6 +2440,12 @@ static void handle_deployment( return; } + if (!ggl_buffer_eq(component_name->buf, pair->key)) { + GGL_LOGE("Component name from recipe does not match component name " + "from recipe file."); + return; + } + // TODO: See if there is a better requirement. If a customer has the // same version as before but somehow updated their component // version their component may not get the updates. @@ -2358,7 +2482,11 @@ static void handle_deployment( ); if (ret != GGL_ERR_OK) { - GGL_LOGE("Failed to write component version to ggconfigd."); + GGL_LOGE( + "Failed to write version of %.*s to ggconfigd.", + (int) pair->key.len, + pair->key.data + ); return; } @@ -2367,7 +2495,11 @@ static void handle_deployment( ); if (ret != GGL_ERR_OK) { - GGL_LOGE("Failed to write configuration arn to ggconfigd."); + GGL_LOGE( + "Failed to write configuration arn of %.*s to ggconfigd.", + (int) pair->key.len, + pair->key.data + ); return; } @@ -2375,7 +2507,11 @@ static void handle_deployment( deployment, component_name->buf, GGL_STR("reset") ); if (ret != GGL_ERR_OK) { - GGL_LOGE("Failed to apply reset configuration update."); + GGL_LOGE( + "Failed to apply reset configuration update for %.*s.", + (int) pair->key.len, + pair->key.data + ); return; } @@ -2412,17 +2548,29 @@ static void handle_deployment( return; } } else { - GGL_LOGI("DefaultConfiguration not found in the recipe."); + GGL_LOGI( + "DefaultConfiguration not found in the recipe of %.*s.", + (int) pair->key.len, + pair->key.data + ); } } else { - GGL_LOGI("ComponentConfiguration not found in the recipe"); + GGL_LOGI( + "ComponentConfiguration not found in the recipe of %.*s.", + (int) pair->key.len, + pair->key.data + ); } ret = apply_configurations( deployment, component_name->buf, GGL_STR("merge") ); if (ret != GGL_ERR_OK) { - GGL_LOGE("Failed to apply merge configuration update."); + GGL_LOGE( + "Failed to apply merge configuration update for %.*s.", + (int) pair->key.len, + pair->key.data + ); return; } @@ -2480,7 +2628,8 @@ static void handle_deployment( } // Skip redeploying components in a RUNNING state - if (ggl_buffer_eq(component_status, GGL_STR("RUNNING"))) { + if (ggl_buffer_eq(component_status, GGL_STR("RUNNING")) + || ggl_buffer_eq(component_status, GGL_STR("FINISHED"))) { GGL_LOGD( "Component %.*s is already running. Will not redeploy.", (int) pair->key.len, @@ -2586,7 +2735,7 @@ static void handle_deployment( component_name.data ); } else { // relevant install service file exists - + disable_and_unlink_service(&component_name, INSTALL); // add relevant component name into the vector ret = ggl_buf_vec_push( &install_comp_name_buf_vec, component_name @@ -2750,6 +2899,7 @@ static void handle_deployment( component_name.data ); } else { + disable_and_unlink_service(&component_name, RUN_STARTUP); // run link command static uint8_t link_command_buf[PATH_MAX]; GglByteVec link_command_vec @@ -2890,9 +3040,11 @@ static GglError ggl_deployment_listen(GglDeploymentHandlerThreadArgs *args) { GglDeployment bootstrap_deployment = { 0 }; uint8_t jobs_id_resp_mem[64] = { 0 }; GglBuffer jobs_id = GGL_BUF(jobs_id_resp_mem); + int64_t jobs_version = 0; - GglError ret - = retrieve_in_progress_deployment(&bootstrap_deployment, &jobs_id); + GglError ret = retrieve_in_progress_deployment( + &bootstrap_deployment, &jobs_id, &jobs_version + ); if (ret != GGL_ERR_OK) { GGL_LOGD("No deployments previously in progress detected."); } else { @@ -2902,8 +3054,10 @@ static GglError ggl_deployment_listen(GglDeploymentHandlerThreadArgs *args) { (int) bootstrap_deployment.deployment_id.len, bootstrap_deployment.deployment_id.data ); - update_current_jobs_deployment( - bootstrap_deployment.deployment_id, GGL_STR("IN_PROGRESS") + update_bootstrap_jobs_deployment( + bootstrap_deployment.deployment_id, + GGL_STR("IN_PROGRESS"), + jobs_version ); bool bootstrap_deployment_succeeded = false; @@ -2916,14 +3070,18 @@ static GglError ggl_deployment_listen(GglDeploymentHandlerThreadArgs *args) { if (bootstrap_deployment_succeeded) { GGL_LOGI("Completed deployment processing and reporting job as " "SUCCEEDED."); - update_current_jobs_deployment( - bootstrap_deployment.deployment_id, GGL_STR("SUCCEEDED") + update_bootstrap_jobs_deployment( + bootstrap_deployment.deployment_id, + GGL_STR("SUCCEEDED"), + jobs_version ); } else { GGL_LOGW("Completed deployment processing and reporting job as " "FAILED."); - update_current_jobs_deployment( - bootstrap_deployment.deployment_id, GGL_STR("FAILED") + update_bootstrap_jobs_deployment( + bootstrap_deployment.deployment_id, + GGL_STR("FAILED"), + jobs_version ); } // clear any potential saved deployment info for next deployment diff --git a/ggdeploymentd/src/deployment_model.h b/ggdeploymentd/src/deployment_model.h index 6d64ea314..955071533 100644 --- a/ggdeploymentd/src/deployment_model.h +++ b/ggdeploymentd/src/deployment_model.h @@ -15,6 +15,12 @@ typedef enum { GGL_DEPLOYMENT_IN_PROGRESS, } GglDeploymentState; +typedef enum { + INSTALL, + RUN_STARTUP, + BOOTSTRAP +} PhaseSelection; + typedef enum { LOCAL_DEPLOYMENT, THING_GROUP_DEPLOYMENT, diff --git a/ggdeploymentd/src/deployment_queue.c b/ggdeploymentd/src/deployment_queue.c index ff2889bdd..4a133913c 100644 --- a/ggdeploymentd/src/deployment_queue.c +++ b/ggdeploymentd/src/deployment_queue.c @@ -471,7 +471,9 @@ GglError ggl_deployment_dequeue(GglDeployment **deployment) { } void ggl_deployment_release(GglDeployment *deployment) { - assert(deployment == &deployments[queue_index]); + assert(ggl_buffer_eq( + deployment->deployment_id, deployments[queue_index].deployment_id + )); GGL_LOGD("Removing deployment from queue."); diff --git a/ggdeploymentd/src/iot_jobs_listener.c b/ggdeploymentd/src/iot_jobs_listener.c index 0f28333ee..d47bd6ceb 100644 --- a/ggdeploymentd/src/iot_jobs_listener.c +++ b/ggdeploymentd/src/iot_jobs_listener.c @@ -212,13 +212,19 @@ static GglError update_job( } ++(*version); - // save jobs ID to config in case of bootstrap + // save jobs ID and version to config in case of bootstrap ret = save_iot_jobs_id(job_id); if (ret != GGL_ERR_OK) { GGL_LOGE("Failed to save job ID to config."); return ret; } + ret = save_iot_jobs_version(current_job_version); + if (ret != GGL_ERR_OK) { + GGL_LOGE("Failed to save job version to config."); + return ret; + } + return GGL_ERR_OK; } @@ -485,3 +491,14 @@ GglError update_current_jobs_deployment( // overwriting current_job_id while deployment thread updates job state return update_job(current_job_id.buf, status, ¤t_job_version); } + +GglError update_bootstrap_jobs_deployment( + GglBuffer deployment_id, GglBuffer status, int64_t version +) { + if (!ggl_buffer_eq(deployment_id, current_deployment_id.buf)) { + return GGL_ERR_NOENTRY; + } + + current_job_version = version; + return update_job(current_job_id.buf, status, &version); +} diff --git a/ggdeploymentd/src/iot_jobs_listener.h b/ggdeploymentd/src/iot_jobs_listener.h index 9f679122c..4a013769b 100644 --- a/ggdeploymentd/src/iot_jobs_listener.h +++ b/ggdeploymentd/src/iot_jobs_listener.h @@ -7,11 +7,15 @@ #include #include +#include void listen_for_jobs_deployments(void); GglError update_current_jobs_deployment( GglBuffer deployment_id, GglBuffer status ); +GglError update_bootstrap_jobs_deployment( + GglBuffer deployment_id, GglBuffer status, int64_t version +); #endif diff --git a/ggdeploymentd/src/stale_component.c b/ggdeploymentd/src/stale_component.c index 93564f158..7b99544fc 100644 --- a/ggdeploymentd/src/stale_component.c +++ b/ggdeploymentd/src/stale_component.c @@ -1,6 +1,8 @@ #include "stale_component.h" #include "component_store.h" +#include "deployment_model.h" +#include #include #include #include @@ -183,23 +185,26 @@ static GglError delete_component( (int) version_number.len, version_number.data ); + GglError ret; // Remove component from config as we use that as source of truth for active // running components - GglError ret - = ggl_gg_config_delete(GGL_BUF_LIST(GGL_STR("services"), component_name) + if (delete_all_versions) { + ret = ggl_gg_config_delete( + GGL_BUF_LIST(GGL_STR("services"), component_name) ); - if (ret != GGL_ERR_OK) { - GGL_LOGE( - "Failed to delete component information from the configuration." + if (ret != GGL_ERR_OK) { + GGL_LOGE( + "Failed to delete component information from the configuration." + ); + return ret; + } + GGL_LOGD( + "Removed configuration of stale component %.*s", + (int) component_name.len, + component_name.data ); - return ret; } - GGL_LOGD( - "Removed configuration of stale component %.*s", - (int) component_name.len, - component_name.data - ); static uint8_t root_path_mem[PATH_MAX]; memset(root_path_mem, 0, sizeof(root_path_mem)); @@ -260,7 +265,8 @@ static GglError delete_recipe_script_and_service_files(GglBuffer *component_name // Store index so that we can restore the vector to this state. const size_t INDEX_BEFORE_FILE_EXTENTION = root_path.buf.len; - char *extentions[] = { ".script.install.json", ".script.run", ".service" }; + char *extentions[] + = { ".bootstrap.service", ".install.service", ".service" }; for (size_t i = 0; i < (sizeof(extentions) / sizeof(char *)); i++) { GglBuffer buf = { .data = (uint8_t *) extentions[i], @@ -301,7 +307,9 @@ static GglError delete_recipe_script_and_service_files(GglBuffer *component_name } // NOLINTNEXTLINE(readability-function-cognitive-complexity) -static GglError disable_and_unlink_service(GglBuffer *component_name) { +GglError disable_and_unlink_service( + GglBuffer *component_name, PhaseSelection phase +) { static uint8_t command_array[PATH_MAX]; GglByteVec command_vec = GGL_BYTE_VEC(command_array); @@ -309,6 +317,14 @@ static GglError disable_and_unlink_service(GglBuffer *component_name) { = ggl_byte_vec_append(&command_vec, GGL_STR("systemctl stop ")); ggl_byte_vec_chain_append(&ret, &command_vec, GGL_STR("ggl.")); ggl_byte_vec_chain_append(&ret, &command_vec, *component_name); + if (phase == INSTALL) { + ggl_byte_vec_chain_append(&ret, &command_vec, GGL_STR(".install")); + } else if (phase == BOOTSTRAP) { + ggl_byte_vec_chain_append(&ret, &command_vec, GGL_STR(".bootstrap")); + } else { + // Incase of startup/run nothing to append + assert(phase == RUN_STARTUP); + } ggl_byte_vec_chain_append(&ret, &command_vec, GGL_STR(".service")); ggl_byte_vec_chain_push(&ret, &command_vec, '\0'); if (ret != GGL_ERR_OK) { @@ -320,8 +336,7 @@ static GglError disable_and_unlink_service(GglBuffer *component_name) { int system_ret = system((char *) command_vec.buf.data); if (WIFEXITED(system_ret)) { if (WEXITSTATUS(system_ret) != 0) { - GGL_LOGE("systemctl stop failed"); - return GGL_ERR_FAILURE; + GGL_LOGD("systemctl stop failed"); } GGL_LOGI( "systemctl stop exited with child status %d\n", @@ -329,7 +344,6 @@ static GglError disable_and_unlink_service(GglBuffer *component_name) { ); } else { GGL_LOGE("systemctl stop did not exit normally"); - return GGL_ERR_FAILURE; } memset(command_array, 0, sizeof(command_array)); @@ -350,8 +364,7 @@ static GglError disable_and_unlink_service(GglBuffer *component_name) { system_ret = system((char *) command_vec.buf.data); if (WIFEXITED(system_ret)) { if (WEXITSTATUS(system_ret) != 0) { - GGL_LOGE("systemctl disable failed"); - return GGL_ERR_FAILURE; + GGL_LOGD("systemctl disable failed"); } GGL_LOGI( "systemctl disable exited with child status %d\n", @@ -359,7 +372,6 @@ static GglError disable_and_unlink_service(GglBuffer *component_name) { ); } else { GGL_LOGE("systemctl disable did not exit normally"); - return GGL_ERR_FAILURE; } memset(command_array, 0, sizeof(command_array)); @@ -381,8 +393,7 @@ static GglError disable_and_unlink_service(GglBuffer *component_name) { system_ret = system((char *) command_vec.buf.data); if (WIFEXITED(system_ret)) { if (WEXITSTATUS(system_ret) != 0) { - GGL_LOGE("removing symlink failed"); - return GGL_ERR_FAILURE; + GGL_LOGD("removing symlink failed"); } GGL_LOGI( "rm /etc/systemd/system/[service] exited with child status %d\n", @@ -390,7 +401,6 @@ static GglError disable_and_unlink_service(GglBuffer *component_name) { ); } else { GGL_LOGE("rm /etc/systemd/system/[service] did not exit normally"); - return GGL_ERR_FAILURE; } memset(command_array, 0, sizeof(command_array)); @@ -416,8 +426,7 @@ static GglError disable_and_unlink_service(GglBuffer *component_name) { system_ret = system((char *) command_vec.buf.data); if (WIFEXITED(system_ret)) { if (WEXITSTATUS(system_ret) != 0) { - GGL_LOGE("removing symlink failed"); - return GGL_ERR_FAILURE; + GGL_LOGD("removing symlink failed"); } GGL_LOGI( "rm /usr/lib/systemd/system/[service] exited with child status " @@ -426,7 +435,6 @@ static GglError disable_and_unlink_service(GglBuffer *component_name) { ); } else { GGL_LOGE("rm /usr/lib/systemd/system/[service] did not exit normally"); - return GGL_ERR_FAILURE; } memset(command_array, 0, sizeof(command_array)); @@ -547,7 +555,15 @@ GglError cleanup_stale_versions(GglMap latest_components_map) { ); // Also stop any running service for this component. - disable_and_unlink_service(&component_name_buffer_iterator); + disable_and_unlink_service( + &component_name_buffer_iterator, RUN_STARTUP + ); + disable_and_unlink_service( + &component_name_buffer_iterator, INSTALL + ); + disable_and_unlink_service( + &component_name_buffer_iterator, BOOTSTRAP + ); // Also delete the .script.install and .script.run and .service // files. diff --git a/ggdeploymentd/src/stale_component.h b/ggdeploymentd/src/stale_component.h index 2dae316c5..1e217e1cf 100644 --- a/ggdeploymentd/src/stale_component.h +++ b/ggdeploymentd/src/stale_component.h @@ -1,9 +1,14 @@ #ifndef GGL_STALE_COMPONENT_H #define GGL_STALE_COMPONENT_H +#include "deployment_model.h" +#include #include #include +GglError disable_and_unlink_service( + GglBuffer *component_name, PhaseSelection phase +); GglError cleanup_stale_versions(GglMap latest_components_map); #endif diff --git a/gghealthd/CMakeLists.txt b/gghealthd/CMakeLists.txt index c65719a12..9c8d6cf30 100644 --- a/gghealthd/CMakeLists.txt +++ b/gghealthd/CMakeLists.txt @@ -2,5 +2,12 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -ggl_init_module(gghealthd LIBS ggl-lib ggl-file ggl-exec core-bus - core-bus-gg-config PkgConfig::libsystemd) +ggl_init_module( + gghealthd + LIBS ggl-lib + ggl-file + ggl-socket + ggl-exec + core-bus + core-bus-gg-config + PkgConfig::libsystemd) diff --git a/gghealthd/src/bus_server.c b/gghealthd/src/bus_server.c index 6cf7169b1..ca7d448f8 100644 --- a/gghealthd/src/bus_server.c +++ b/gghealthd/src/bus_server.c @@ -143,9 +143,6 @@ static GglError subscribe_to_lifecycle_completion( return ret; } - GGL_LOGD("Accepting subscription."); - ggl_sub_accept(handle, gghealthd_unregister_lifecycle_subscription, NULL); - GglBuffer status; GglError error = gghealthd_get_status(component_name->buf, &status); if (error != GGL_ERR_OK) { diff --git a/gghealthd/src/health.c b/gghealthd/src/health.c index ffa6db126..c00ecedd7 100644 --- a/gghealthd/src/health.c +++ b/gghealthd/src/health.c @@ -6,7 +6,6 @@ #include "bus_client.h" #include "sd_bus.h" #include "subscriptions.h" -#include #include #include #include @@ -16,7 +15,6 @@ #include #include #include -#include #include #include #include @@ -162,10 +160,8 @@ GglError gghealthd_get_health(GglBuffer *status) { return GGL_ERR_OK; } -static pthread_t event_thread; - GglError gghealthd_init(void) { - pthread_create(&event_thread, NULL, health_event_loop_thread, NULL); sd_notify(0, "READY=1"); + init_health_events(); return GGL_ERR_OK; } diff --git a/gghealthd/src/subscriptions.c b/gghealthd/src/subscriptions.c index 025e2d468..afbfefaef 100644 --- a/gghealthd/src/subscriptions.c +++ b/gghealthd/src/subscriptions.c @@ -5,7 +5,6 @@ #include "subscriptions.h" #include "health.h" #include "sd_bus.h" -#include #include #include #include @@ -15,15 +14,12 @@ #include #include #include +#include #include #include -#include #include -#include -#include #include #include -#include #include #include #include @@ -34,25 +30,12 @@ // SoA subscription layout static sd_bus_slot *slots[GGHEALTHD_MAX_SUBSCRIPTIONS]; -static _Atomic(uint32_t) handles[GGHEALTHD_MAX_SUBSCRIPTIONS]; +static uint32_t handles[GGHEALTHD_MAX_SUBSCRIPTIONS]; static size_t component_names_len[GGHEALTHD_MAX_SUBSCRIPTIONS]; static uint8_t component_names[GGHEALTHD_MAX_SUBSCRIPTIONS] [COMPONENT_NAME_MAX_LEN]; -// only to be used by sd_bus thread. -static sd_bus *bus; - -// used to register and unregister subscriptions -static pthread_mutex_t mtx = PTHREAD_MUTEX_INITIALIZER; -static int event_fd; -static pthread_cond_t cond = PTHREAD_COND_INITIALIZER; -static _Atomic(GglError) last_result = GGL_ERR_EXPECTED; -static atomic_int operation_index = -1; - -// event_fd must be created before either ggl_listen or dbus threads start -__attribute__((constructor)) static void create_event_fd(void) { - event_fd = eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK | EFD_SEMAPHORE); -} +static sd_bus *global_bus; static GglBuffer component_name_buf(int index) { assert((index >= 0) && (index < GGHEALTHD_MAX_SUBSCRIPTIONS)); @@ -77,9 +60,17 @@ static int properties_changed_handler( GGL_LOGD("Signal received after unref."); return -1; } + uint32_t handle = handles[index]; + if (handle == 0) { + GGL_LOGD("Signal received after handle closed."); + return -1; + } - uint32_t handle = atomic_load(&handles[index]); GglBuffer component_name = component_name_buf((int) index); + sd_bus *bus = sd_bus_message_get_bus(m); + if (bus == NULL) { + GGL_LOGW("No bus connection?"); + } const char *unit_path = sd_bus_message_get_path(m); if (unit_path == NULL) { @@ -132,7 +123,7 @@ static GglError register_dbus_signal(int index) { sd_bus_message *reply = NULL; const char *unit_path = NULL; ret = get_unit_path( - bus, (const char *) qualified_name.data, &reply, &unit_path + global_bus, (const char *) qualified_name.data, &reply, &unit_path ); GGL_CLEANUP(sd_bus_message_unrefp, reply); if (ret != GGL_ERR_OK) { @@ -141,7 +132,7 @@ static GglError register_dbus_signal(int index) { sd_bus_slot *slot = NULL; int sd_err = sd_bus_match_signal( - bus, + global_bus, &slot, NULL, unit_path, @@ -157,58 +148,45 @@ static GglError register_dbus_signal(int index) { return translate_dbus_call_error(sd_err); } slots[index] = slot; + GGL_LOGD("Accepting subscription."); + ggl_sub_accept( + handles[index], gghealthd_unregister_lifecycle_subscription, NULL + ); return GGL_ERR_OK; } static GglError unregister_dbus_signal(int index) { GGL_LOGD("Event loop thread disabling signal for %d.", index); sd_bus_slot_unref(slots[index]); - atomic_store(&handles[index], 0); slots[index] = NULL; + handles[index] = 0; component_names_len[index] = 0; return GGL_ERR_OK; } -static int event_fd_handler( - sd_event_source *s, int fd, uint32_t revents, void *userdata -) { - (void) userdata; - (void) revents; - (void) s; - uint8_t event_bytes[8] = { 0 }; - GglError ret = ggl_file_read_exact(fd, GGL_BUF(event_bytes)); - pthread_mutex_lock(&mtx); - if (ret == GGL_ERR_OK) { - int index = atomic_load(&operation_index); - assert((index >= 0) && (index < GGHEALTHD_MAX_SUBSCRIPTIONS)); - if (slots[index] == NULL) { - ret = register_dbus_signal(index); - } else { - ret = unregister_dbus_signal(index); - } - } - atomic_store(&last_result, ret); - pthread_cond_signal(&cond); - pthread_mutex_unlock(&mtx); - return 0; +static sd_event *sd_event_ctx; + +static void event_handle_callback(void) { + GGL_LOGD("Event handle callback."); + int ret; + while ((ret = sd_event_run(sd_event_ctx, 0)) > 0) { } + GGL_LOGD("Event loop returned %d.", ret); } -void *health_event_loop_thread(void *ctx) { - (void) ctx; +void init_health_events(void) { while (true) { - GglError ret = open_bus(&bus); + GglError ret = open_bus(&global_bus); if (ret == GGL_ERR_OK) { break; } GGL_LOGE("Failed to open bus."); ggl_sleep(1); } - GGL_CLEANUP(sd_bus_unrefp, bus); do { sd_bus_error error = SD_BUS_ERROR_NULL; int sd_ret = sd_bus_call_method( - bus, + global_bus, DEFAULT_DESTINATION, DEFAULT_PATH, MANAGER_INTERFACE, @@ -240,47 +218,20 @@ void *health_event_loop_thread(void *ctx) { ggl_sleep(1); } - while (true) { - int sd_ret = sd_event_add_io( - e, NULL, event_fd, EPOLLIN, event_fd_handler, NULL - ); - if (sd_ret >= 0) { - break; - } - GGL_LOGE("Failed to add event_fd event (errno=%d)", -sd_ret); - ggl_sleep(1); + int sd_ret = sd_bus_attach_event(global_bus, e, 0); + if (sd_ret < 0) { + GGL_LOGE("Failed to attach bus event %p", (void *) global_bus); } - sd_bus_attach_event(bus, e, 0); - - GGL_LOGD("Started event loop."); - while (true) { - int sd_ret = sd_event_loop(e); - GGL_LOGE("Bailed out of event loop (ret=%d)", sd_ret); - ggl_sleep(1); - } + // TODO: replace with setting up a larger epoll + sd_event_ctx = e; + ggl_socket_server_ext_fd = sd_event_get_fd(e); + ggl_socket_server_ext_handler = event_handle_callback; + GGL_LOGD("sd_event_fd %d", ggl_socket_server_ext_fd); + event_handle_callback(); } -// core-bus thread functions // - -static GglError signal_event_loop_and_wait(int index) { - atomic_store(&operation_index, index); - uint64_t event = 1; - uint8_t event_bytes[sizeof(event)]; - memcpy(event_bytes, &event, sizeof(event)); - GglError ret = ggl_file_write(event_fd, GGL_BUF(event_bytes)); - if (ret != GGL_ERR_OK) { - return ret; - } - GGL_LOGD("Waiting for sd_bus thread to handle request for %d.", index); - pthread_mutex_lock(&mtx); - while ((ret = atomic_exchange(&last_result, GGL_ERR_EXPECTED)) - == GGL_ERR_EXPECTED) { - pthread_cond_wait(&cond, &mtx); - } - pthread_mutex_unlock(&mtx); - return ret; -} +// core-bus functions // GglError gghealthd_register_lifecycle_subscription( GglBuffer component_name, uint32_t handle @@ -293,9 +244,10 @@ GglError gghealthd_register_lifecycle_subscription( ); // find first free slot + int index = 0; for (; index < GGHEALTHD_MAX_SUBSCRIPTIONS; ++index) { - if (atomic_load(&handles[index]) == 0) { + if (handles[index] == 0) { break; } } @@ -305,11 +257,10 @@ GglError gghealthd_register_lifecycle_subscription( } GGL_LOGT("Initializing subscription (index=%d).", index); - memcpy(component_names[index], component_name.data, component_name.len); component_names_len[index] = component_name.len; - atomic_store(&handles[index], handle); - GglError ret = signal_event_loop_and_wait(index); + handles[index] = handle; + GglError ret = register_dbus_signal(index); return ret; } @@ -317,9 +268,9 @@ void gghealthd_unregister_lifecycle_subscription(void *ctx, uint32_t handle) { GGL_LOGT("Unregistering %" PRIu32, handle); (void) ctx; for (int index = 0; index < GGHEALTHD_MAX_SUBSCRIPTIONS; ++index) { - if (atomic_load(&handles[index]) == handle) { + if (handles[index] == handle) { GGL_LOGT("Found handle (index=%d).", index); - signal_event_loop_and_wait(index); + unregister_dbus_signal(index); } } } diff --git a/gghealthd/src/subscriptions.h b/gghealthd/src/subscriptions.h index 3519a2cd5..bd72d8401 100644 --- a/gghealthd/src/subscriptions.h +++ b/gghealthd/src/subscriptions.h @@ -15,6 +15,6 @@ GglError gghealthd_register_lifecycle_subscription( void gghealthd_unregister_lifecycle_subscription(void *ctx, uint32_t handle); -void *health_event_loop_thread(void *ctx); +void init_health_events(void); #endif diff --git a/ggipc-client/include/ggipc/client.h b/ggipc-client/include/ggipc/client.h index 91792adae..0b3c5b798 100644 --- a/ggipc-client/include/ggipc/client.h +++ b/ggipc-client/include/ggipc/client.h @@ -37,6 +37,14 @@ GglError ggipc_get_config_str( int conn, GglBufList key_path, GglBuffer *component_name, GglBuffer *value ); +GglError ggipc_get_config_obj( + int conn, + GglBufList key_path, + GglBuffer *component_name, + GglAlloc *alloc, + GglObject *value +); + GglError ggipc_publish_to_iot_core( int conn, GglBuffer topic_name, diff --git a/ggipc-client/src/client.c b/ggipc-client/src/client.c index a4d66d7a8..395d7af2a 100644 --- a/ggipc-client/src/client.c +++ b/ggipc-client/src/client.c @@ -354,6 +354,73 @@ GglError ggipc_get_config_str( return GGL_ERR_OK; } +GglError ggipc_get_config_obj( + int conn, + GglBufList key_path, + GglBuffer *component_name, + GglAlloc *alloc, + GglObject *value +) { + GglObjVec path_vec = GGL_OBJ_VEC((GglObject[GGL_MAX_OBJECT_DEPTH]) { 0 }); + GglError ret = GGL_ERR_OK; + for (size_t i = 0; i < key_path.len; i++) { + ggl_obj_vec_chain_push(&ret, &path_vec, GGL_OBJ_BUF(key_path.bufs[i])); + } + if (ret != GGL_ERR_OK) { + GGL_LOGE("Key path too long."); + return GGL_ERR_NOMEM; + } + + GglKVVec args = GGL_KV_VEC((GglKV[2]) { 0 }); + (void) ggl_kv_vec_push( + &args, (GglKV) { GGL_STR("keyPath"), GGL_OBJ_LIST(path_vec.list) } + ); + if (component_name != NULL) { + (void) ggl_kv_vec_push( + &args, + (GglKV) { GGL_STR("componentName"), GGL_OBJ_BUF(*component_name) } + ); + } + + static uint8_t resp_mem[sizeof(GglKV) + sizeof("value") + PATH_MAX]; + GglBumpAlloc balloc = ggl_bump_alloc_init(GGL_BUF(resp_mem)); + GglObject resp; + ret = ggipc_call( + conn, + GGL_STR("aws.greengrass#GetConfiguration"), + args.map, + &balloc.alloc, + &resp + ); + if (ret != GGL_ERR_OK) { + return ret; + } + + if (resp.type != GGL_TYPE_MAP) { + GGL_LOGE("Config value is not a map."); + return GGL_ERR_FAILURE; + } + + GglObject *resp_value; + ret = ggl_map_validate( + resp.map, + GGL_MAP_SCHEMA({ GGL_STR("value"), true, GGL_TYPE_NULL, &resp_value }) + ); + if (ret != GGL_ERR_OK) { + GGL_LOGE("Failed validating server response."); + return GGL_ERR_INVALID; + } + + ret = ggl_obj_deep_copy(resp_value, alloc); + if (ret != GGL_ERR_OK) { + GGL_LOGE("Insufficent memory provided for response."); + return ret; + } + + *value = *resp_value; + return GGL_ERR_OK; +} + // TODO: use GglByteVec for payload to allow in-place base64 encoding and remove // alloc GglError ggipc_publish_to_iot_core( diff --git a/ggipcd/src/entry.c b/ggipcd/src/entry.c index 115ed614f..b773a6bad 100644 --- a/ggipcd/src/entry.c +++ b/ggipcd/src/entry.c @@ -29,6 +29,7 @@ GglError run_ggipcd(GglIpcArgs *args) { GGL_BUF_LIST(GGL_STR("system"), GGL_STR("rootPath")), &path_buf ); if (ret != GGL_ERR_OK) { + GGL_LOGE("Failed to read system/rootPath from config."); return ret; } diff --git a/ggipcd/src/services/config/get_configuration.c b/ggipcd/src/services/config/get_configuration.c index aa6ac9e16..cb1bf0e40 100644 --- a/ggipcd/src/services/config/get_configuration.c +++ b/ggipcd/src/services/config/get_configuration.c @@ -30,10 +30,11 @@ GglError ggl_handle_get_configuration( GglObject *key_path_obj; GglObject *component_name_obj; GglBuffer component_name; + GglError ret = ggl_map_validate( args, GGL_MAP_SCHEMA( - { GGL_STR("keyPath"), true, GGL_TYPE_LIST, &key_path_obj }, + { GGL_STR("keyPath"), false, GGL_TYPE_LIST, &key_path_obj }, { GGL_STR("componentName"), false, GGL_TYPE_BUF, @@ -45,6 +46,11 @@ GglError ggl_handle_get_configuration( return GGL_ERR_INVALID; } + GglObject empty_object = GGL_OBJ_LIST({ 0 }); + if (key_path_obj == NULL) { + key_path_obj = &empty_object; + } + ret = ggl_list_type_check(key_path_obj->list, GGL_TYPE_BUF); if (ret != GGL_ERR_OK) { GGL_LOGE("Received invalid parameters."); diff --git a/ggl-recipe/include/ggl/recipe.h b/ggl-recipe/include/ggl/recipe.h index a2b0fee40..49aad4880 100644 --- a/ggl-recipe/include/ggl/recipe.h +++ b/ggl-recipe/include/ggl/recipe.h @@ -37,4 +37,6 @@ GglError select_linux_manifest( GglMap recipe_map, GglMap *out_selected_linux_manifest ); +GglBuffer get_current_architecture(void); + #endif diff --git a/ggl-recipe/src/recipe.c b/ggl-recipe/src/recipe.c index b31d2c338..df5d47bbe 100644 --- a/ggl-recipe/src/recipe.c +++ b/ggl-recipe/src/recipe.c @@ -224,7 +224,7 @@ static GglError lifecycle_selection( return GGL_ERR_OK; } -static GglBuffer get_current_architecture(void) { +GglBuffer get_current_architecture(void) { GglBuffer current_arch = { 0 }; #if defined(__x86_64__) current_arch = GGL_STR("amd64"); @@ -281,12 +281,8 @@ static GglError manifest_selection( // Then check if architecture is also supported if (((architecture_obj == NULL) || (architecture_obj->buf.len == 0) - || (strncmp( - (char *) architecture_obj->buf.data, - (char *) curr_arch.data, - architecture_obj->buf.len - ) - == 0))) { + || ggl_buffer_eq(architecture_obj->buf, GGL_STR("*")) + || ggl_buffer_eq(architecture_obj->buf, curr_arch))) { GglObject *selections; if (ggl_map_get( *manifest_map, diff --git a/ggl-zip/src/zip.c b/ggl-zip/src/zip.c index 04ac6f42d..451d07025 100644 --- a/ggl-zip/src/zip.c +++ b/ggl-zip/src/zip.c @@ -13,7 +13,9 @@ #include #include #include +#include #include +#include static inline void cleanup_zip_fclose(zip_file_t **zip_entry) { if (*zip_entry != NULL) { @@ -50,6 +52,35 @@ static GglError write_entry_to_fd(zip_file_t *entry, int fd) { } } +static bool validate_path(GglBuffer path) { + if (path.len == 0) { + GGL_LOGW("Skipping empty path"); + return false; + } + + if (path.data[0] == '/') { + GGL_LOGW( + "Skipping absolute path in \"%.*s\"", (int) path.len, path.data + ); + return false; + } + + for (size_t i = 0; i + 2 < path.len; ++i) { + if (ggl_buffer_has_prefix( + ggl_buffer_substr(path, i, SIZE_MAX), GGL_STR("../") + )) { + GGL_LOGW( + "Skipping path with \"../\" component(s) in \"%.*s\"", + (int) path.len, + path.data + ); + return false; + } + } + + return true; +} + GglError ggl_zip_unarchive( int source_dest_dir_fd, GglBuffer zip_path, int dest_dir_fd, mode_t mode ) { @@ -71,7 +102,6 @@ GglError ggl_zip_unarchive( } GGL_CLEANUP(cleanup_zip_close, zip); - GglBuffer init_name; zip_uint64_t num_entries = (zip_uint64_t) zip_get_num_entries(zip, 0); for (zip_uint64_t i = 0; i < num_entries; i++) { const char *name = zip_get_name(zip, i, 0); @@ -85,27 +115,18 @@ GglError ggl_zip_unarchive( return GGL_ERR_FAILURE; } - // Avoid creating duplicate zip file name - if (i == 0) { - init_name = ggl_buffer_from_null_term((char *) name); + GglBuffer name_buf = ggl_buffer_from_null_term((char *) name); + if (!validate_path(name_buf)) { continue; } - // Remove the first segment from the path - GglBuffer name_buf = ggl_buffer_from_null_term((char *) name); - GglBuffer trunc_name = name_buf; - if (ggl_buffer_has_prefix(name_buf, init_name)) { - trunc_name - = ggl_buffer_substr(name_buf, init_name.len, name_buf.len); - }; - zip_file_t *entry = zip_fopen_index(zip, i, 0); if (entry == NULL) { int err = zip_error_code_zip(zip_get_error(zip)); GGL_LOGE( "Failed to open file \"%s\" (index %" PRIu64 ") from zip with error %d.", - trunc_name.data, + name_buf.data, i, err ); @@ -115,14 +136,14 @@ GglError ggl_zip_unarchive( GglError ret; int dest_file_fd; - if (ggl_buffer_has_suffix(trunc_name, GGL_STR("/"))) { + if (ggl_buffer_has_suffix(name_buf, GGL_STR("/"))) { ret = ggl_dir_openat( - dest_dir_fd, trunc_name, O_PATH, mode, &dest_file_fd + dest_dir_fd, name_buf, O_PATH, mode, &dest_file_fd ); } else { ret = ggl_file_openat( dest_dir_fd, - trunc_name, + name_buf, O_WRONLY | O_CREAT | O_TRUNC, mode, &dest_file_fd diff --git a/misc/dictionary.txt b/misc/dictionary.txt index e50c3e883..69687ff2f 100644 --- a/misc/dictionary.txt +++ b/misc/dictionary.txt @@ -1,7 +1,9 @@ aarch64 ALLOCN argp +armv awsiotsdk +boto cgexec clientfd clientv @@ -39,6 +41,7 @@ ggconfigd ggconfiglib ggconflib ggcore +ggcredentials GGDEPLOYMENT ggdeploymentd gghealthd @@ -97,7 +100,9 @@ semp serde serverd SERVUNAVAIL +Setenv sigv4 +Skipif ssssssouso statusd stdenv diff --git a/recipe-runner/src/runner.c b/recipe-runner/src/runner.c index 64073f328..cad0d9851 100644 --- a/recipe-runner/src/runner.c +++ b/recipe-runner/src/runner.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -28,7 +29,7 @@ #define MAX_SCRIPT_LENGTH 10000 #define MAX_THING_NAME_LEN 128 -#define MAX_RECIPE_LEN 256000 +#define MAX_RECIPE_LEN 25000 pid_t child_pid = -1; // To store child process ID @@ -46,14 +47,20 @@ static GglError insert_config_value(int conn, int out_fd, GglBuffer json_ptr) { } static uint8_t config_value[10000]; - GglBuffer result = GGL_BUF(config_value); - ret = ggipc_get_config_str(conn, key_path.buf_list, NULL, &result); + static uint8_t copy_config_value[10000]; + GglBumpAlloc buffer = ggl_bump_alloc_init(GGL_BUF(config_value)); + GglObject result = { 0 }; + ret = ggipc_get_config_obj( + conn, key_path.buf_list, NULL, &buffer.alloc, &result + ); if (ret != GGL_ERR_OK) { GGL_LOGE("Failed to get config value for substitution."); return ret; } + GglBuffer final_result = GGL_BUF(copy_config_value); + ggl_json_encode(result, &final_result); - return ggl_file_write(out_fd, result); + return ggl_file_write(out_fd, final_result); } static GglError split_escape_seq( @@ -492,6 +499,23 @@ static GglError write_script_with_replacement( return ret; } + // if startup, send a ready notification before exiting + // otherwise, simple startup scripts will fail with 'protocol' by systemd + if (ggl_buffer_eq(GGL_STR("startup"), phase)) { + ret = ggl_file_write(out_fd, GGL_STR("\n")); + if (ret != GGL_ERR_OK) { + return ret; + } + ret = ggl_file_write(out_fd, GGL_STR("systemd-notify --ready\n")); + if (ret != GGL_ERR_OK) { + return ret; + } + ret = ggl_file_write(out_fd, GGL_STR("systemd-notify --stopping\n")); + if (ret != GGL_ERR_OK) { + return ret; + } + } + return GGL_ERR_OK; } @@ -566,46 +590,6 @@ GglError runner(const RecipeRunnerArgs *args) { GGL_LOGE("setenv failed: %d.", errno); } - // TODO: Check if TES is dependency within the recipe - GglByteVec resp_vec = GGL_BYTE_VEC(resp_mem); - ret = ggl_byte_vec_append(&resp_vec, GGL_STR("http://localhost:")); - if (ret != GGL_ERR_OK) { - GGL_LOGE("Failed to append http://localhost:"); - return ret; - } - GglBuffer rest = ggl_byte_vec_remaining_capacity(resp_vec); - - ret = ggipc_get_config_str( - conn, - GGL_BUF_LIST(GGL_STR("port")), - &GGL_STR("aws.greengrass.TokenExchangeService"), - &rest - ); - if (ret != GGL_ERR_OK) { - GGL_LOGW("Failed to get port from config. errono: %d", ret); - } else { - // Only set the env var if port number is valid - resp_vec.buf.len += rest.len; - ret = ggl_byte_vec_append( - &resp_vec, GGL_STR("/2016-11-01/credentialprovider/\0") - ); - if (ret != GGL_ERR_OK) { - GGL_LOGE("Failed to append /2016-11-01/credentialprovider/"); - return ret; - } - - sys_ret = setenv( - "AWS_CONTAINER_CREDENTIALS_FULL_URI", - (char *) resp_vec.buf.data, - true - ); - if (sys_ret != 0) { - GGL_LOGE( - "setenv AWS_CONTAINER_CREDENTIALS_FULL_URI failed: %d.", errno - ); - } - } - sys_ret = setenv("GGC_VERSION", GGL_VERSION, true); if (sys_ret != 0) { GGL_LOGE("setenv failed: %d.", errno); @@ -665,6 +649,61 @@ GglError runner(const RecipeRunnerArgs *args) { return ret; } + // Check if TES is the dependency within the recipe + GglObject *val; + if (ggl_map_get(recipe.map, GGL_STR("ComponentDependencies"), &val)) { + if (val->type != GGL_TYPE_MAP) { + return GGL_ERR_PARSE; + } + GglObject *inner_val; + GglMap inner_map = val->map; + if (ggl_map_get( + inner_map, + GGL_STR("aws.greengrass.TokenExchangeService"), + &inner_val + )) { + static uint8_t resp_mem2[PATH_MAX]; + GglByteVec resp_vec = GGL_BYTE_VEC(resp_mem2); + ret = ggl_byte_vec_append(&resp_vec, GGL_STR("http://localhost:")); + if (ret != GGL_ERR_OK) { + GGL_LOGE("Failed to append http://localhost:"); + return ret; + } + GglBuffer rest = ggl_byte_vec_remaining_capacity(resp_vec); + + ret = ggipc_get_config_str( + conn, + GGL_BUF_LIST(GGL_STR("port")), + &GGL_STR("aws.greengrass.TokenExchangeService"), + &rest + ); + if (ret != GGL_ERR_OK) { + GGL_LOGW("Failed to get port from config. errono: %d", ret); + } else { + resp_vec.buf.len += rest.len; + ret = ggl_byte_vec_append( + &resp_vec, GGL_STR("/2016-11-01/credentialprovider/\0") + ); + if (ret != GGL_ERR_OK) { + GGL_LOGE("Failed to append /2016-11-01/credentialprovider/" + ); + return ret; + } + + sys_ret = setenv( + "AWS_CONTAINER_CREDENTIALS_FULL_URI", + (char *) resp_vec.buf.data, + true + ); + if (sys_ret != 0) { + GGL_LOGE( + "setenv AWS_CONTAINER_CREDENTIALS_FULL_URI failed: %d.", + errno + ); + } + } + } + } int dir_fd; ret = ggl_dir_open(root_path, O_PATH, false, &dir_fd); if (ret != GGL_ERR_OK) { diff --git a/recipe2unit-test/src/entry.c b/recipe2unit-test/src/entry.c index 41a9f860d..5939c6a41 100644 --- a/recipe2unit-test/src/entry.c +++ b/recipe2unit-test/src/entry.c @@ -20,10 +20,10 @@ GglError run_recipe2unit_test(void) { static Recipe2UnitArgs args = { 0 }; - char component_name[] = "recipe"; - char version[] = "1.0.0"; + char component_name[] = "[Component Name here]"; + char version[] = "[Component Version here]"; char root_dir[] = "."; - char recipe_runner_path[] = "/home/reciperunner"; + char recipe_runner_path[] = "[Path to recipe runner here]"; int root_path_fd; GglError ret @@ -49,9 +49,9 @@ GglError run_recipe2unit_test(void) { GglObject recipe_map; GglObject *component_name_obj; - static uint8_t big_buffer_for_bump[5000]; + static uint8_t big_buffer_for_bump[50000]; GglBumpAlloc bump_alloc = ggl_bump_alloc_init(GGL_BUF(big_buffer_for_bump)); - HasPhase phases; + HasPhase phases = { 0 }; convert_to_unit( &args, &bump_alloc.alloc, &recipe_map, &component_name_obj, &phases diff --git a/recipe2unit/src/file_operation.c b/recipe2unit/src/file_operation.c deleted file mode 100644 index ec06a9f4b..000000000 --- a/recipe2unit/src/file_operation.c +++ /dev/null @@ -1,138 +0,0 @@ -// aws-greengrass-lite - AWS IoT Greengrass runtime for constrained devices -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX - License - Identifier : Apache - 2.0 - -#include "file_operation.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -static GglError deserialize_json( - GglBuffer recipe_buffer, GglAlloc *alloc, GglObject *recipe_obj -) { - ggl_json_decode_destructive(recipe_buffer, alloc, recipe_obj); - if (recipe_obj->type != GGL_TYPE_MAP) { - return GGL_ERR_FAILURE; - } - - return GGL_ERR_OK; -} - -static GglError deserialize_yaml( - GglBuffer recipe_buffer, GglAlloc *alloc, GglObject *recipe_obj -) { - ggl_yaml_decode_destructive(recipe_buffer, alloc, recipe_obj); - if (recipe_obj->type != GGL_TYPE_MAP) { - return GGL_ERR_FAILURE; - } - - return GGL_ERR_OK; -} - -GglError deserialize_file_content( - char *file_path, - GglBuffer recipe_str_buf, - GglAlloc *alloc, - GglObject *recipe_obj -) { - GglError ret; - // Find the last occurrence of a dot in the filepath - const char *dot = strrchr(file_path, '.'); - if (!dot || dot == file_path) { - return GGL_ERR_INVALID; // No extension found or dot is at the start - } - - if (strcmp(dot, ".json") == 0) { - ret = deserialize_json(recipe_str_buf, alloc, recipe_obj); - if (ret != GGL_ERR_OK) { - return ret; - } - - } else if (strcmp(dot, ".yaml") == 0 || strcmp(dot, ".yml") == 0) { - ret = deserialize_yaml(recipe_str_buf, alloc, recipe_obj); - if (ret != GGL_ERR_OK) { - return ret; - } - } else { - return GGL_ERR_INVALID; - } - return GGL_ERR_OK; -} - -GglError open_file(char *file_path, GglBuffer *recipe_obj) { - int fd = open(file_path, O_RDONLY | O_CLOEXEC); - if (fd == -1) { - GGL_LOGE("Failed to open recipe file."); - return GGL_ERR_FAILURE; - } - GGL_CLEANUP(cleanup_close, fd); - - struct stat st; - if (fstat(fd, &st) == -1) { - GGL_LOGE("Failed to get recipe file info."); - return GGL_ERR_FAILURE; - } - - size_t file_size = (size_t) st.st_size; - uint8_t *file_str - = mmap(NULL, file_size, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0); - // NOLINTNEXTLINE(performance-no-int-to-ptr) - if (file_str == MAP_FAILED) { - GGL_LOGE("Failed to load recipe file."); - return GGL_ERR_FAILURE; - } - - *recipe_obj = (GglBuffer) { .data = file_str, .len = file_size }; - - return GGL_ERR_OK; -} - -GglError write_to_file( - char *directory_path, GglBuffer filename, GglBuffer write_data, int mode -) { - int root_dir_fd; - GglError ret = ggl_dir_open( - ggl_buffer_from_null_term(directory_path), O_PATH, true, &root_dir_fd - ); - if (ret != GGL_ERR_OK) { - GGL_LOGE("Failed to open dir"); - return GGL_ERR_FAILURE; - } - - int script_as_file; - - ret = ggl_file_openat( - root_dir_fd, - filename, - O_CREAT | O_WRONLY | O_TRUNC, - (mode_t) mode, - &script_as_file - ); - if (ret != GGL_ERR_OK) { - GGL_LOGE("Failed to open file at the dir"); - return GGL_ERR_FAILURE; - } - ret = ggl_file_write(script_as_file, write_data); - if (ret != GGL_ERR_OK) { - GGL_LOGE("Write to file failed"); - return GGL_ERR_FAILURE; - } - ret = ggl_file_write(script_as_file, GGL_STR("\n")); - if (ret != GGL_ERR_OK) { - GGL_LOGE("Write to file failed"); - return GGL_ERR_FAILURE; - } - return GGL_ERR_OK; -} diff --git a/recipe2unit/src/file_operation.h b/recipe2unit/src/file_operation.h deleted file mode 100644 index 5ebc0b150..000000000 --- a/recipe2unit/src/file_operation.h +++ /dev/null @@ -1,25 +0,0 @@ -// aws-greengrass-lite - AWS IoT Greengrass runtime for constrained devices -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX - License - Identifier : Apache - 2.0 - -#ifndef RECIPE_FILE_OPERATION_H -#define RECIPE_FILE_OPERATION_H - -#include -#include -#include -#include - -GglError deserialize_file_content( - char *file_path, - GglBuffer recipe_str_buf, - GglAlloc *alloc, - GglObject *recipe_obj -); - -GglError open_file(char *file_path, GglBuffer *recipe_obj); -GglError write_to_file( - char *directory_path, GglBuffer filename, GglBuffer write_data, int mode -); - -#endif diff --git a/recipe2unit/src/parser.c b/recipe2unit/src/parser.c index 67fd5ca45..256645cf1 100644 --- a/recipe2unit/src/parser.c +++ b/recipe2unit/src/parser.c @@ -5,6 +5,7 @@ #include "ggl/recipe2unit.h" #include "unit_file_generator.h" #include "validate_args.h" +#include #include #include #include @@ -49,6 +50,9 @@ static GglError create_unit_file( ggl_byte_vec_chain_append( &ret, &file_name_vector, GGL_STR(".bootstrap") ); + } else { + // Incase of startup/run nothing to append + assert(phase == RUN_STARTUP); } ggl_byte_vec_chain_append(&ret, &file_name_vector, GGL_STR(".service\0")); if (ret != GGL_ERR_OK) { @@ -97,7 +101,7 @@ GglError convert_to_unit( recipe_obj ); if (ret != GGL_ERR_OK) { - GGL_LOGI("No recipe found"); + GGL_LOGE("No recipe found"); return ret; } @@ -122,7 +126,7 @@ GglError convert_to_unit( } if (ret == GGL_ERR_NOENTRY) { - GGL_LOGW("No bootstrap phase present"); + GGL_LOGD("No bootstrap phase present"); } else if (ret != GGL_ERR_OK) { return ret; @@ -154,7 +158,7 @@ GglError convert_to_unit( } if (ret == GGL_ERR_NOENTRY) { - GGL_LOGW("No Install phase present"); + GGL_LOGD("No Install phase present"); } else if (ret != GGL_ERR_OK) { return ret; @@ -181,7 +185,7 @@ GglError convert_to_unit( RUN_STARTUP ); if (ret == GGL_ERR_NOENTRY) { - GGL_LOGW("Neither run nor startup phase present"); + GGL_LOGD("Neither run nor startup phase present"); } else if (ret != GGL_ERR_OK) { return ret; } else { @@ -201,6 +205,9 @@ GglError convert_to_unit( && existing_phases->has_run_startup == false) { GGL_LOGE("Recipes without at least 1 valid lifecycle step aren't " "currently supported by GGLite"); + + GGL_LOGW("Note that in GG Lite, keys are case sensitive. Check the " + "recipe reference for the correct casing."); return GGL_ERR_INVALID; } diff --git a/recipe2unit/src/unit_file_generator.c b/recipe2unit/src/unit_file_generator.c index 70d999534..ac6b10d22 100644 --- a/recipe2unit/src/unit_file_generator.c +++ b/recipe2unit/src/unit_file_generator.c @@ -384,9 +384,7 @@ static GglError manifest_builder( return GGL_ERR_INVALID; } } else { - GGL_LOGW("No bootstrap phase found"); - GGL_LOGW("Note that in GG Lite, keys are case sensitive. Check the " - "recipe reference for the correct casing."); + GGL_LOGD("No bootstrap phase found"); return GGL_ERR_NOENTRY; } @@ -407,9 +405,7 @@ static GglError manifest_builder( return GGL_ERR_INVALID; } } else { - GGL_LOGW("No install phase found"); - GGL_LOGW("Note that in GG Lite, keys are case sensitive. Check the " - "recipe reference for the correct casing."); + GGL_LOGD("No install phase found"); return GGL_ERR_NOENTRY; } @@ -451,9 +447,7 @@ static GglError manifest_builder( return GGL_ERR_FAILURE; } } else { - GGL_LOGW("No startup or run phase found"); - GGL_LOGW("Note that in GG Lite, keys are case sensitive. Check the " - "recipe reference for the correct casing."); + GGL_LOGD("No startup or run phase found"); return GGL_ERR_NOENTRY; } } diff --git a/samples/status-monitor-client/bin/status-monitor-client.c b/samples/status-monitor-client/bin/status-monitor-client.c index 7b35e3873..066d767c7 100644 --- a/samples/status-monitor-client/bin/status-monitor-client.c +++ b/samples/status-monitor-client/bin/status-monitor-client.c @@ -25,7 +25,7 @@ int main(void) { GGL_STR("gg_health"), GGL_STR("get_status"), GGL_MAP({ GGL_STR("component_name"), - GGL_OBJ_BUF(GGL_STR("gg_health")) }), + GGL_OBJ_BUF(GGL_STR("gghealthd")) }), &method_error, &alloc.alloc, &result diff --git a/tes-serverd/src/http_server.c b/tes-serverd/src/http_server.c index a0bd5880c..617b74793 100644 --- a/tes-serverd/src/http_server.c +++ b/tes-serverd/src/http_server.c @@ -166,7 +166,6 @@ static void deafult_handler(struct evhttp_request *req, void *arg) { GglBuffer response_cred_buffer = GGL_STR("Only /2016-11-01/credentialprovider/ uri is supported"); - struct evbuffer *buf = evbuffer_new(); if (!buf) { @@ -177,7 +176,7 @@ static void deafult_handler(struct evhttp_request *req, void *arg) { // Add the response data to the evbuffer evbuffer_add(buf, response_cred_buffer.data, response_cred_buffer.len); - evhttp_send_reply(req, HTTP_NOCONTENT, "Forbidden", buf); + evhttp_send_reply(req, HTTP_BADREQUEST, "Bad Request", buf); evbuffer_free(buf); } diff --git a/version b/version index 6e8bf73aa..227cea215 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.1.0 +2.0.0