From 992b3eec3a1e23ae2fb4bf6fd8053f0bdce41197 Mon Sep 17 00:00:00 2001 From: Vincent Le Date: Sat, 29 Apr 2023 16:35:47 -0700 Subject: [PATCH 1/8] [BUILD] Initial setup of Workflows YAML by Vincent - Removed docker- and travis-related files - Created .github/workflows/pull_request_ci.yml - Triggers failing builds and PR commits; installs dependencies --- .travis.yml | 46 --------------------- docker-compose.yml | 15 ------- docker/Dockerfile | 16 -------- docker/README.md | 93 ------------------------------------------ docker/base/Dockerfile | 41 ------------------- 5 files changed, 211 deletions(-) delete mode 100644 .travis.yml delete mode 100644 docker-compose.yml delete mode 100644 docker/Dockerfile delete mode 100644 docker/README.md delete mode 100644 docker/base/Dockerfile diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 728c1674..00000000 --- a/.travis.yml +++ /dev/null @@ -1,46 +0,0 @@ -os: linux -dist: bionic # Ubuntu 18 - -language: c - -env: - - DOCKER_CLI_EXPERIMENTAL=enabled # Needed to enable experimental features in the CLI, like --platform and buildx - -services: - - docker - -# requires that commit is for a PR or is on master branch -if: (type = pull_request) OR (type = push AND branch = master) - -jobs: - include: - - name: format - arch: amd64 - script: - - ./runtime format -r . # Runs clang-format checker to see if code is formatted properly - - - name: test - arch: arm64 - before_script: - - echo '{"experimental":true}' | sudo tee /etc/docker/daemon.json # Need to enable experimental features in the Docker daemon to choose platform. For some reason, only enabling the features in the CLI doesn't work. - - sudo service docker restart - script: - - docker build --platform linux/arm/v7 -t $DOCKER_REPO:test -f docker/Dockerfile ./ # Build new image with updated code for arm32v7 - - docker run -t $DOCKER_REPO:test ./runtime test int # Runs the integration tests in a new container - - - name: push - arch: amd64 - if: type = push AND branch = master # Build and push to latest only if this is a master commit - before_script: - # Need to install updated version of Docker 19.03 to get experimental features for buildx, from https://www.docker.com/blog/multi-arch-build-what-about-travis/ - - sudo rm -rf /var/lib/apt/lists/* - - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - - - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) edge" - - sudo apt-get update - - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce - script: - # DOCKER_USERNAME and DOCKER_ACCESS_TOKEN are defined as a repository secret environment variable on travis-ci.org - - echo "$DOCKER_ACCESS_TOKEN" | docker login -u "$DOCKER_USERNAME" --password-stdin - - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes # Need to install QEMU to emulate ARM architectures - - docker buildx create --use - - docker buildx build --platform linux/amd64,linux/arm/v7 -t $DOCKER_REPO:$TAG --build-arg BUILDKIT_INLINE_CACHE=1 --push -f docker/Dockerfile ./ diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index c70d6d97..00000000 --- a/docker-compose.yml +++ /dev/null @@ -1,15 +0,0 @@ -version: '3.8' - -services: - runtime: - build: - context: . - dockerfile: docker/Dockerfile - image: pierobotics/runtime:latest - command: bash -c "./runtime build && tail -f /dev/null" - volumes: - # Changes in local runtime/ and container /root/runtime are synced - - .:/root/runtime - ports: - - "5005:5005" # Used to connect SoundDevice to localhost - - "5006:5006" # Used to connect keyboard_inputs.py with keyboard_interface.c diff --git a/docker/Dockerfile b/docker/Dockerfile deleted file mode 100644 index 1f77330c..00000000 --- a/docker/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM pierobotics/runtime:base - -WORKDIR /root/runtime - -# Adds all files in the repo that are not in .dockerignore -ADD ./ ./ - -# Add the runtime command to our PATH -RUN echo 'export PATH=$PATH:~/runtime' >> ~/.bashrc - -# Build Runtime processes -RUN ./runtime build -RUN cd tests && make cli && cd .. # runtime build does not build the cli's; we want to build this in case something fails here - -# By default, run Runtime when the Docker container starts -CMD ./runtime run diff --git a/docker/README.md b/docker/README.md deleted file mode 100644 index 4a7b40cf..00000000 --- a/docker/README.md +++ /dev/null @@ -1,93 +0,0 @@ -# Docker - -Docker is a system that allows you to consistently reconstruct the same development environment every time. This is especially important to use since not everyone has a Raspberry Pi to test on but Docker allows you to test your code in the same environment. To learn more about Docker and why we use it, read our [wiki page](https://github.com/pioneers/c-runtime/wiki/Docker). - -Our latest Docker images will be stored on Docker Hub here https://hub.docker.com/repository/docker/pierobotics/runtime. - -## Installation - -To install Docker, go here https://docs.docker.com/get-docker/. Note that to install Docker for Windows 10 Home, you first need to install WSL 2 here https://docs.microsoft.com/en-us/windows/wsl/install-win10. - -# Usage with Docker Compose (Recommended) - -We recommend using `docker-compose` for all your Docker development needs, especially if you're new to Docker. - -## Installing - -If you are on Windows/Mac, you will be using Docker Desktop which will already have `docker-compose` installed. If you are on Linux, you can install it with - - pip3 install docker-compose - -## Running - -If you are just starting out and want to get into development quickly, go to the `runtime` folder and do - - docker-compose up -d - -which will start the Docker container. Now you can do - - docker-compose exec runtime bash - -to open up a bash terminal in the container. You can repeat this command to get however many bash terminals you want. From the container, you can then call `runtime run` and the other `runtime` commands mentioned in the root README. Additionally, the `runtime/` git folder and `/root/runtime` will be linked so changes in the container will also change on your machine. - -## Stopping - -To stop the container, do - - docker-compose down -t 1 - -# Usage with Docker CLI (Intermediate) - -We recommend using the Docker CLI instead only if you are very interested in Docker and want to know more. - -## Pulling - -To get the image with the code from the `master` branch, do - - docker pull pierobotics/runtime:latest - -## Building - -To build your own image instead of using the one on Docker Hub, so that it uses your up to date code instead of code on `master`, do - - docker build -t pierobotics/runtime:latest -f docker/Dockerfile ./ - -## Running - -To run the Docker container, do - - docker run -it --rm pierobotics/runtime:latest - -This CMD can be `./runtime run` to start Runtime, or `bash` to open a shell, or any other command you want to run inside the container. You can also run other things in the container by in another terminal doing `docker exec -it $(docker ps -q) bash` and running whatever you want inside the shell. The `runtime` Git repo will be located at `/root/runtime`. - -### Devices - -If you want to have the container access any Arduino devices, add the flag `--device /dev/ttyACM0` or whichever path to you device you want to `docker run`. You can do this multiple times for multiple devices. - -### Mounting - -If you want to mount the `runtime` folder to the Docker container, add the flag `-v $PWD:/root/runtime` to `docker run`. This will allow changes you make inside the Docker container to change the files within your `runtime` directory on your computer, and vice versa as well. This is especially useful if you are using the Docker container to do development. - -## Stopping - -To stop the container, either exit from the `./runtime run` shell with Ctrl+C or do `docker stop -t 5 $(docker ps -q)` in a separate shell. - -# Advanced Topics - -## Multi-Architecture Images - -The Raspberry Pi 4 uses the arm32/v7 architecture while most consumer computers nowadays use x64/amd64 architecture. To have Docker images that work for both development on our computers and production on the Pis, we make it so that the same tag points to both architectures. When you do `docker pull` or `docker build`, Docker will automatically get the image that has your architecture. This is ideal in most cases. - -If you're on a x64 machine and would like to run the arm32 images instead though (other direction isn't possible), you first need to have QEMU, a binary emulator, installed. If you are on Windows/Mac and so have Docker Desktop, QEMU is already installed for you. If you are on Linux, you can install QEMU on Linux with `docker run --rm --privileged multiarch/qemu-user-static --reset -p yes`. Now you can build the `arm32` image with - - DOCKER_CLI_EXPERIMENTAL=enabled docker build --platform linux/arm/v7 -t pierobotics/runtime:latest -f docker/Dockerfile ./ - -## Base Image (only for PMs) - -We use a base image to make sure that the building of the actual image is fast. Building it for multi-architectures requires some experimental Docker features to enable `buildx`, which can be done by setting an environment variable. If for some reason the base image needs to be updated and pushed, do - - cd docker/base - DOCKER_CLI_EXPERIMENTAL=enabled docker buildx create --use # Only needed the very first time on a computer - DOCKER_CLI_EXPERIMENTAL=enabled docker buildx build --platform linux/amd64,linux/arm/v7 --build-arg BUILDKIT_INLINE_CACHE=1 -t pierobotics/runtime:base --push ./ - -If you haven't run this already on your machine and so don't have local caches, it will take several hours. Importantly, this can only be run on x64 machines that have QEMU, a binary emulator, installed. If you're on Windows/Mac, Docker Desktop already has QEMU preinstalled. If you are using Linux, you will need to install QEMU before building by doing `docker run --rm --privileged multiarch/qemu-user-static --reset -p yes`. diff --git a/docker/base/Dockerfile b/docker/base/Dockerfile deleted file mode 100644 index 6b219b77..00000000 --- a/docker/base/Dockerfile +++ /dev/null @@ -1,41 +0,0 @@ -FROM python:3.7-buster - -# Install Python.h and other apt dependencies -RUN apt-get -q update && \ - apt-get -y install python3-dev wget tar pkg-config && \ - apt-get clean - -WORKDIR /tmp -SHELL ["/bin/bash", "-c"] - -# Install protobuf -ENV protobuf_folder protobuf -RUN wget https://github.com/protocolbuffers/protobuf/releases/download/v3.12.3/protobuf-cpp-3.12.3.tar.gz -O ${protobuf_folder}.tar.gz && \ - mkdir ${protobuf_folder} && tar xzf ${protobuf_folder}.tar.gz -C ${protobuf_folder} --strip-components 1 && \ - rm ${protobuf_folder}.tar.gz && \ - pushd ${protobuf_folder} && \ - ./configure && make && make install && ldconfig && \ - popd && rm -r ${protobuf_folder} - -# Install protobuf-c -ENV protobuf_c_folder protobuf-c -RUN wget https://github.com/protobuf-c/protobuf-c/releases/download/v1.3.3/protobuf-c-1.3.3.tar.gz -O ${protobuf_c_folder}.tar.gz && \ - mkdir ${protobuf_c_folder} && tar xzf ${protobuf_c_folder}.tar.gz -C ${protobuf_c_folder} --strip-components 1 && \ - rm ${protobuf_c_folder}.tar.gz && \ - pushd ${protobuf_c_folder} && \ - ./configure && make && make install && ldconfig && \ - popd && rm -r ${protobuf_c_folder} - -# Install Cython -RUN pip3 install Cython - -# Install arduino-cli -RUN curl -fsSL https://raw.githubusercontent.com/arduino/arduino-cli/master/install.sh | BINDIR=/usr/bin sh && \ - arduino-cli core update-index - -# Install debug packages -RUN apt-get -y install procps htop tmux nano net-tools clang-format - -WORKDIR /root - -CMD ["bash"] From 92b479047750ca7ddeb1b8fa937ca2b52e196ef9 Mon Sep 17 00:00:00 2001 From: Vincent Le Date: Sat, 29 Apr 2023 16:38:02 -0700 Subject: [PATCH 2/8] [BUILD] Changed virtual device sockets from /var to /tmp --- .dockerignore | 10 ------ .github/workflows/pull_request_ci.yml | 49 +++++++++++++++++++++++++++ dev_handler/dev_handler.c | 4 +-- tests/client/dev_handler_client.c | 2 +- 4 files changed, 52 insertions(+), 13 deletions(-) delete mode 100644 .dockerignore create mode 100644 .github/workflows/pull_request_ci.yml diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index e1cf44c9..00000000 --- a/.dockerignore +++ /dev/null @@ -1,10 +0,0 @@ -.git/ -.dockerignore -.travis.yml -*/*.log -docker/ -Dockerfile -.vscode/ -images/ -*/build/ -docs/ diff --git a/.github/workflows/pull_request_ci.yml b/.github/workflows/pull_request_ci.yml new file mode 100644 index 00000000..7b63175a --- /dev/null +++ b/.github/workflows/pull_request_ci.yml @@ -0,0 +1,49 @@ +# This configures the Continuous Integration builds on every PR +name: CI + +on: + pull_request: + branches: [master] + types: [opened, synchronize] + +jobs: + build-and-test: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + submodules: 'recursive' + + - name: Install deb packages + run: | + sudo apt-get update + sudo apt-get install -y \ + python3-dev \ + python3-pip \ + clang-format + + - name: Install Cython + run: | + python3 -m pip install Cython + + - name: Install protobuf + run: | + wget https://github.com/protocolbuffers/protobuf/releases/download/v3.15.1/protobuf-cpp-3.15.1.tar.gz + tar -xzf protobuf-cpp-3.15.1.tar.gz + cd protobuf-3.15.1 && ./configure && make && sudo make install && sudo ldconfig + + - name: Install protobuf-c + run: | + wget https://github.com/protobuf-c/protobuf-c/releases/download/v1.3.3/protobuf-c-1.3.3.tar.gz + tar -xzf protobuf-c-1.3.3.tar.gz + cd protobuf-c-1.3.3 && ./configure && make && sudo make install && sudo ldconfig + + - name: Build + run: | + ./runtime build + + - name: Test + run: | + ./runtime test int diff --git a/dev_handler/dev_handler.c b/dev_handler/dev_handler.c index 22427b5f..e4c0a2f9 100644 --- a/dev_handler/dev_handler.c +++ b/dev_handler/dev_handler.c @@ -16,7 +16,7 @@ * a file with path "/dev/ttyACM0". A second device connected will appear as * "/dev/ttyACM1". * Virtual devices (not Arduinos) on the other hand are UNIX sockets that appear as - * "/tmp/ttyACM0" + * "/var/ttyACM0" * In the code, the number is referred to as "port_num" * Depending on whether a device is an Arduino ("lowcar") or a virtual device, * dev handler has to open a connection with it differently. @@ -24,7 +24,7 @@ * These file paths may also be referred to as "port_prefix" in the code. */ #define LOWCAR_FILE_PATH "/dev/ttyACM" -#define VIRTUAL_FILE_PATH "/tmp/ttyACM" +#define VIRTUAL_FILE_PATH "/var/ttyACM" #define LOWCAR_USB_FILE_PATH "/dev/ttyUSB" // **************************** PRIVATE STRUCT ****************************** // diff --git a/tests/client/dev_handler_client.c b/tests/client/dev_handler_client.c index c55c6996..99dfa6f0 100644 --- a/tests/client/dev_handler_client.c +++ b/tests/client/dev_handler_client.c @@ -4,7 +4,7 @@ #define TIMEOUT 2000 // The name of a socket's file to be succeeded by an integer -#define SOCKET_PREFIX "/tmp/ttyACM" +#define SOCKET_PREFIX "/var/ttyACM" // The maximum number of devices that can be connected #define MAX_DEVICES 32 From e6c5addbf5383c26b8bf732b5da6405dd5d16885 Mon Sep 17 00:00:00 2001 From: Ben Liao Date: Sat, 20 May 2023 12:43:40 -0700 Subject: [PATCH 3/8] [BUILD] Used updated actions/checkout; no more building Protobuf - Most tests still failing - Found apt package that when installed allows protobuf-c to compile - Crucially allows us to remove 10+ minutes off build time --- .github/workflows/pull_request_ci.yml | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/.github/workflows/pull_request_ci.yml b/.github/workflows/pull_request_ci.yml index 7b63175a..8beeb8b8 100644 --- a/.github/workflows/pull_request_ci.yml +++ b/.github/workflows/pull_request_ci.yml @@ -12,7 +12,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: submodules: 'recursive' @@ -22,23 +22,19 @@ jobs: sudo apt-get install -y \ python3-dev \ python3-pip \ - clang-format + clang-format \ + protobuf-compiler \ + libprotoc-dev - name: Install Cython run: | python3 -m pip install Cython - - name: Install protobuf - run: | - wget https://github.com/protocolbuffers/protobuf/releases/download/v3.15.1/protobuf-cpp-3.15.1.tar.gz - tar -xzf protobuf-cpp-3.15.1.tar.gz - cd protobuf-3.15.1 && ./configure && make && sudo make install && sudo ldconfig - - name: Install protobuf-c run: | - wget https://github.com/protobuf-c/protobuf-c/releases/download/v1.3.3/protobuf-c-1.3.3.tar.gz - tar -xzf protobuf-c-1.3.3.tar.gz - cd protobuf-c-1.3.3 && ./configure && make && sudo make install && sudo ldconfig + wget https://github.com/protobuf-c/protobuf-c/releases/download/v1.4.1/protobuf-c-1.4.1.tar.gz + tar -xzf protobuf-c-1.4.1.tar.gz + cd protobuf-c-1.4.1 && ./configure && make && sudo make install && sudo ldconfig - name: Build run: | @@ -46,4 +42,4 @@ jobs: - name: Test run: | - ./runtime test int + ./runtime test integration/tc_150_1 From 70e3c1aaee5f6b32318840ef89f262fa75c61afd Mon Sep 17 00:00:00 2001 From: Ben Liao Date: Tue, 23 May 2023 00:32:38 -0700 Subject: [PATCH 4/8] [BUILD] Trying to get UNIX sockets to work on Github Worfklows - Changed virtual devices from /var/ttyACM* to $HOME/ttyACM* --- .github/workflows/pull_request_ci.yml | 2 +- dev_handler/dev_handler.c | 42 ++++++++++++++++-- tests/client/dev_handler_client.c | 43 +++++++++++++++---- .../client/virtual_devices/SimpleTestDevice.c | 41 ++++++++++++++++++ .../virtual_devices/virtual_device_util.c | 28 ++++++------ 5 files changed, 128 insertions(+), 28 deletions(-) diff --git a/.github/workflows/pull_request_ci.yml b/.github/workflows/pull_request_ci.yml index 8beeb8b8..c353cf22 100644 --- a/.github/workflows/pull_request_ci.yml +++ b/.github/workflows/pull_request_ci.yml @@ -42,4 +42,4 @@ jobs: - name: Test run: | - ./runtime test integration/tc_150_1 + ./runtime test integration/tc_71_5 diff --git a/dev_handler/dev_handler.c b/dev_handler/dev_handler.c index e4c0a2f9..27489660 100644 --- a/dev_handler/dev_handler.c +++ b/dev_handler/dev_handler.c @@ -10,6 +10,8 @@ #include "../shm_wrapper/shm_wrapper.h" #include "message.h" +#include // ************************************************************************************* + /** * Each device will have a unique port number. * For example, if there is only one Arduino connected, it will (probably) appear as @@ -24,7 +26,7 @@ * These file paths may also be referred to as "port_prefix" in the code. */ #define LOWCAR_FILE_PATH "/dev/ttyACM" -#define VIRTUAL_FILE_PATH "/var/ttyACM" +#define VIRTUAL_FILE_PATH "ttyACM" // will be created in the home directory #define LOWCAR_USB_FILE_PATH "/dev/ttyUSB" // **************************** PRIVATE STRUCT ****************************** // @@ -92,7 +94,10 @@ uint32_t used_virtual_ports = 0; uint32_t used_lowcar_usb_ports = 0; pthread_mutex_t used_ports_lock; // poll_connected_devices() and relay_clean_up() shouldn't access used_ports at the same time -#define MAX_PORT_NAME_SIZE 16 +// String to hold the home directory path (for looking for virtual device sockets) +const char *home_dir; + +#define MAX_PORT_NAME_SIZE 64 // ***************************** MAIN FUNCTIONS ***************************** // @@ -185,11 +190,14 @@ static int get_new_devices_helper(bool is_virtual, bool is_usb, uint32_t* found_ construct_port_name(device_path, is_virtual, is_usb, i); // If that port currently connected (file exists), it's a new device if (access(device_path, F_OK) != -1) { + log_printf(DEBUG, "device found at path: %s\n", device_path); // ********************************************************* // Turn bit on *found_devices |= (1 << i); // Mark that we've taken care of this device *used_ports |= (1 << i); + log_printf(DEBUG, "virtual device bitmap is %d\n", *used_ports); // ********************************************************* num_devices_found++; + log_printf(DEBUG, "num_Devices_found = %d\n", num_devices_found); // ********************************************************* } } pthread_mutex_unlock(&used_ports_lock); @@ -241,6 +249,7 @@ void communicate(bool is_virtual, bool is_usb, uint8_t port_num) { if (relay->is_virtual) { // Bind to socket relay->file_descriptor = connect_socket(port_name); + log_printf(DEBUG, "dev_handler: relay->file_descriptor = %d\n", relay->file_descriptor); // ******************************************************* if (relay->file_descriptor == -1) { log_printf(ERROR, "communicate: Couldn't connect to socket %s\n", port_name); relay_clean_up(relay); @@ -312,7 +321,7 @@ void* relayer(void* relay_cast) { // If the device disconnects or times out, clean up log_printf(DEBUG, "Monitoring %s (0x%016llX)", get_device_name(relay->dev_id.type), relay->dev_id.uid); - char port_name[14]; + char port_name[MAX_PORT_NAME_SIZE]; construct_port_name(port_name, relay->is_virtual, relay->is_usb, relay->port_num); while (1) { // If Arduino port file doesn't exist, it disconnected @@ -486,6 +495,8 @@ void* receiver(void* relay_cast) { // Try to read a message if (receive_message(relay, msg) != 0) { // Message was broken... try to read the next message + printf("something happpened"); + fflush(stdout); continue; } if (msg->message_id == DEVICE_DATA || msg->message_id == LOG || msg->message_id == DEVICE_PING) { @@ -568,10 +579,29 @@ int receive_message(relay_t* relay, message_t* msg) { /* Haven't verified device is lowcar yet * read() is set to timeout while waiting for an ACK (see serialport_open())*/ pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL); + log_printf(DEBUG, "before read call: relay->file_descriptor = %d", relay->file_descriptor); + log_printf(DEBUG, "before read call"); // ******************************************************************************************* + fflush(stdout); + + usleep(100000); // stop so virtul device can print out its fd's before spammy + + // make a call that will give EBADF if fd is bad that is not close() XD + struct stat fd_stat; + int ret = fstat(relay->file_descriptor, &fd_stat); + + + log_printf(DEBUG, "after fstat call"); + usleep(100000); // ***************************** make sure that this log has time to print + num_bytes_read = read(relay->file_descriptor, &last_byte_read, 1); + usleep(100000); // throttle?????? // ********************************************************* + log_printf(DEBUG, "after read call"); // ******************************************************************************************* + fflush(stdout); pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, NULL); if (num_bytes_read == -1) { log_printf(ERROR, "receive_message: Error on read() for ACK--%s", strerror(errno)); + fflush(stdout); + usleep(100000); // throttle? please please please throttle // ********************************************************* return 3; } else if (num_bytes_read == 0) { // read() returned due to timeout log_printf(WARN, "Timed out when waiting for ACK from %s!", port_name); @@ -722,6 +752,7 @@ int verify_device(relay_t* relay) { int connect_socket(const char* socket_name) { // Make a local socket for sending/receiving raw byte streams int fd = socket(AF_UNIX, SOCK_STREAM, 0); + log_printf(DEBUG, "dev_handler fd = %d\n", fd); // ********************************************************* if (fd == -1) { log_printf(ERROR, "connect_socket: Couldn't create socket--%s", strerror(errno)); return -1; @@ -811,6 +842,7 @@ int serialport_open(const char* port_name) { * -1 on error and sets errno */ int serialport_close(int fd) { + log_printf(DEBUG, "closing! ???"); // ************************************************************ return close(fd); } @@ -829,7 +861,8 @@ void cleanup_handler(void* args) { void construct_port_name(char* port_name, bool is_virtual, bool is_usb, int port_num) { if (is_virtual) { - sprintf(port_name, "%s%d", VIRTUAL_FILE_PATH, port_num); + // append home directory in front of the socket name + sprintf(port_name, "%s/%s%d", home_dir, VIRTUAL_FILE_PATH, port_num); } else if (is_usb) { sprintf(port_name, "%s%d", LOWCAR_USB_FILE_PATH, port_num); } else { @@ -853,6 +886,7 @@ int main(int argc, char* argv[]) { // If SIGINT (Ctrl+C) is received, call stop() to clean up signal(SIGINT, stop); init(); + home_dir = getenv("HOME"); // set the home directory log_printf(INFO, "DEV_HANDLER initialized."); poll_connected_devices(); return 0; diff --git a/tests/client/dev_handler_client.c b/tests/client/dev_handler_client.c index 99dfa6f0..0dea22b3 100644 --- a/tests/client/dev_handler_client.c +++ b/tests/client/dev_handler_client.c @@ -3,8 +3,9 @@ // Timeout time in ms for a socket read() #define TIMEOUT 2000 -// The name of a socket's file to be succeeded by an integer -#define SOCKET_PREFIX "/var/ttyACM" +// The name of a socket's file to be preceded by the home directory and succeeded by an integer +#define SOCKET_PREFIX "ttyACM" +const char* home_dir; // holds home directory path // The maximum number of devices that can be connected #define MAX_DEVICES 32 @@ -66,9 +67,14 @@ static int connect_socket() { return -1; } + // printf("successfully (?) created socket: server_fd = %d\n", server_fd); // ******************************************************************************************* + // fflush(stdout); + // Build socket name - char socket_name[14]; - sprintf(socket_name, "%s%d", SOCKET_PREFIX, socket_num); + char socket_name[64]; + sprintf(socket_name, "%s/%s%d", home_dir, SOCKET_PREFIX, socket_num); + printf("socket_name = %s\n", socket_name); + fflush(stdout); // Build socket address struct sockaddr_un dev_socket_addr; @@ -81,18 +87,25 @@ static int connect_socket() { remove(socket_name); return -1; } + + printf("listening...\n"); + fflush(stdout); // Listen for dev_handler to connect to the virtual device socket listen(server_fd, 1); // Accept the connection int connection_fd; + printf("listening before accept ...\n"); if ((connection_fd = accept(server_fd, NULL, NULL)) < 0) { printf("connect_socket: Couldn't accept socket connection\n"); return -1; } close(server_fd); + // printf("successful connection from dev_handler to device\n"); + // fflush(stdout); + // Indicate in global variable that socket is now used used_sockets[socket_num] = malloc(sizeof(device_socket_t)); if (used_sockets[socket_num] == NULL) { @@ -107,6 +120,9 @@ static int connect_socket() { tv.tv_usec = 0; setsockopt(connection_fd, SOL_SOCKET, SO_RCVTIMEO, (const char*) &tv, sizeof(tv)); + printf("socket_num = %d\n", socket_num); // ******************************************************************************************* + fflush(stdout); + return socket_num; } @@ -123,8 +139,10 @@ void start_dev_handler() { } // execute the device handler process if (execlp("./../bin/dev_handler", "dev_handler", (char*) 0) < 0) { - printf("execlp: %s\n", strerror(errno)); + printf("start_dev_handler execlp: %s\n", strerror(errno)); } + } else { // in parent + home_dir = getenv("HOME"); } } @@ -166,9 +184,11 @@ int connect_virtual_device(char* dev_name, uint64_t uid) { } // Become the virtual device by calling "./ " - char exe_name[32], fd_str[2], uid_str[20]; + char exe_name[32], fd_str[4], uid_str[20]; sprintf(exe_name, "./%s", used_sockets[socket_num]->dev_name); sprintf(fd_str, "%d", used_sockets[socket_num]->fd); + printf("fd_str = %s\n", fd_str); // ********************************************************************** + fflush(stdout); sprintf(uid_str, "0x%016llX", uid); if (execlp(exe_name, used_sockets[socket_num]->dev_name, fd_str, uid_str, (char*) NULL) < 0) { printf("connect_device: execlp %s failed -- %s\n", exe_name, strerror(errno)); @@ -177,6 +197,9 @@ int connect_virtual_device(char* dev_name, uint64_t uid) { } else { // Parent process // Take note of child pid so we can kill it in disconnect_device() used_sockets[socket_num]->pid = pid; + + // close duplicate connection file descriptor + close(used_sockets[socket_num]->fd); } return socket_num; } @@ -189,11 +212,13 @@ int disconnect_virtual_device(int socket_num) { // Kill virtual device process kill(used_sockets[socket_num]->pid, SIGTERM); waitpid(used_sockets[socket_num]->pid, NULL, 0); // Block until killed - // CLose file descriptor + // CLose file descriptor + printf("DEV HANDLER CLIENT CLOSE ?????? heh? \n"); + fflush(stdout); close(used_sockets[socket_num]->fd); // Remove socket (dev handler should recognize disconnect after removal) - char socket_name[14]; - sprintf(socket_name, "%s%d", SOCKET_PREFIX, socket_num); + char socket_name[32]; + sprintf(socket_name, "%s/%s%d", home_dir, SOCKET_PREFIX, socket_num); remove(socket_name); // Mark socket as unoccupied free(used_sockets[socket_num]->dev_name); diff --git a/tests/client/virtual_devices/SimpleTestDevice.c b/tests/client/virtual_devices/SimpleTestDevice.c index 8c74cb02..bba37afb 100644 --- a/tests/client/virtual_devices/SimpleTestDevice.c +++ b/tests/client/virtual_devices/SimpleTestDevice.c @@ -37,6 +37,30 @@ void device_actions(param_val_t params[]) { params[FLIP_FLOP].p_b = 1 - params[FLIP_FLOP].p_b; } +/* check first 1024 (usual size of FD_SESIZE) file handles */ +int test_fds() +{ + int i; + int fd_dup; + char errst[64]; + for (i = 0; i < FD_SETSIZE; i++) { + *errst = 0; + fflush(stdout); + fd_dup = dup(i); + if (fd_dup == -1) { + // EBADF oldfd isn’t an open file descriptor, or newfd is out of the allowed range for file descriptors. + // EBUSY (Linux only) This may be returned by dup2() during a race condition with open(2) and dup(). + // EINTR The dup2() call was interrupted by a signal; see signal(7). + // EMFILE The process already has the maximum number of file descriptors open and tried to open a new one. + } else { + close(fd_dup); + printf("File descriptor %d exists!\n", i); + fflush(stdout); + } + } + return 0; +} + /** * A device that behaves like a lowcar device, connected to dev handler via a socket * Arguments: @@ -48,6 +72,8 @@ int main(int argc, char* argv[]) { printf("Incorrect number of arguments: %d out of %d\n", argc, 3); exit(1); } + + printf("SimpleTestDevice arguments: %s %s %s\n", argv[0], argv[1], argv[2]); int fd = atoi(argv[1]); uint64_t uid = strtoull(argv[2], NULL, 0); @@ -57,6 +83,21 @@ int main(int argc, char* argv[]) { param_val_t params[dev->num_params]; init_params(params); + + printf("SimpleTestDevice communication socket fd = %d\n", fd); + fflush(stdout); + + // // print out all open file descriptors +// // long max_fd = sysconf(_SC_OPEN_MAX); // get maximum number of open file descriptors +// for (int i = 0; i < 32; i++) { +// int ret; +// if ((ret = fcntl(i, F_GETFD)) == -1) { +// printf("file descriptor %d exists!\n", ret); +// fflush(stdout); +// } +// } + + test_fds(); lowcar_protocol(fd, dev_type, dev_type, uid, params, &device_actions, 1000); return 0; diff --git a/tests/client/virtual_devices/virtual_device_util.c b/tests/client/virtual_devices/virtual_device_util.c index 58d4209b..229198eb 100644 --- a/tests/client/virtual_devices/virtual_device_util.c +++ b/tests/client/virtual_devices/virtual_device_util.c @@ -204,20 +204,20 @@ void lowcar_protocol(int fd, uint8_t type, uint8_t year, uint64_t uid, continue; } - // Make sure we're receiving DEVICE_PING messages still - if ((now - last_received_ping_time) >= TIMEOUT) { - printf("lowcar_protocol: DEV_HANDLER timed out!\n"); - exit(1); - } - // Check if we should send another DEVICE_PING - // TODO: Physical lowcar devices don't send DEVICE_PING messages. - // We should remove this. See issue #164. - if ((now - last_sent_ping_time) >= PING_FREQ) { - outgoing_msg = make_ping(); - send_message(fd, outgoing_msg); - destroy_message(outgoing_msg); - last_sent_ping_time = now; - } + // // Make sure we're receiving DEVICE_PING messages still + // if ((now - last_received_ping_time) >= TIMEOUT) { + // printf("lowcar_protocol: DEV_HANDLER timed out!\n"); + // exit(1); + // } + // // Check if we should send another DEVICE_PING + // // TODO: Physical lowcar devices don't send DEVICE_PING messages. + // // We should remove this. See issue #164. + // if ((now - last_sent_ping_time) >= PING_FREQ) { + // outgoing_msg = make_ping(); + // send_message(fd, outgoing_msg); + // destroy_message(outgoing_msg); + // last_sent_ping_time = now; + // } // Change read-only params periodically if (action_interval != -1 && (now - last_device_action) >= action_interval) { (*device_actions)(params); From a03c7ea30d99ef4ed689a2a33a59d05f9ce04129 Mon Sep 17 00:00:00 2001 From: Ben Liao Date: Fri, 16 Jun 2023 22:54:41 -0700 Subject: [PATCH 5/8] [BUILD] Working tc_71_5 - Error was the buffer on line 172 of dev_handler_client.c was ... - too short and could not hold the null terminator of the argument --- dev_handler/dev_handler.c | 27 ------------ tests/client/dev_handler_client.c | 21 +--------- .../client/virtual_devices/SimpleTestDevice.c | 41 ------------------- .../virtual_devices/virtual_device_util.c | 28 ++++++------- 4 files changed, 15 insertions(+), 102 deletions(-) diff --git a/dev_handler/dev_handler.c b/dev_handler/dev_handler.c index 27489660..3fbf6784 100644 --- a/dev_handler/dev_handler.c +++ b/dev_handler/dev_handler.c @@ -190,14 +190,11 @@ static int get_new_devices_helper(bool is_virtual, bool is_usb, uint32_t* found_ construct_port_name(device_path, is_virtual, is_usb, i); // If that port currently connected (file exists), it's a new device if (access(device_path, F_OK) != -1) { - log_printf(DEBUG, "device found at path: %s\n", device_path); // ********************************************************* // Turn bit on *found_devices |= (1 << i); // Mark that we've taken care of this device *used_ports |= (1 << i); - log_printf(DEBUG, "virtual device bitmap is %d\n", *used_ports); // ********************************************************* num_devices_found++; - log_printf(DEBUG, "num_Devices_found = %d\n", num_devices_found); // ********************************************************* } } pthread_mutex_unlock(&used_ports_lock); @@ -249,7 +246,6 @@ void communicate(bool is_virtual, bool is_usb, uint8_t port_num) { if (relay->is_virtual) { // Bind to socket relay->file_descriptor = connect_socket(port_name); - log_printf(DEBUG, "dev_handler: relay->file_descriptor = %d\n", relay->file_descriptor); // ******************************************************* if (relay->file_descriptor == -1) { log_printf(ERROR, "communicate: Couldn't connect to socket %s\n", port_name); relay_clean_up(relay); @@ -495,8 +491,6 @@ void* receiver(void* relay_cast) { // Try to read a message if (receive_message(relay, msg) != 0) { // Message was broken... try to read the next message - printf("something happpened"); - fflush(stdout); continue; } if (msg->message_id == DEVICE_DATA || msg->message_id == LOG || msg->message_id == DEVICE_PING) { @@ -579,29 +573,10 @@ int receive_message(relay_t* relay, message_t* msg) { /* Haven't verified device is lowcar yet * read() is set to timeout while waiting for an ACK (see serialport_open())*/ pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL); - log_printf(DEBUG, "before read call: relay->file_descriptor = %d", relay->file_descriptor); - log_printf(DEBUG, "before read call"); // ******************************************************************************************* - fflush(stdout); - - usleep(100000); // stop so virtul device can print out its fd's before spammy - - // make a call that will give EBADF if fd is bad that is not close() XD - struct stat fd_stat; - int ret = fstat(relay->file_descriptor, &fd_stat); - - - log_printf(DEBUG, "after fstat call"); - usleep(100000); // ***************************** make sure that this log has time to print - num_bytes_read = read(relay->file_descriptor, &last_byte_read, 1); - usleep(100000); // throttle?????? // ********************************************************* - log_printf(DEBUG, "after read call"); // ******************************************************************************************* - fflush(stdout); pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, NULL); if (num_bytes_read == -1) { log_printf(ERROR, "receive_message: Error on read() for ACK--%s", strerror(errno)); - fflush(stdout); - usleep(100000); // throttle? please please please throttle // ********************************************************* return 3; } else if (num_bytes_read == 0) { // read() returned due to timeout log_printf(WARN, "Timed out when waiting for ACK from %s!", port_name); @@ -752,7 +727,6 @@ int verify_device(relay_t* relay) { int connect_socket(const char* socket_name) { // Make a local socket for sending/receiving raw byte streams int fd = socket(AF_UNIX, SOCK_STREAM, 0); - log_printf(DEBUG, "dev_handler fd = %d\n", fd); // ********************************************************* if (fd == -1) { log_printf(ERROR, "connect_socket: Couldn't create socket--%s", strerror(errno)); return -1; @@ -842,7 +816,6 @@ int serialport_open(const char* port_name) { * -1 on error and sets errno */ int serialport_close(int fd) { - log_printf(DEBUG, "closing! ???"); // ************************************************************ return close(fd); } diff --git a/tests/client/dev_handler_client.c b/tests/client/dev_handler_client.c index 0dea22b3..c421440c 100644 --- a/tests/client/dev_handler_client.c +++ b/tests/client/dev_handler_client.c @@ -67,14 +67,9 @@ static int connect_socket() { return -1; } - // printf("successfully (?) created socket: server_fd = %d\n", server_fd); // ******************************************************************************************* - // fflush(stdout); - // Build socket name char socket_name[64]; sprintf(socket_name, "%s/%s%d", home_dir, SOCKET_PREFIX, socket_num); - printf("socket_name = %s\n", socket_name); - fflush(stdout); // Build socket address struct sockaddr_un dev_socket_addr; @@ -87,25 +82,18 @@ static int connect_socket() { remove(socket_name); return -1; } - - printf("listening...\n"); - fflush(stdout); // Listen for dev_handler to connect to the virtual device socket listen(server_fd, 1); // Accept the connection int connection_fd; - printf("listening before accept ...\n"); if ((connection_fd = accept(server_fd, NULL, NULL)) < 0) { printf("connect_socket: Couldn't accept socket connection\n"); return -1; } close(server_fd); - // printf("successful connection from dev_handler to device\n"); - // fflush(stdout); - // Indicate in global variable that socket is now used used_sockets[socket_num] = malloc(sizeof(device_socket_t)); if (used_sockets[socket_num] == NULL) { @@ -120,9 +108,6 @@ static int connect_socket() { tv.tv_usec = 0; setsockopt(connection_fd, SOL_SOCKET, SO_RCVTIMEO, (const char*) &tv, sizeof(tv)); - printf("socket_num = %d\n", socket_num); // ******************************************************************************************* - fflush(stdout); - return socket_num; } @@ -187,8 +172,6 @@ int connect_virtual_device(char* dev_name, uint64_t uid) { char exe_name[32], fd_str[4], uid_str[20]; sprintf(exe_name, "./%s", used_sockets[socket_num]->dev_name); sprintf(fd_str, "%d", used_sockets[socket_num]->fd); - printf("fd_str = %s\n", fd_str); // ********************************************************************** - fflush(stdout); sprintf(uid_str, "0x%016llX", uid); if (execlp(exe_name, used_sockets[socket_num]->dev_name, fd_str, uid_str, (char*) NULL) < 0) { printf("connect_device: execlp %s failed -- %s\n", exe_name, strerror(errno)); @@ -212,9 +195,7 @@ int disconnect_virtual_device(int socket_num) { // Kill virtual device process kill(used_sockets[socket_num]->pid, SIGTERM); waitpid(used_sockets[socket_num]->pid, NULL, 0); // Block until killed - // CLose file descriptor - printf("DEV HANDLER CLIENT CLOSE ?????? heh? \n"); - fflush(stdout); + // CLose file descriptor close(used_sockets[socket_num]->fd); // Remove socket (dev handler should recognize disconnect after removal) char socket_name[32]; diff --git a/tests/client/virtual_devices/SimpleTestDevice.c b/tests/client/virtual_devices/SimpleTestDevice.c index bba37afb..8c74cb02 100644 --- a/tests/client/virtual_devices/SimpleTestDevice.c +++ b/tests/client/virtual_devices/SimpleTestDevice.c @@ -37,30 +37,6 @@ void device_actions(param_val_t params[]) { params[FLIP_FLOP].p_b = 1 - params[FLIP_FLOP].p_b; } -/* check first 1024 (usual size of FD_SESIZE) file handles */ -int test_fds() -{ - int i; - int fd_dup; - char errst[64]; - for (i = 0; i < FD_SETSIZE; i++) { - *errst = 0; - fflush(stdout); - fd_dup = dup(i); - if (fd_dup == -1) { - // EBADF oldfd isn’t an open file descriptor, or newfd is out of the allowed range for file descriptors. - // EBUSY (Linux only) This may be returned by dup2() during a race condition with open(2) and dup(). - // EINTR The dup2() call was interrupted by a signal; see signal(7). - // EMFILE The process already has the maximum number of file descriptors open and tried to open a new one. - } else { - close(fd_dup); - printf("File descriptor %d exists!\n", i); - fflush(stdout); - } - } - return 0; -} - /** * A device that behaves like a lowcar device, connected to dev handler via a socket * Arguments: @@ -72,8 +48,6 @@ int main(int argc, char* argv[]) { printf("Incorrect number of arguments: %d out of %d\n", argc, 3); exit(1); } - - printf("SimpleTestDevice arguments: %s %s %s\n", argv[0], argv[1], argv[2]); int fd = atoi(argv[1]); uint64_t uid = strtoull(argv[2], NULL, 0); @@ -83,21 +57,6 @@ int main(int argc, char* argv[]) { param_val_t params[dev->num_params]; init_params(params); - - printf("SimpleTestDevice communication socket fd = %d\n", fd); - fflush(stdout); - - // // print out all open file descriptors -// // long max_fd = sysconf(_SC_OPEN_MAX); // get maximum number of open file descriptors -// for (int i = 0; i < 32; i++) { -// int ret; -// if ((ret = fcntl(i, F_GETFD)) == -1) { -// printf("file descriptor %d exists!\n", ret); -// fflush(stdout); -// } -// } - - test_fds(); lowcar_protocol(fd, dev_type, dev_type, uid, params, &device_actions, 1000); return 0; diff --git a/tests/client/virtual_devices/virtual_device_util.c b/tests/client/virtual_devices/virtual_device_util.c index 229198eb..58d4209b 100644 --- a/tests/client/virtual_devices/virtual_device_util.c +++ b/tests/client/virtual_devices/virtual_device_util.c @@ -204,20 +204,20 @@ void lowcar_protocol(int fd, uint8_t type, uint8_t year, uint64_t uid, continue; } - // // Make sure we're receiving DEVICE_PING messages still - // if ((now - last_received_ping_time) >= TIMEOUT) { - // printf("lowcar_protocol: DEV_HANDLER timed out!\n"); - // exit(1); - // } - // // Check if we should send another DEVICE_PING - // // TODO: Physical lowcar devices don't send DEVICE_PING messages. - // // We should remove this. See issue #164. - // if ((now - last_sent_ping_time) >= PING_FREQ) { - // outgoing_msg = make_ping(); - // send_message(fd, outgoing_msg); - // destroy_message(outgoing_msg); - // last_sent_ping_time = now; - // } + // Make sure we're receiving DEVICE_PING messages still + if ((now - last_received_ping_time) >= TIMEOUT) { + printf("lowcar_protocol: DEV_HANDLER timed out!\n"); + exit(1); + } + // Check if we should send another DEVICE_PING + // TODO: Physical lowcar devices don't send DEVICE_PING messages. + // We should remove this. See issue #164. + if ((now - last_sent_ping_time) >= PING_FREQ) { + outgoing_msg = make_ping(); + send_message(fd, outgoing_msg); + destroy_message(outgoing_msg); + last_sent_ping_time = now; + } // Change read-only params periodically if (action_interval != -1 && (now - last_device_action) >= action_interval) { (*device_actions)(params); From c2465066d54ceea4b4978b5e2fbb6c412f183a5e Mon Sep 17 00:00:00 2001 From: Matthew Harrigan <31233369+FIREdog5@users.noreply.github.com> Date: Fri, 16 Jun 2023 23:08:37 -0700 Subject: [PATCH 6/8] [BUILD] Implemented caching for protobuf-c for faster runs --- .github/workflows/pull_request_ci.yml | 30 ++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pull_request_ci.yml b/.github/workflows/pull_request_ci.yml index c353cf22..f8653eb9 100644 --- a/.github/workflows/pull_request_ci.yml +++ b/.github/workflows/pull_request_ci.yml @@ -5,6 +5,8 @@ on: pull_request: branches: [master] types: [opened, synchronize] + push: + branches: [master] jobs: build-and-test: @@ -29,13 +31,39 @@ jobs: - name: Install Cython run: | python3 -m pip install Cython + + - name: Give tar root permissions + run: | + sudo chown root:root /usr/bin/tar && sudo chmod u+s /usr/bin/tar + + - name: Cache protobuf-c + id: cache-protobuf-c + uses: actions/cache@v3 + with: + path: | + /usr/local/lib/libprotobuf-c.* + /usr/local/lib/pkgconfig + /usr/local/bin/protoc-* + /usr/local/include/protobuf-c + /usr/local/include/google + key: libprotobuf + + - name: Take tar root permissions away + run: | + sudo chown runner:runner /usr/bin/tar && sudo chmod u+s /usr/bin/tar - - name: Install protobuf-c + - if: ${{ steps.cache-protobuf-c.outputs.cache-hit != 'true' }} + name: Install protobuf-c run: | wget https://github.com/protobuf-c/protobuf-c/releases/download/v1.4.1/protobuf-c-1.4.1.tar.gz tar -xzf protobuf-c-1.4.1.tar.gz cd protobuf-c-1.4.1 && ./configure && make && sudo make install && sudo ldconfig + - if: ${{ steps.cache-protobuf-c.outputs.cache-hit == 'true' }} + name: Run ldconfig + run: | + sudo ldconfig + - name: Build run: | ./runtime build From a04364c2eb2cd1e7cb3ae9d28526f0bb40734327 Mon Sep 17 00:00:00 2001 From: Ben Liao Date: Sat, 17 Jun 2023 01:42:47 -0700 Subject: [PATCH 7/8] [BUILD] Working one test with caching, comments, and indenting --- .github/workflows/pull_request_ci.yml | 83 ++++++++++++++++++--------- 1 file changed, 55 insertions(+), 28 deletions(-) diff --git a/.github/workflows/pull_request_ci.yml b/.github/workflows/pull_request_ci.yml index f8653eb9..96f6cb64 100644 --- a/.github/workflows/pull_request_ci.yml +++ b/.github/workflows/pull_request_ci.yml @@ -1,6 +1,12 @@ -# This configures the Continuous Integration builds on every PR +# This configures the Continuous Integration builds on every PR, +# as well as every commit on master. + +# This is the name of this workflow name: CI +# Under what conditions do we run this workflow? Either +# a) It is a commit on a pull request on a branch to be merged into master +# b) It is a push to master (which only happens when we close a pull request) on: pull_request: branches: [master] @@ -8,66 +14,87 @@ on: push: branches: [master] +# Here is the list of jobs that this workflow will run. There is only one job. jobs: + # The name of the job: "build-and-test" build-and-test: + # This specifies the OS that we are running the workflow on: the latest version of Ubuntu runs-on: ubuntu-latest + # Here is the list of specific commands this job will run. They are run in order steps: + # First, we check out the repository's code using the Github Actions' "Checkout" tool. - name: Checkout code uses: actions/checkout@v3 with: submodules: 'recursive' - - - name: Install deb packages + + # Next we install a bunch of dev packages that are needed to build Runtime + - name: Install dev packages run: | sudo apt-get update sudo apt-get install -y \ - python3-dev \ - python3-pip \ - clang-format \ - protobuf-compiler \ - libprotoc-dev - + python3-dev \ + python3-pip \ + clang-format \ + protobuf-compiler \ + libprotoc-dev + + # Next we install Cython using pip - name: Install Cython run: | python3 -m pip install Cython - + + # The next three steps do the following: + # Give the program "tar" root permissions + # Retrieve cached files associated with the protobuf-c library, if they exist + # Take away root permission from the "tar" program + # We need to do this because the Cache Action uses "tar" to restore cached files + # into the runner. However, the cached files need root permission to get restored. + # Thus, we need to give tar root permissions to do the cache restoration, then + # take it away once we're done (since if the caching step fails and we do the + # insallation of protobuf-c with root permissions, we will get permission errors) - name: Give tar root permissions run: | - sudo chown root:root /usr/bin/tar && sudo chmod u+s /usr/bin/tar - + sudo chown root:root /usr/bin/tar && sudo chmod u+s /usr/bin/tar - name: Cache protobuf-c id: cache-protobuf-c - uses: actions/cache@v3 + uses: actions/cache@v3 # Uses the Github Actions' "Cache" tool with: + # These are the files we cache path: | - /usr/local/lib/libprotobuf-c.* - /usr/local/lib/pkgconfig - /usr/local/bin/protoc-* - /usr/local/include/protobuf-c - /usr/local/include/google + /usr/local/lib/libprotobuf-c.* + /usr/local/lib/pkgconfig + /usr/local/bin/protoc-* + /usr/local/include/protobuf-c + /usr/local/include/google + # This is the name (key) of the cache that we look for each time key: libprotobuf - - name: Take tar root permissions away run: | sudo chown runner:runner /usr/bin/tar && sudo chmod u+s /usr/bin/tar - - if: ${{ steps.cache-protobuf-c.outputs.cache-hit != 'true' }} - name: Install protobuf-c - run: | - wget https://github.com/protobuf-c/protobuf-c/releases/download/v1.4.1/protobuf-c-1.4.1.tar.gz - tar -xzf protobuf-c-1.4.1.tar.gz - cd protobuf-c-1.4.1 && ./configure && make && sudo make install && sudo ldconfig - + # This if statement determine whether the cache lookup was a hit (true) or miss (false) + # If the cache lookup hit, we just run "sudo ldconfig" to tell the linker where to look for + # protobuf-c files. If the cache lookup missed, we have to install protobuf-c from scratch + # See https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#using-the-output-of-the-cache-action - if: ${{ steps.cache-protobuf-c.outputs.cache-hit == 'true' }} name: Run ldconfig run: | - sudo ldconfig + sudo ldconfig + - if: ${{ steps.cache-protobuf-c.outputs.cache-hit == 'false' }} + name: Install protobuf-c + run: | + wget https://github.com/protobuf-c/protobuf-c/releases/download/v1.4.1/protobuf-c-1.4.1.tar.gz + tar -xzf protobuf-c-1.4.1.tar.gz + cd protobuf-c-1.4.1 && ./configure && make && sudo make install && sudo ldconfig + # Now that we are done installing Runtime's dependencies, we build Runtime - name: Build run: | ./runtime build - + + # And finally, we test Runtime - name: Test run: | ./runtime test integration/tc_71_5 From 33eb6237001ab3377750f44acd49678f988fc0b1 Mon Sep 17 00:00:00 2001 From: Ben Liao Date: Sat, 17 Jun 2023 03:49:03 -0700 Subject: [PATCH 8/8] [BUILD] All tests passing! - Also updated README.md - Added a scheduled run of the workflow so the cache never expires - Change cache key to accurately reflect contents - Removed ./runtime build from test.sh - Removed unnecessary installations of some packages --- .github/workflows/pull_request_ci.yml | 37 +++++++++++---------- README.md | 47 ++++++++++++--------------- dev_handler/dev_handler.c | 2 -- scripts/test.sh | 3 -- 4 files changed, 40 insertions(+), 49 deletions(-) diff --git a/.github/workflows/pull_request_ci.yml b/.github/workflows/pull_request_ci.yml index 96f6cb64..38720311 100644 --- a/.github/workflows/pull_request_ci.yml +++ b/.github/workflows/pull_request_ci.yml @@ -7,12 +7,15 @@ name: CI # Under what conditions do we run this workflow? Either # a) It is a commit on a pull request on a branch to be merged into master # b) It is a push to master (which only happens when we close a pull request) +# c) It's 10:45 UTC on the 1st, 8th, 15th, 22nd, or 29th of a month (to prevent the cache from expiring) on: pull_request: branches: [master] types: [opened, synchronize] push: branches: [master] + schedule: + - cron: '45 10 1,8,15,22,29 * *' # Here is the list of jobs that this workflow will run. There is only one job. jobs: @@ -28,27 +31,27 @@ jobs: uses: actions/checkout@v3 with: submodules: 'recursive' - - # Next we install a bunch of dev packages that are needed to build Runtime + + # Next, we install dev packages needed for Runtime - name: Install dev packages run: | - sudo apt-get update sudo apt-get install -y \ python3-dev \ - python3-pip \ clang-format \ protobuf-compiler \ libprotoc-dev - - # Next we install Cython using pip + + # Next, we install Cython - name: Install Cython run: | - python3 -m pip install Cython + python3 -m pip install Cython - # The next three steps do the following: - # Give the program "tar" root permissions - # Retrieve cached files associated with the protobuf-c library, if they exist - # Take away root permission from the "tar" program + # The next steps steps do the following: + # + # 1) Give the program "tar" root permissions + # 2) Retrieve cached files associated with the protobuf-c library, if they exist + # 3) Take away root permission from the "tar" program + # # We need to do this because the Cache Action uses "tar" to restore cached files # into the runner. However, the cached files need root permission to get restored. # Thus, we need to give tar root permissions to do the cache restoration, then @@ -69,12 +72,12 @@ jobs: /usr/local/include/protobuf-c /usr/local/include/google # This is the name (key) of the cache that we look for each time - key: libprotobuf - - name: Take tar root permissions away + key: libprotobuf-c + - name: Take away tar root permissions run: | - sudo chown runner:runner /usr/bin/tar && sudo chmod u+s /usr/bin/tar + sudo chown runner:runner /usr/bin/tar && sudo chmod u+x /usr/bin/tar - # This if statement determine whether the cache lookup was a hit (true) or miss (false) + # This if statement determine whether the cache lookup was a hit (true) or miss (not true--idk, false doesn't work) # If the cache lookup hit, we just run "sudo ldconfig" to tell the linker where to look for # protobuf-c files. If the cache lookup missed, we have to install protobuf-c from scratch # See https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#using-the-output-of-the-cache-action @@ -82,7 +85,7 @@ jobs: name: Run ldconfig run: | sudo ldconfig - - if: ${{ steps.cache-protobuf-c.outputs.cache-hit == 'false' }} + - if: ${{ steps.cache-protobuf-c.outputs.cache-hit != 'true' }} name: Install protobuf-c run: | wget https://github.com/protobuf-c/protobuf-c/releases/download/v1.4.1/protobuf-c-1.4.1.tar.gz @@ -97,4 +100,4 @@ jobs: # And finally, we test Runtime - name: Test run: | - ./runtime test integration/tc_71_5 + ./runtime test \ No newline at end of file diff --git a/README.md b/README.md index 8b30291f..33ac6cb3 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# PiE Runtime [![Build Status](https://travis-ci.org/pioneers/runtime.svg?branch=master)](https://travis-ci.org/pioneers/runtime) +# PiE Runtime [![Build](https://github.com/pioneers/runtime/actions/workflows/pull_request_ci.yml/badge.svg)](https://github.com/pioneers/runtime/actions/workflows/pull_request_ci.yml) Welcome to the PiE Runtime repo! This README will only cover how to install, build, and run Runtime. If you want to learn more info about how Runtime works, or are a new member, please check out our wiki https://github.com/pioneers/runtime/wiki. @@ -23,18 +23,16 @@ Runtime can be divided into a few neatly containerized parts: In addition to these parts, there are a number of configuration files for Runtime to manage the various tools that we use. They are listed here, with a brief explanation about what they do: -* `.dockerignore`: this file lists out all of the directories, files, and other information that we do not want to include when building Runtime's Docker image. * `.gitignore`: this file lists out all of the directories and files that we don't want in our Git repository. This includes things like executables, build files, and `.DS_Store`. * `.gitattributes`: this file is purely for aesthetic purposes. It tells Github which files to exclude when calculating the repository language makeup you see in the repo (below the "Contributors" section in the sidebar on the web page that you're probably looking at right now). -* `.travis.yml`: this file tells Travis (the continuous integration tool that we use) what to run when checking if Runtime is working properly before a new feature is merged into the master branch. It is essentially responsible for running the integration tests and for updating Runtime's Docker image. -* `docker-compose.yml`: this file is used to describe to the `docker-compose` command what to run. +* `.github/workflows/pull_request_ci.yml`: this file outlines the steps taken by the Github Action that we use to test our code before we release Runtime. See our most recent CI builds [here](https://github.com/pioneers/runtime/actions/workflows/pull_request_ci.yml)! * `runtime`: this file is a convenience shell script that makes it easy to call the various other shell scripts in our directory, which you will read more about later. This README will not go into each of these parts into exhaustive detail; explanations for each part can be found in the corresponding folder's README in the repo as well as [our wiki.](https://github.com/pioneers/runtime/wiki) ## Dependencies -Runtime is designed to be used on the Raspberry Pi, and with a distribution of Linux called Raspbian, a slight variant of the extremely popular Debian. We tested Runtime to work well on Linux systems. If you do not have a Linux system, please instead use Docker (which might even be easier than using Linux). The major benefit of using Docker is you can skip this entire dependency section. To learn more about it, check out the README in the `docker/` folder. +Runtime is designed to be used on the Raspberry Pi, and with a distribution of Linux called Raspbian, a slight variant of the extremely popular Debian. We tested Runtime to work well on Linux systems. Only some parts of Runtime work on macOS and may require different commands to compile; no parts of Runtime can operate on Windows. Please use a Linux machine or (preferably) a Raspberry Pi to run Runtime! ### Commonly Used Tools @@ -60,21 +58,19 @@ Runtime has the following third-party library dependencies: #### Installing Google Protobufs and `protobuf-c` -1. From `https://github.com/protocolbuffers/protobuf/releases`, download the latest `protobuf-cpp-.tar.gz` and extract it (for help, click [here](#extract)) -2. From `https://github.com/protobuf-c/protobuf-c/releases` download the latest `protobuf-c-.tar.gz` and extract it (for help, click [here](#extract)) -3. You may need to install some tools (`libtool`, `pkg-config`, `g++`). To install a tool you don't have, run `sudo apt-get -y install `, replace `` with what you want to install. -4. `cd` into the folder for `protobuf-cpp-.tar.gz` and run: - 1. `./configure` - 2. `make` (this takes a long time) - 3. `sudo make install` - 4. `sudo ldconfig` -5. `cd` into the folder for `protobuf-c-.tar.gz` and run: +1. From `https://github.com/protobuf-c/protobuf-c/releases` download version 1.4.1, i.e. [`protobuf-c-1.4.1.tar.gz`](https://github.com/protobuf-c/protobuf-c/releases/download/v1.4.1/protobuf-c-1.4.1.tar.gz) and extract it (for help, click [here](#extract)) +2. You may need to install some tools (`libtool`, `pkg-config`, `g++`). To install a tool you don't have, run `sudo apt-get -y install `, replace `` with what you want to install. + +3. To install Google Protobuf, run: + 1. `sudo apt-get -y install protobuf-compiler` + 2. `sudo apt-get -y install libprotoc-dev` +4. To install `protobuf-c`, first `cd` into the folder for `protobuf-c-.tar.gz` and run: 1. `./configure` 2. `make` 3. `sudo make install` 4. `sudo ldconfig` -6. (optional) To view `protobuf-c` documentation: +5. (optional) To view `protobuf-c` documentation: 1. Install `doxygen`: `sudo apt-get -y install doxygen` 2. Repeat steps 5.i and 5.ii from above in the `protobuf-c` directory, then do `make html` 3. Then navigate to that directory in Explorer/Finder and open the `html/index.html` file by double clicking on it @@ -90,26 +86,22 @@ Then, extract the files with You might need to install `wget` and `tar` with `sudo apt-get -y install wget tar`. -### Installing ncurses for Shared Memory UI -The Shared Memory UI allows the user to have a real-time view of data flowing in and out of shared memory. -This is analogous to the "shm_cli" among dev_handler_cli, executor_cli, and net_handler_cli. -Note that the UI is used in development only as a debugging tool and should not be used in production. -The command below will install ncurses on Raspbian / Debian machines: +### Installing `ncurses` for Shared Memory UI + +The Shared Memory UI allows the user to have a real-time view of data flowing in and out of shared memory. This is analogous to the "`shm_cli`" among `dev_handler_cli`, `executor_cli`, and `net_handler_cli`. Note that the UI is used in development only as a debugging tool and should not be used in production. The command below will install `ncurses` on Linux machines: sudo apt-get -y install libncurses5-dev libncursesw5-dev ### Installing Clang Format -Clang Format is used to format code to comply with our [code style](https://github.com/pioneers/runtime/wiki/Project-Conventions). Code that is not formatted with Clang cannot be merged into master. After installing, use the runtime script to format code. (See `runtime format`) -``` -sudo apt-get -y install clang-format -``` +Clang Format is used to format code to comply with our [code style](https://github.com/pioneers/runtime/wiki/Project-Conventions). Code that is not formatted with Clang cannot be merged into `master`. After installing, use the runtime script to format code. (See `runtime format`) + + sudo apt-get -y install clang-format + ## Runtime Script and Usage We use the bash script `runtime` in this root directory as the entrypoint to our code. You can call it with `./runtime ` in this folder. However, you can also have it be callable from any directory by adding it to your `PATH` variable. This can be done automatically adding the following line to your `~/.bashrc` file: -``` -export PATH="$PATH:" -``` + export PATH="$PATH:" Then either close and reopen the terminal, or do `source ~/.bashrc`. @@ -122,6 +114,7 @@ If you now type `runtime` it will list all the possible subcommands you can run. * `flash`: will flash an Arduino as a Lowcar device * `clean`: will remove all artifacts generated when building or running Runtime * `update`: waits for new `runtime.zip` to appear in `/tmp` and then will unzip it and reboot the robot +* `status`: will display the currently running processes under the `ubuntu` user (to list active Runtime processes) ## Authors diff --git a/dev_handler/dev_handler.c b/dev_handler/dev_handler.c index 3fbf6784..70935ac1 100644 --- a/dev_handler/dev_handler.c +++ b/dev_handler/dev_handler.c @@ -10,8 +10,6 @@ #include "../shm_wrapper/shm_wrapper.h" #include "message.h" -#include // ************************************************************************************* - /** * Each device will have a unique port number. * For example, if there is only one Arduino connected, it will (probably) appear as diff --git a/scripts/test.sh b/scripts/test.sh index 680b3427..71af787a 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -67,9 +67,6 @@ function run_tests { # installs function clean_up as SIGINT handler trap 'sigint_handler' INT -# build all of Runtime -./runtime build - # replace the logger config with the production config mv logger/logger.config logger/logger.config.orig cp -p tests/logger.config logger/logger.config