forked from PX4/PX4-Autopilot
-
Notifications
You must be signed in to change notification settings - Fork 13
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Download fpga files based on json file in saluki-fpga repo Generate json file of the px4 build container
- Loading branch information
1 parent
519e800
commit 35cdd02
Showing
7 changed files
with
287 additions
and
24 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,57 @@ | ||
| # combine all fpga json files to one file | ||
| # this has to be done because each fpga target has their own json file and we need to | ||
| # save the info. Each fpga json file includes info about fpga and bootloader (sha, | ||
| # link to build, filename, hw, etc.) and this has to be passed to px4 json file. | ||
|
|
||
| # as an input takes | ||
| # 1: fpga directory which contains json files | ||
| # 2: an output filename | ||
|
|
||
| import sys | ||
| import os | ||
| import json | ||
|
|
||
| # Check if input is provided for directory and output file | ||
| if len(sys.argv) < 3: | ||
| print("usage: {} [dir] [output_file.json]".format(sys.argv[0])) | ||
| sys.exit(1) | ||
|
|
||
| # Collect all JSON files from the specified directory and process one by one | ||
| json_files = [] | ||
| for root, dirs, files in os.walk(sys.argv[1]): | ||
| for file in files: | ||
| if file.endswith(".json"): | ||
| json_files.append(os.path.join(root, file)) | ||
|
|
||
| json_data = {} | ||
| for file in json_files: | ||
| print("{}: found file: {}".format(sys.argv[0], file)) | ||
|
|
||
| with open(file, 'r') as f: | ||
| data = json.load(f) | ||
|
|
||
| # files array has to be merged separately | ||
| fpga_files_tmp_data = json_data.get("fpga-files", []) + data.get("fpga-files", []) | ||
|
|
||
| # WORKAROUND: different naming schemes used in different repos | ||
| # fpga and bootloader uses for example saluki_pi (underscore) | ||
| # px4 uses saluki-pi (dash) | ||
| # replace all saluki_ with saluki- in fpga-files[].hw | ||
| for fpga_file in fpga_files_tmp_data: | ||
| if fpga_file.get("hw").startswith("saluki_"): | ||
| fpga_file["hw"] = fpga_file["hw"].replace("saluki_", "saluki-") | ||
|
|
||
| # combine all json data into one | ||
| json_data = {**json_data, **data} | ||
|
|
||
| # add fpga-files from temporary merged array | ||
| json_data["fpga-files"] = fpga_files_tmp_data | ||
|
|
||
|
|
||
| print( "{}: this is the combined json data".format(sys.argv[0])) | ||
| print (json.dumps(json_data, indent=4, sort_keys=True)) | ||
|
|
||
| # write the json_data to the output file sorted by keys | ||
| with open(sys.argv[2], 'w') as outfile: | ||
| print("{}: writing to file: {}".format(sys.argv[0], sys.argv[2])) | ||
| json.dump(json_data, outfile, indent=4, sort_keys=True) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,73 @@ | ||
| # combine multiple json files to one file | ||
| # takes as an input | ||
| # 1: output json filename | ||
| # 2: px4 directory from there the px4 json files are collected | ||
| # 3: fpga json file compiled by collect_json_files.py | ||
| # 4: build info file compiled by generate_basic_build_info.sh | ||
|
|
||
| # This has to be ran after all px4 builds are done because they are ran in parallel on separate environments | ||
| # combine all px4 json files, fpga json file and basic build info to one file | ||
| # the result is the final saluki_build_info.json file which get packaged to px4 container | ||
|
|
||
| import sys | ||
| import os | ||
| import json | ||
| import glob | ||
|
|
||
|
|
||
| # Check if input is provided for output file | ||
| if len(sys.argv) < 5: | ||
| print("usage: {} [output_file.json] [px4_json_directory] [fpga_json_file] [build_info_file]".format(sys.argv[0])) | ||
| sys.exit(1) | ||
|
|
||
| output_file = sys.argv[1] | ||
| px4_json_directory = sys.argv[2] | ||
| fpga_json_file = sys.argv[3] | ||
| build_info_file = sys.argv[4] | ||
|
|
||
| # Collect all px4 JSON files | ||
| px4_json_files = glob.glob(os.path.join(px4_json_directory, "*.json")) | ||
| px4_json_data = [] | ||
|
|
||
| # collect px4 file info to array | ||
| for file in px4_json_files: | ||
| print("{}: processing file: {}".format(sys.argv[0], file)) | ||
|
|
||
| with open(file, 'r') as f: | ||
| temp_data = json.load(f) | ||
| # append "filename" with "/firmware" prefix | ||
| temp_data["filename"] = "/firmware/" + temp_data["filename"] | ||
| px4_json_data.append(temp_data) | ||
|
|
||
| print("{}: collected px4 json data".format(sys.argv[0], file)) | ||
| print(px4_json_data) | ||
|
|
||
| # read fpga json file | ||
| with open(fpga_json_file, 'r') as f: | ||
| fpga_json_data = json.load(f) | ||
| fpga_json_data['files'] = fpga_json_data.pop('fpga-files') | ||
| print("{}: fpga json data".format(sys.argv[0], file)) | ||
| print(json.dumps(fpga_json_data, indent=4, sort_keys=True)) | ||
|
|
||
| # fpga json files to px4 json data | ||
| px4_json_data.extend(fpga_json_data.pop('files')) | ||
|
|
||
| # combine all json data into one | ||
| json_data = {} | ||
| json_data = {**json_data, **fpga_json_data} | ||
|
|
||
| # add file array from combined json data | ||
| json_data["files"] = px4_json_data | ||
|
|
||
| # combine basic build info from build_info_file | ||
| with open(build_info_file, 'r') as f: | ||
| build_info_json_data = json.load(f) | ||
| json_data = {**json_data, **build_info_json_data} | ||
|
|
||
| print("{}: this is the combined json data".format(sys.argv[0], file)) | ||
| print(json.dumps(json_data, indent=4, sort_keys=True)) | ||
|
|
||
| # write the json_data to the output file sorted by keys | ||
| with open(output_file, 'w') as outfile: | ||
| print("{}: writing to file: {}".format(sys.argv[0], output_file)) | ||
| json.dump(json_data, outfile, indent=4, sort_keys=True) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,64 @@ | ||
| set -eu | ||
|
|
||
| # input parameters: | ||
| # 1: repo url | ||
| # 2: product | ||
| # 3: output dir | ||
| if [ -z ${3+x} ]; then | ||
| echo "Error: ${0} requires 3 parameters" | ||
| echo "Usage: ${0} <repo url> <product> <output dir>" | ||
| exit 1 | ||
| fi | ||
|
|
||
| saluki_v2_fpga_container="${1:-}" | ||
| product="${2:-}" | ||
| output_dir="${3:-}" | ||
| fileinfo_json="saluki_file_info.json" | ||
| fileinfo_json_out=${output_dir}/${fileinfo_json} | ||
|
|
||
| echo "fetching fpga files from ${saluki_v2_fpga_container}" | ||
|
|
||
| mkdir -p ${output_dir} | ||
|
|
||
| # run temporary container to copy the files | ||
| container_name=fpga_tmp_container_$(date +%s) | ||
| docker run -d --pull always --name ${container_name} ${saluki_v2_fpga_container} | ||
|
|
||
| # try to get the build info json | ||
| docker cp ${container_name}:/${fileinfo_json} ${fileinfo_json_out} || true | ||
|
|
||
| # legacy: if saluki build info doesnt exist, copy the whole /firmware directory | ||
| if [ ! -f ${fileinfo_json_out} ]; then | ||
| echo "Using legacy method to copy the whole /firmware directory" | ||
| docker cp ${container_name}:/firmware ${output_dir}/ | ||
| else | ||
| echo "using info from ${fileinfo_json_out}" | ||
| # find out info from json | ||
| # files to be used from fileinfo_json files[] -> filename | ||
| # product=$(echo ${product}|sed 's/_/-/g') | ||
| filename=$(jq '.files[] | select(.hw=="'${product}'").filename' ${fileinfo_json_out}|sed 's/"//g') | ||
|
|
||
| tmp_reponame=$(jq '.reponame' ${fileinfo_json_out}) | ||
| tmp_sha=$(jq '.sha' ${fileinfo_json_out}) | ||
| tmp_build_url=$(jq '.build_url' ${fileinfo_json_out}) | ||
|
|
||
| # filter files for this product and append $output_dir to .files[].filename | ||
| # file path is appended because the files are copied to a different directory so it will match the fileinfo.json | ||
| tmp_files=$(jq '.files[] | select(.hw=="'${product}'") | .filename = "/firmware/'${output_dir}'"+.filename' ${fileinfo_json_out}|jq -s .) | ||
|
|
||
| tmp_json="{\"${product}_fpga_reponame\":${tmp_reponame},\"${product}_fpga_sha\":${tmp_sha},\"${product}_fpga_build_url\":${tmp_build_url},\"fpga-files\":${tmp_files}}" | ||
| echo $tmp_json | ||
| echo $(echo $tmp_json|jq .) | ||
| echo $tmp_json|jq . > ${fileinfo_json_out} | ||
|
|
||
| for file in ${filename[@]}; do | ||
| # copy each file | ||
| echo "make target dir (dirname of) ${output_dir}/${file}" | ||
| mkdir -p $(dirname ${output_dir}/${file}) | ||
| echo "copying ${file} to ${output_dir}/${file}" | ||
| docker cp ${container_name}:${file} ${output_dir}/${file} | ||
| done | ||
| fi | ||
|
|
||
| docker stop ${container_name} | ||
| docker rm ${container_name} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,42 @@ | ||
| #!/bin/bash | ||
| set -eu | ||
|
|
||
| # This script generates a build info json file with the basic info of the build | ||
| # "reponame": "repo_name", | ||
| # "sha": "short_sha", | ||
| # "build_url": "build_url" | ||
| # this has to be ran in github environment because it collects info from the build environment | ||
| # it will be then passed to the fpga build container | ||
|
|
||
|
|
||
| # check if an input is provided for the build_info.json file | ||
| if [ -z "${1:-}" ]; then | ||
| echo "usage: $0 [output_file.json]" | ||
| exit 1 | ||
| fi | ||
|
|
||
| repo=$(basename `git rev-parse --show-toplevel`) | ||
| sha=$(git rev-parse --short HEAD) | ||
| # github variables are set only in github | ||
| if [ -z "${GITHUB_SERVER_URL:-}" ] || [ -z "${GITHUB_REPOSITORY:-}" ] || [ -z "${GITHUB_RUN_ID:-}" ]; then | ||
| build_url="undefined" | ||
| else | ||
| build_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" | ||
| fi | ||
|
|
||
| # associative array of sha and build_url | ||
| declare -A build_info | ||
| build_info["reponame"]=${repo} | ||
| build_info["sha"]=${sha} | ||
| build_info["build_url"]=${build_url} | ||
|
|
||
| # loop thru associative array and print key value pairs | ||
| json_info="{" | ||
| for key in "${!build_info[@]}"; do | ||
| json_info+=$(echo "\"$key\": \"${build_info[$key]}\",") | ||
| echo $key: ${build_info[$key]} | ||
| done | ||
| json_info="${json_info%,} }" | ||
|
|
||
| echo "json_info: ${json_info}" | ||
| echo ${json_info} | jq .>${1} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.