diff --git a/.github/workflows/static.check.scripts/flawfinder.sh b/.github/workflows/static.check.scripts/flawfinder.sh new file mode 100644 index 0000000000..78cd7b33f6 --- /dev/null +++ b/.github/workflows/static.check.scripts/flawfinder.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash + +## +# Imported from TAOS-CI +# Copyright (c) 2018 Samsung Electronics Co., Ltd. All Rights Reserved. +# Modified for Github-Action in 2024. +# +# Argument 1 ($1): the file containing the list of files to be checked +# Argument 2 ($2): the flawfinder check level. (1 by default) +# + +## +# @file flawfinder.sh +# @brief This module examines C/C++ source code to find a possible security weaknesses. +# Originally, pr-prebuild-flawfinder.sh +# +# Flawfinder searches through C/C++ source code looking for potential security flaws. It examines +# all of the project's C/C++ source code. It is very useful for quickly finding and removing some +# security problems before a program is widely released. +# +# Flawfinder produces a list of `hits` (potential security flaws), sorted by risk; the riskiest +# hits are shown first. The risk level is shown inside square brackets and varies from 0 (very +# little risk) to 5 (great risk). This risk level depends not only on the function, +# but on the values of the parameters of the function. For example, constant strings are often less risky +# than fully variable strings in many contexts, and in those contexts the hit will have a lower risk level. +# Hit descriptions also note the relevant Common Weakness Enumeration (CWE) identifier(s) in parentheses. +# +# @see https://dwheeler.com/flawfinder/ +# @see https://sourceforge.net/projects/flawfinder/ +# @see https://github.com/nnstreamer/TAOS-CI +# @see https://github.com/nnstreamer/nnstreamer +# @author Geunsik Lim +# @author MyungJoo Ham + +if [ -z $1 ]; then + echo "::error The argument (file path) is not given." + exit 1 +fi + +pr_flawfinder_check_level=1 +if [ -z $2 ]; then + echo "Flawfinder check level is not given. The default value 1 is applied" +else + if [ -n "$2" ] && [ "$2" -eq "$2" ] 2>/dev/null; then + echo "" + else + echo ":error The second argument '$2' should be a number." + exit 1 + fi + pr_flawfinder_check_level=$2 +fi + +files=$1 +failed=0 + +if [ ! -f $files ]; then + echo "::error The file $files does not exists." + exit 1 +fi + +function check(){ + which $1 + if [[ $? -ne 0 ]]; then + echo "::error The command $1 is required, but not found." + exit 1 + fi +} +# Check if server administrator install required commands +check flawfinder +check file +check grep +check cat +check wc +check git +check awk + +static_analysis_sw="flawfinder" +static_analysis_rules="--html --context --minlevel=$pr_flawfinder_check_level" +flawfinder_result=$(mktemp) + +# Display the flawfinder version that is installed in the CI server. +# Note that the out-of-date version can generate an incorrect result. +flawfinder --version + +# Read file names that a contributor modified (e.g., added, moved, deleted, and updated) from a last commit. +# Then, inspect C/C++ source code files from *.patch files of the last commit. +for file in `cat $file`; do + # Skip the obsolete folder + if [[ $file =~ ^obsolete/.* ]]; then + continue + fi + # Skip the external folder + if [[ $file =~ ^external/.* ]]; then + continue + fi + # Handle only text files in case that there are lots of files in one commit. + if [[ `file $file | grep "ASCII text" | wc -l` -gt 0 ]]; then + # in case of C/C++ source code + case $file in + # in case of C/C++ code + *.c|*.cc|*.cpp|*.c++) + $static_analysis_sw $static_analysis_rules $file > ${flawfinder_result} + bug_nums=`cat ${flawfinder_result} | grep "Hits = " | awk '{print $3}'` + # Report the execution result. + if [[ $bug_nums -gt 0 ]]; then + echo "[ERROR] $static_analysis_sw: failed. file name: $file, There are $bug_nums bug(s)." + echo "::group::The flawfinder result of $file:" + cat ${flawfinder_result} + echo "::endgroup::" + failed=1 + else + echo "[DEBUG] $static_analysis_sw: passed. file name: $file, There are $bug_nums bug(s)." + fi + ;; + esac + fi +done + +if [[ "$failed" == "1" ]]; then + echo "::error There is an error from flawfinder. Please review the detailed results above." + exit 1 +fi diff --git a/.github/workflows/static.check.scripts/prohibited-words.sh b/.github/workflows/static.check.scripts/prohibited-words.sh new file mode 100644 index 0000000000..29ea1b985e --- /dev/null +++ b/.github/workflows/static.check.scripts/prohibited-words.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash + +## +# Imported from TAOS-CI +# Copyright (c) 2018 Samsung Electronics Co., Ltd. All Rights Reserved. +# Modified for Github-Action in 2024. +# +# Argument 1 ($1): the file containing the list of files to be checked +# + +## +# @file prohibited-words.sh +# @brief Check if there are prohibited words in the text files. +# Originally, pr-prebuild-prohibited-words.sh +# +# It to check a prohibited word if there are unnecessary words in the source codes. +# +# @see https://github.com/nnstreamer/TAOS-CI +# @see https://github.com/nnstreamer/nnstreamer +# @author Geunsik Lim +# @author MyungJoo Ham +# + +if [ -z $1 ]; then + echo "::error The argument (file path) is not given." + exit 1 +fi + +files=$1 +failed=0 + +if [ ! -f $files ]; then + echo "::error The file $files does not exists." + exit 1 +fi + +function check(){ + which $1 + if [[ $? -ne 0 ]]; then + echo "::error The command $1 is required, but not found." + exit 1 + fi +} + +check git +check grep +check wc + +# Inspect all files that contributor modifed. +target_files="" +for file in `cat $files`; do + # Skip obsolete folder + if [[ $file =~ ^obsolete/.* ]]; then + continue + fi + # Skip external folder + if [[ $file =~ ^external/.* ]]; then + continue + fi + # Handle only text files among files in one commit. + if [[ `file $file | grep "ASCII text" | wc -l` -gt 0 ]]; then + case $file in + # Declare source code files to inspect a prohibited word + *.c | *.h | *.cpp | *.hpp| *.py | *.sh | *.php | *.md ) + target_files="$target_files $file" + ;; + esac + fi +done + +bad_words_sw="grep" +bad_words_list=".github/workflows/static.check.scripts/prohibited-words.txt" +bad_words_rules="--color -n -r -H -f $bad_words_list" +bad_words_log_file=$(mktemp) + +# Run a prohibited-words module in case that a PR includes text files. +if [[ -n "${target_files/[ ]*\n/}" ]]; then + if [[ ! -f $bad_words_list ]]; then + echo "::error A prohibited word list file, $bad_words_list, doesn't exist." + exit 1 + fi + + # Step 1: Run this module to filter prohibited words from a text file. + # (e.g., grep --color -f "$PROHIBITED_WORDS" $filename) + $bad_words_sw $bad_words_rules $target_files > ${bad_words_log_file} + + # Step 2: Display the execution result for debugging in case of a failure + cat ${bad_words_log_file} + + # Step 3: Count prohibited words from variable result_content + result_count=$(cat ${bad_word_log_file} | grep -c '^' ) + + # Step 4: change a value of the check result + if [[ $result_count -gt 0 ]]; then + echo "::error There are prohibited words in this PR (counted $result_count)." + exit 1 + else + echo "There is no prohibited word found in this PR." + fi +fi diff --git a/.github/workflows/static.check.scripts/prohibited-words.txt b/.github/workflows/static.check.scripts/prohibited-words.txt new file mode 100644 index 0000000000..9771f6ac23 --- /dev/null +++ b/.github/workflows/static.check.scripts/prohibited-words.txt @@ -0,0 +1,3 @@ +samsung.net +FUCK +file diff --git a/.github/workflows/static.check.scripts/shellcheck.sh b/.github/workflows/static.check.scripts/shellcheck.sh new file mode 100644 index 0000000000..f36ac510f0 --- /dev/null +++ b/.github/workflows/static.check.scripts/shellcheck.sh @@ -0,0 +1,99 @@ +#!/usr/bin/env bash + +## +# Imported from TAOS-CI +# Copyright (c) 2018 Samsung Electronics Co., Ltd. All Rights Reserved. +# Modified for Github-Action in 2024. +# +# Argument 1 ($1): the file containing the list of files to be checked +# + +## +# @file shellcheck.sh +# @brief This module is a static analysis tool for shell scripts such as sh, bash. +# Originally, pr-prebuild-shellcheck.sh +# +# It is mainly focused on handling typical beginner and intermediate level syntax errors +# and pitfalls where the shell just gives a cryptic error message or strange behavior, +# but it also reports on a few more advanced issues where corner cases can cause delayed +# failures. +# +# @see https://www.shellcheck.net/ +# @see https://github.com/nnstreamer/TAOS-CI +# @see https://github.com/nnstreamer/nnstreamer +# @author Geunsik Lim +# @author MyungJoo Ham +# + +if [ -z $1 ]; then + echo "::error The argument (file path) is not given." + exit 1 +fi + +files=$1 +failed=0 + +if [ ! -f $files ]; then + echo "::error The file $files does not exists." + exit 1 +fi + +function check(){ + which $1 + if [[ $? -ne 0 ]]; then + echo "::error The command $1 is required, but not found." + exit 1 + fi +} +# Check if required commands are installed by server administrator +check cat +check shellcheck +check file +check grep +check wc + +shell_syntax_analysis_sw="shellcheck" +shell_syntax_analysis_rules="-s bash" +shell_syntax_check_result=$(mktemp) + +# Inspect all files that contributor modifed. +for file in `cat $files`; do + # Skip obsolete folder + if [[ $file =~ ^obsolete/.* ]]; then + continue + fi + # Skip external folder + if [[ $file =~ ^external/.* ]]; then + continue + fi + # Handle only text files in case that there are lots of files in one commit. + echo "[DEBUG] file name is ($file)." + if [[ `file $file | grep "shell script" | wc -l` -gt 0 ]]; then + case $file in + # In case of .sh or .bash file + *.sh | *.bash) + echo "($file) file is a shell script file with the 'shell script' text format." + + cat $file | $shell_syntax_analysis_sw $shell_syntax_analysis_rules > ${shell_syntax_check_result} + line_count=`cat ${shell_syntax_check_result} | wc -l` + + echo "::group::shellcheck result of $file" + cat ${shell_syntax_check_result} + echo "::endgroup::" + + # TODO: 9,000 is declared by heuristic method from our experiment. + if [[ $line_count -gt 9000 ]]; then + echo "$shell_syntax_analysis_sw: failed. file name: $file There are $line_count lines." + failed=1 + else + echo "$shell_syntax_analysis_sw: passed. file name: $file There are $line_count lines." + fi + ;; + esac + fi +done + +if [[ "$failed" == "1" ]]; then + echo "::error These is a enough number of errors in a shell file with shellcheck. Please refer to the log above". + exit 1 +fi diff --git a/.github/workflows/static.check.scripts/signed-off-by.sh b/.github/workflows/static.check.scripts/signed-off-by.sh new file mode 100644 index 0000000000..e1aaa58431 --- /dev/null +++ b/.github/workflows/static.check.scripts/signed-off-by.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash + +## +# Imported from TAOS-CI +# Copyright (c) 2018 Samsung Electronics Co., Ltd. All Rights Reserved. +# Modified for Github-Action in 2024. +# +# Argument 1 ($1): the number of commits in this PR +# + +## +# @file signed-off-by.sh +# @brief Check if contributor write Sigend-off-by message. Originally, pr-prebuild-signed-off-by.sh +# @see https://github.com/nnstreamer/TAOS-CI +# @see https://github.com/nnstreamer/nnstreamer +# @author Sewon oh +# @author MyungJoo Ham +# + +if [ -z $1 ]; then + echo "::error The argument (the number of commits in this PR) is not given." + exit 1 +fi + +if [ -n "$1" ] && [ "$1" -eq "$1" ] 2>/dev/null; then + echo "" +else + echo ":error The first argument '$1' should be a number." + exit 1 +fi + +function check(){ + which $1 + if [[ $? -ne 0 ]]; then + echo "::error The command $1 is required, but not found." + exit 1 + fi +} + +# Check if server administrator install required commands +check git + +for i in $(seq 0 $[$1 - 1]); do + count=`git show --no-notes -s HEAD~$i | grep "Signed-off-by: [^ ].*[^ ].*<[^ ].*@[^ ].*>" | wc -l` + if [[ $count -lt 1 ]]; then + git show --stat HEAD~$i + echo "============================================" + echo "" + echo "::error This commit does not have a proper Signed-off-by. Refer to https://ltsi.linuxfoundation.org/software/signed-off-process/ for some information about Signed-off-by. We require Signed-off-by: NAME signed by indivisual contributors." + exit 1 + else + id=`git show --pretty="format:%h" --no-notes -s HEAD~$i` + echo "Commit $id has proper a signed-off-by string." + fi +done diff --git a/.github/workflows/static.check.scripts/sloccount.sh b/.github/workflows/static.check.scripts/sloccount.sh new file mode 100644 index 0000000000..3d9aeb49da --- /dev/null +++ b/.github/workflows/static.check.scripts/sloccount.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash + +## +# Imported from TAOS-CI +# Copyright (c) 2018 Samsung Electronics Co., Ltd. All Rights Reserved. +# Modified for Github-Action in 2024. +# +# Argument 1 ($1): the file containing the list of files to be checked +# + +## +# @file sloccount.sh +# @brief Count physical source lines of code (SLOC). Originally, pr-prebuild-sloccount.sh +# +# It is a set of tools for counting physical Source Lines of Code (SLOC) in a large +# number of languages of a potentially large set of programs. +# +# @see https://packages.ubuntu.com/search?keywords=sloccount +# @see https://github.com/nnstreamer/TAOS-CI +# @see https://github.com/nnstreamer/nnstreamer +# @author Geunsik Lim +# @author MyungJoo Ham +# + +if [ -z $1 ]; then + echo "::error The argument (file path) is not given." + exit 1 +fi + +files=$1 +failed=0 + +if [ ! -f $files ]; then + echo "::error The file $files does not exists." + exit 1 +fi + +function check(){ + which $1 + if [[ $? -ne 0 ]]; then + echo "::error The command $1 is required, but not found." + exit 1 + fi +} + +# Check if server administrator install required commands +check sloccount +check git +check file +check mkdir +check grep + +sloc_analysis_sw="sloccount" +sloc_data_folder=$(mktemp -d) +sloc_analysis_rules="--wide --multiproject --datadir $sloc_data_folder" +sloc_check_result=$(mktemp) + +# Inspect all files that contributor modifed. +for file in `cat $files`; do + # skip obsolete folder + if [[ $file =~ ^obsolete/.* ]]; then + continue + fi + # skip external folder + if [[ $file =~ ^external/.* ]]; then + continue + fi + # Handle only text files in case that there are lots of files in one commit. + if [[ `file $file | grep "ASCII text" | wc -l` -gt 0 ]]; then + # Run a SLOCCount module in case that a PR includes source codes. + case $file in + *.c | *.cpp | *.py | *.sh | *.php ) + sloc_target_dir=${SRC_PATH} + + # Run this module + echo "::group::sloccount result" + $sloc_analysis_sw $sloc_analysis_rules . > ${sloc_check_result} + echo "::endgroup::" + run_result=$? + if [[ $run_result -eq 0 ]]; then + echo "" + exit 0 + else + echo "::error Sloccount failed." + exit 1 + fi + ;; + esac + fi +done diff --git a/.github/workflows/static.check.yml b/.github/workflows/static.check.yml index 011fedba4c..762ee1e213 100644 --- a/.github/workflows/static.check.yml +++ b/.github/workflows/static.check.yml @@ -21,7 +21,7 @@ jobs: ref: ${{ github.event.pull_request.head.sha }} fetch-depth: -${{ github.event.pull_request.commits }} - name: Preparing step 2... Installing packages - run: sudo apt-get update && sudo apt-get install clang-format git grep gawk exuberant-ctags indent pylint rpmlint aha cppcheck aspell doxygen + run: sudo apt-get update && sudo apt-get install clang-format git grep gawk exuberant-ctags indent pylint rpmlint aha cppcheck aspell doxygen sloccount shellcheck flawfinder - name: Preparing step 3... Identify changed files run: | tmpfile_pre=$(mktemp) @@ -132,3 +132,27 @@ jobs: # Originally from "pr-prebuild-doxygen-build" run: | bash .github/workflows/static.check.scripts/doxygen-build.sh $changed_file_list + - name: /Checker/ sloccount limit + # Originally from "pr-prebuild-sloccount" + run: | + bash .github/workflows/static.check.scripts/sloccount.sh $changed_file_list + - name: /Checker/ prohibited words + # Originally from "pr-prebuild-prohibited-words" + run: | + bash .github/workflows/static.check.scripts/prohibited-words.sh $changed_file_list + - name: /Checker/ signed-off-by required + # Originally from "pr-prebuild-signed-off-by" + run: | + bash .github/workflows/static.check.scripts/signed-off-by.sh ${{ github.event.pull_request.commits }} + - name: /Checker/ shellcheck for shell scripts + # Originally from "pr-prebuild-shellcheck" + run: | + bash .github/workflows/static.check.scripts/shellcheck.sh $changed_file_list + - name: /Checker/ flawfinder for C/C++ files + # Originally from "pr-prebuild-flawfinder" + run: | + bash .github/workflows/static.check.scripts/flawfinder.sh $changed_file_list 1 + - name: /Checker/ covertity for C/C++ files + # Originally from "pr-prebuild-coverity" + run: | + #bash .github/workflows/static.check.scripts/coverity.sh $changed_file_list