From a766a04ade90988f82197267cbc8f33b7f227ba2 Mon Sep 17 00:00:00 2001 From: "Rule Timothy (CC/EMT2)" Date: Thu, 2 May 2024 16:50:40 +0200 Subject: [PATCH] Correct integer handling in YAML. Docker build speed-up. Signed-off-by: Rule Timothy (VM/EMT3) --- Makefile | 1 + Taskfile.yml | 9 ++-- dse/network/examples/stub/stub.yaml | 49 ------------------- extra/tools/network/README.md | 2 +- extra/tools/network/build/package/Dockerfile | 45 ++++++++++++----- extra/tools/network/build/package/gencode.py | 14 +++--- .../network/internal/app/generate/network.go | 28 +++++------ .../dse.schemas/code/go/dse/kind/metadata.go | 2 +- 8 files changed, 63 insertions(+), 87 deletions(-) delete mode 100644 dse/network/examples/stub/stub.yaml diff --git a/Makefile b/Makefile index 86c6e9d..cea646d 100644 --- a/Makefile +++ b/Makefile @@ -79,6 +79,7 @@ tools: mkdir -p extra/tools/$$d/build/stage ;\ cp -r licenses -t extra/tools/$$d/build/stage ;\ docker build -f extra/tools/$$d/build/package/Dockerfile \ + --build-arg="CCACHE_SECONDARY_STORAGE=$(CCACHE_SECONDARY_STORAGE)" \ --tag $$d:test extra/tools/$$d ;\ done; diff --git a/Taskfile.yml b/Taskfile.yml index 4788bc2..0491175 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -139,12 +139,15 @@ tasks: DBCNAME: sh: basename {{.DBCFILE}} .dbc OUTDIR: '{{.OUTDIR | default "OUTDIR_NOT_SPECIFIED"}}' - FILTER: '{{.FILTER | default ""}}' - CYCLE_TIME: '{{.CYCLE_TIME | default ""}}' + FILTER: '{{if .FILTER}}--filter {{.FILTER}}{{else}}{{end}}' + CYCLE_TIME: '{{if .CYCLE_TIME}}--cycle_time {{.CYCLE_TIME}}{{else}}{{end}}' IMAGE: '{{.IMAGE | default .NETWORK_IMAGE}}' TAG: '{{.TAG | default .NETWORK_TAG}}' cmds: - - docker run --rm -v $(pwd):/sim {{.IMAGE}}:{{.TAG}} gen-code {{.DBCFILE}} {{.OUTDIR}} --filter {{.FILTER}} --cycle_time {{.CYCLE_TIME}} + - docker run --rm -v $(pwd):/sim {{.IMAGE}}:{{.TAG}} gen-code + {{.DBCFILE}} {{.OUTDIR}} + {{.FILTER}} + {{.CYCLE_TIME}} requires: vars: [DBCFILE, OUTDIR] sources: diff --git a/dse/network/examples/stub/stub.yaml b/dse/network/examples/stub/stub.yaml deleted file mode 100644 index 5508200..0000000 --- a/dse/network/examples/stub/stub.yaml +++ /dev/null @@ -1,49 +0,0 @@ -frames: - ExampleMessage: - cycle_time_ms: null - frame_id: 0x1f0u - frame_length: 8u - is_can_fd: false - is_extended_frame: false - ExampleMessage2: - cycle_time_ms: null - frame_id: 0x1f1u - frame_length: 8u - is_can_fd: false - is_extended_frame: false - FloatTypes: - cycle_time_ms: null - frame_id: 0x1f5u - frame_length: 16u - is_can_fd: false - is_extended_frame: false - FunctionExample: - cycle_time_ms: null - frame_id: 0x1f2u - frame_length: 8u - is_can_fd: false - is_extended_frame: false - MuxMessage: - cycle_time_ms: null - frame_id: 0x258u - frame_length: 12u - is_can_fd: false - is_extended_frame: false - ScheduledMessage: - cycle_time_ms: 10u - frame_id: 0x1f6u - frame_length: 8u - is_can_fd: false - is_extended_frame: false - SignedTypes: - cycle_time_ms: null - frame_id: 0x1f4u - frame_length: 16u - is_can_fd: false - is_extended_frame: false - UnsignedTypes: - cycle_time_ms: null - frame_id: 0x1f3u - frame_length: 16u - is_can_fd: false - is_extended_frame: false diff --git a/extra/tools/network/README.md b/extra/tools/network/README.md index cbcce2f..32b1ee0 100644 --- a/extra/tools/network/README.md +++ b/extra/tools/network/README.md @@ -55,7 +55,7 @@ $ make $ make tools # Run the Task (which uses the Toolset container). -$ task --force generate DBCFILE=stub/stub.dbc +$ task --force generate DBCFILE=dse/network/examples/stub/stub.dbc ``` diff --git a/extra/tools/network/build/package/Dockerfile b/extra/tools/network/build/package/Dockerfile index 4275b35..da047c1 100644 --- a/extra/tools/network/build/package/Dockerfile +++ b/extra/tools/network/build/package/Dockerfile @@ -7,24 +7,40 @@ # ===================== FROM python:3.8.12-slim-bullseye as nuitka +ARG CCACHE_VERSION=4.6.1 +ARG CCACHE_SECONDARY_STORAGE + ENV PATH="/opt/venv/bin:$PATH" -ADD third_party/cantools /src/third_party/cantools -ADD build/package/gencode.py /src/build/package/gencode.py -ADD build/package/convert.py /src/build/package/convert.py -ADD build/package/mimetype.py /src/build/package/mimetype.py RUN set -eux; \ apt-get -y update; \ apt-get -y upgrade; \ apt-get -y install --no-install-recommends \ build-essential \ + cmake \ + curl \ gcc \ git \ + make \ patchelf \ ; \ apt-get clean; \ - rm -rf /var/lib/apt/lists/*; \ - \ + rm -rf /var/lib/apt/lists/*; + +RUN curl -fSL https://github.com/ccache/ccache/archive/refs/tags/v${CCACHE_VERSION}.tar.gz -o ccache-${CCACHE_VERSION}.tar.gz \ + && tar xvzf ccache-${CCACHE_VERSION}.tar.gz \ + && cd ccache-${CCACHE_VERSION} \ + && mkdir build; cd build \ + && cmake \ + -DCMAKE_BUILD_TYPE=Release \ + -DHIREDIS_FROM_INTERNET=ON \ + -DZSTD_FROM_INTERNET=ON \ + .. \ + && make; make install; which ccache; + +ADD third_party/cantools /src/third_party/cantools +RUN set -eux; \ + ccache -c; \ python -m venv /opt/venv; \ pip install --upgrade pip; pip install \ build \ @@ -35,20 +51,25 @@ RUN set -eux; \ nala \ nuitka \ pyyaml \ - ; \ - pip list; - -RUN set -eux; \ + && \ mkdir -p /tmp/cantools; \ tar -xzf /src/third_party/cantools/cantools.tar.gz -C /tmp/cantools --strip-components=1; \ ls -R /tmp/cantools \ && \ (cd /tmp/cantools; python -m build --wheel --outdir dist/ .); \ pip install /tmp/cantools cantools; \ + pip list + +ADD build/package/gencode.py /src/build/package/gencode.py +ADD build/package/convert.py /src/build/package/convert.py +ADD build/package/mimetype.py /src/build/package/mimetype.py +RUN set -eux; \ python -m nuitka --output-dir=/gencode --standalone --onefile /src/build/package/gencode.py; \ python -m nuitka --output-dir=/convert --standalone --onefile --include-package=canmatrix /src/build/package/convert.py; \ - python -m nuitka --output-dir=/mimetype --standalone --onefile /src/build/package/mimetype.py; \ - pip list; + python -m nuitka --output-dir=/mimetype --standalone --onefile /src/build/package/mimetype.py \ + && \ + pip list; \ + ccache -p | grep secondary_storage; ccache -s -v diff --git a/extra/tools/network/build/package/gencode.py b/extra/tools/network/build/package/gencode.py index 9b5d4ba..3cfbf58 100644 --- a/extra/tools/network/build/package/gencode.py +++ b/extra/tools/network/build/package/gencode.py @@ -43,9 +43,9 @@ def scan_messages(dbc_file, out_path, filter, cycle_time): isContainer = True messageName = camel_to_snake_case(message.name) frames[messageName] = { - 'frame_id': hex(message.frame_id), - 'frame_length': str(message.length), - 'cycle_time_ms': str(message.cycle_time) if message.cycle_time else None, + 'frame_id': int(message.frame_id), + 'frame_length': int(message.length), + 'cycle_time_ms': int(message.cycle_time) if message.cycle_time else None, 'is_can_fd': message.is_fd, 'is_extended_frame': message.is_extended_frame, 'is_container': isContainer @@ -56,10 +56,10 @@ def scan_messages(dbc_file, out_path, filter, cycle_time): for mux_id, signals in message.signal_tree[0]['Header_ID'].items(): frames[messageName + '_' + hex(mux_id)] = { 'container': messageName, - 'container_mux_id': hex(mux_id), - 'frame_id': hex(message.frame_id), - 'frame_length': str(message.length), - 'cycle_time_ms': str(message.cycle_time) if message.cycle_time else None, + 'container_mux_id': int(mux_id), + 'frame_id': int(message.frame_id), + 'frame_length': int(message.length), + 'cycle_time_ms': int(message.cycle_time) if message.cycle_time else None, 'is_can_fd': message.is_fd, 'is_extended_frame': message.is_extended_frame, 'signals' : [camel_to_snake_case(s) for s in signals] diff --git a/extra/tools/network/internal/app/generate/network.go b/extra/tools/network/internal/app/generate/network.go index 417fdbe..99d3c99 100644 --- a/extra/tools/network/internal/app/generate/network.go +++ b/extra/tools/network/internal/app/generate/network.go @@ -103,15 +103,15 @@ func (c *GenNetworkCommand) Run() error { if frameInfo != nil { annotations["frame_id"] = frameInfo.FrameId annotations["frame_length"] = frameInfo.FrameLength - annotations["frame_type"] = strconv.Itoa(getFrameType(frameInfo)) - if frameInfo.CycleTime != "" { + annotations["frame_type"] = getFrameType(frameInfo) + if frameInfo.CycleTime != 0 { annotations["cycle_time_ms"] = frameInfo.CycleTime } } annotations["struct_name"] = structName // Signals (field/member names of the struct). signals := []kind.NetworkSignal{} - offset := 0 + var offset int = 0 for _, s := range signalList { name := getSignalName(s.Name, c.signalStyle) signal := kind.NetworkSignal{Signal: name} @@ -131,14 +131,14 @@ func (c *GenNetworkCommand) Run() error { } if strings.ToLower(name) == "header_id" || strings.ToLower(name) == "header_dlc" { annotations["struct_member_name"] = s.Name - annotations["struct_member_offset"] = strconv.Itoa(offset) + annotations["struct_member_offset"] = offset annotations["struct_member_primitive_type"] = s.TypeName signal.Annotations = &annotations signals = append(signals, signal) } } else { //normal frames annotations["struct_member_name"] = s.Name - annotations["struct_member_offset"] = strconv.Itoa(offset) + annotations["struct_member_offset"] = offset annotations["struct_member_primitive_type"] = s.TypeName signal.Annotations = &annotations signals = append(signals, signal) @@ -170,13 +170,13 @@ func (c *GenNetworkCommand) Run() error { } type FrameInfo struct { - FrameId string `yaml:"frame_id"` - FrameLength string `yaml:"frame_length"` - CycleTime string `yaml:"cycle_time_ms"` + FrameId int `yaml:"frame_id"` + FrameLength int `yaml:"frame_length"` + CycleTime int `yaml:"cycle_time_ms"` CanFD bool `yaml:"is_can_fd"` ExtendedFrame bool `yaml:"is_extended_frame"` Container string `yaml:"container"` - ContainerMuxId string `yaml:"container_mux_id"` + ContainerMuxId int `yaml:"container_mux_id"` IsContainer bool `yaml:"is_container"` Signals []string `yaml:"signals"` } @@ -262,8 +262,8 @@ func processContainer(net *kind.Network, message kind.NetworkMessage, fmd *Frame muxAnnotations := kind.Annotations{} muxAnnotations["frame_id"] = frameInfo.FrameId muxAnnotations["frame_length"] = frameInfo.FrameLength - muxAnnotations["frame_type"] = strconv.Itoa(getFrameType(&frameInfo)) - if frameInfo.CycleTime != "" { + muxAnnotations["frame_type"] = getFrameType(&frameInfo) + if frameInfo.CycleTime != 0 { muxAnnotations["cycle_time_ms"] = frameInfo.CycleTime } muxAnnotations["container"] = frameInfo.Container @@ -271,7 +271,7 @@ func processContainer(net *kind.Network, message kind.NetworkMessage, fmd *Frame muxAnnotations["struct_name"] = annotations["struct_name"] muxSignalsList := frameInfo.Signals signals := []kind.NetworkSignal{} - offset := 0 + var offset int = 0 for _, s := range signalList { name := getSignalName(s.Name, signalStyle) signal := kind.NetworkSignal{Signal: name} @@ -286,7 +286,7 @@ func processContainer(net *kind.Network, message kind.NetworkMessage, fmd *Frame if strings.ToLower(name) == "header_id" || strings.ToLower(name) == "header_dlc" || slices.Contains(muxSignalsList, name) { annotations := kind.Annotations{} annotations["struct_member_name"] = s.Name - annotations["struct_member_offset"] = strconv.Itoa(offset) + annotations["struct_member_offset"] = offset annotations["struct_member_primitive_type"] = s.TypeName if strings.ToLower(name) == "header_id" { annotations["internal"] = "true" @@ -307,7 +307,7 @@ func processContainer(net *kind.Network, message kind.NetworkMessage, fmd *Frame if offset%8 != 0 { offset = ((offset / 8) * 8) + 8 } - muxAnnotations["struct_size"] = strconv.Itoa(offset) + muxAnnotations["struct_size"] = offset containedMessage.Annotations = &muxAnnotations net.Spec.Messages = append(net.Spec.Messages, containedMessage) } diff --git a/extra/tools/network/vendor/github.com/boschglobal/dse.schemas/code/go/dse/kind/metadata.go b/extra/tools/network/vendor/github.com/boschglobal/dse.schemas/code/go/dse/kind/metadata.go index 66cdaaf..66b46bd 100644 --- a/extra/tools/network/vendor/github.com/boschglobal/dse.schemas/code/go/dse/kind/metadata.go +++ b/extra/tools/network/vendor/github.com/boschglobal/dse.schemas/code/go/dse/kind/metadata.go @@ -1,6 +1,6 @@ package kind -type Annotations map[string]string +type Annotations map[string]interface{} type Labels map[string]string type ObjectMetadata struct { Annotations *Annotations `yaml:"annotations,omitempty"`