diff --git a/Dockerfile b/Dockerfile index bfd0e4d..fc6f02f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,7 +29,7 @@ RUN --mount=target=/var/lib/apt/lists,type=cache,sharing=private \ python python-cffi python-cffi-backend python-ipaddress \ python2-minimal python-ply python-pycparser python2.7 python2.7-minimal \ python3 python3-minimal python3.6 python3-minimal \ - python3-cffi python3-cffi-backend && \ + python3-cffi python3-cffi-backend dumb-init && \ apt-get install -yy clang-9 # TODO: add more packages above that are VPP deps @@ -43,4 +43,8 @@ RUN --mount=target=/var/lib/apt/lists,type=cache,sharing=private \ /debs/libvppinfra_*.deb \ /debs/vpp-api-python_*.deb -ENTRYPOINT /usr/bin/vpp +COPY start.sh /usr/local/bin/ +RUN chmod +x /usr/local/bin/start.sh + +ENTRYPOINT ["dumb-init", "--"] +CMD ["/usr/local/bin/start.sh"] diff --git a/Dockerfile.devel b/Dockerfile.devel index 5c13544..118d0a5 100644 --- a/Dockerfile.devel +++ b/Dockerfile.devel @@ -29,7 +29,7 @@ RUN --mount=target=/var/lib/apt/lists,type=cache,sharing=private \ python python-cffi python-cffi-backend python-ipaddress \ python2-minimal python-ply python-pycparser python2.7 python2.7-minimal \ python3 python3-minimal python3.6 python3-minimal \ - python3-cffi python3-cffi-backend && \ + python3-cffi python3-cffi-backend dumb-init && \ apt-get install -yy clang-9 # TODO: add more packages above that are VPP deps @@ -45,4 +45,8 @@ RUN --mount=target=/var/lib/apt/lists,type=cache,sharing=private \ /debs/libvppinfra-dev_*.deb \ /debs/vpp-api-python_*.deb -ENTRYPOINT /usr/bin/vpp +COPY start.sh /usr/local/bin/ +RUN chmod +x /usr/local/bin/start.sh + +ENTRYPOINT ["dumb-init", "--"] +CMD ["/usr/local/bin/start.sh"] diff --git a/start.sh b/start.sh new file mode 100644 index 0000000..20c230a --- /dev/null +++ b/start.sh @@ -0,0 +1,45 @@ +#!/bin/bash -x +# TODO: remove -x above ! +set -o errexit +set -o nounset +set -o pipefail +set -o errtrace + +trap "sleep 10" EXIT + +sysctl kernel.core_pattern="/tmp/coredump/core.%t.%e.%p" +cp /etc/vpp/startup.conf /tmp/startup.conf + +echo "cpu {" >>/tmp/startup.conf +if [[ ${VPP_USE_WORKER_CORE:-} ]]; then + # this expands ranges like 10-12,42 in the cpuset core list + # into an array by using eval and {x..y} brace expansion + cores=($(eval "echo $(sed 's/\([0-9]*\)-\([0-9]*\)/{\1..\2}/g;s/,/ /g' /sys/fs/cgroup/cpuset/cpuset.cpus)")) + if (( ${#cores[@]} < 2 )); then + echo >&2 "ERROR: need at least 2 cores" + exit 1 + fi + main_core=${cores[0]} + # try to pick a non-sibling core from the rest of the list + for ((i = 1; i < ${#cores[@]}; i++)); do + worker_core="${cores[${i}]}" + if ! grep -qE "\b(${main_core},${worker_core}|${worker_core},${main_core})\b" \ + /sys/devices/system/cpu/cpu*/topology/thread_siblings_list; then + break + fi + done + if (( ${i} == ${#cores[@]} )); then + echo >&2 "ERROR: couldn't find enough cores" + echo >&2 "ERROR: the main core ${main_core} and the worker core ${worker_core} are HT siblings!" + if [[ ! ${VPP_TOLERATE_HT_SIBLING_CORES:-} ]]; then + exit 1 + fi + fi + echo " main-core ${main_core}" >>/tmp/startup.conf + echo " corelist-workers ${worker_core}" >>/tmp/startup.conf +else + echo " workers 0" >>/tmp/startup.conf +fi +echo "}" >>/tmp/startup.conf + +/usr/bin/vpp -c /tmp/startup.conf diff --git a/upf/upf_pfcp_server.c b/upf/upf_pfcp_server.c index 432e83d..f9610a0 100644 --- a/upf/upf_pfcp_server.c +++ b/upf/upf_pfcp_server.c @@ -299,7 +299,7 @@ upf_pfcp_server_rx_msg (pfcp_msg_t * msg) req = pfcp_msg_pool_elt_at_index (psm, p[0]); hash_unset (psm->request_q, msg->seq_no); - upf_pfcp_server_stop_timer (req->timer); + upf_pfcp_server_stop_msg_timer (req); msg->node = req->node; @@ -491,7 +491,7 @@ restart_response_timer (pfcp_msg_t * msg) upf_debug ("Msg Seq No: %u, idx %u\n", msg->seq_no, id); if (msg->timer != ~0) - upf_pfcp_server_stop_timer (msg->timer); + upf_pfcp_server_stop_msg_timer (msg); msg->timer = upf_pfcp_server_start_timer (PFCP_SERVER_RESPONSE, id, RESPONSE_TIMEOUT); } @@ -1047,11 +1047,31 @@ static void upf_validate_session_timers () #endif -void upf_pfcp_server_stop_timer (u32 handle) +void upf_pfcp_server_stop_msg_timer (pfcp_msg_t * msg) { pfcp_server_main_t *psm = &pfcp_server_main; + u32 id_resp, *exp_id, id_t1; - TW (tw_timer_stop) (&psm->timer, handle); + if (msg->timer == ~0) + return; + + id_t1 = ((0x80 | PFCP_SERVER_T1) << 24) | msg->seq_no; + id_resp = + ((0x80 | PFCP_SERVER_RESPONSE) << 24) | pfcp_msg_get_index (psm, msg); + + /* + * Prevent timer handlers from being invoked in case the expiration + * has already been registered for the current iteration of + * pfcp_process() loop + */ + vec_foreach (exp_id, psm->expired) + { + if (*exp_id == id_t1 || *exp_id == id_resp) + *exp_id = ~0; + } + + TW (tw_timer_stop) (&psm->timer, msg->timer); + msg->timer = ~0; } u32 upf_pfcp_server_start_timer (u8 type, u32 id, u32 seconds) @@ -1114,7 +1134,6 @@ static uword pfcp_server_main_t *psm = &pfcp_server_main; uword event_type, *event_data = 0; upf_main_t *gtm = &upf_main; - u32 *expired = NULL; u32 last_expired; pfcp_msg_t *msg; @@ -1144,8 +1163,8 @@ static uword /* run the timing wheel first, to that the internal base for new and updated timers * is set to now */ - expired = - TW (tw_timer_expire_timers_vec) (&psm->timer, psm->now, expired); + psm->expired = + TW (tw_timer_expire_timers_vec) (&psm->timer, psm->now, psm->expired); switch (event_type) { @@ -1219,20 +1238,27 @@ static uword break; } - vec_sort_with_function (expired, timer_id_cmp); + vec_sort_with_function (psm->expired, timer_id_cmp); last_expired = ~0; - for (int i = 0; i < vec_len (expired); i++) + for (int i = 0; i < vec_len (psm->expired); i++) { - switch (expired[i] >> 24) + /* + * Check if the timer has been stopped while handling other + * timers in this iteration of the upf_process() loop + */ + if (psm->expired[i] == ~0) + continue; + + switch (psm->expired[i] >> 24) { case 0 ... 0x7f: - if (last_expired == expired[i]) + if (last_expired == psm->expired[i]) continue; - last_expired = expired[i]; + last_expired = psm->expired[i]; { - const u32 si = expired[i] & 0x7FFFFFFF; + const u32 si = psm->expired[i] & 0x7FFFFFFF; upf_session_t *sx; if (pool_is_free_index (gtm->sessions, si)) @@ -1245,35 +1271,29 @@ static uword case 0x80 | PFCP_SERVER_HB_TIMER: upf_debug ("PFCP Server Heartbeat Timeout: %u", - expired[i] & 0x00FFFFFF); - upf_server_send_heartbeat (expired[i] & 0x00FFFFFF); + psm->expired[i] & 0x00FFFFFF); + upf_server_send_heartbeat (psm->expired[i] & 0x00FFFFFF); break; case 0x80 | PFCP_SERVER_T1: upf_debug ("PFCP Server T1 Timeout: %u", - expired[i] & 0x00FFFFFF); - request_t1_expired (expired[i] & 0x00FFFFFF); + psm->expired[i] & 0x00FFFFFF); + request_t1_expired (psm->expired[i] & 0x00FFFFFF); break; case 0x80 | PFCP_SERVER_RESPONSE: upf_debug ("PFCP Server Response Timeout: %u", - expired[i] & 0x00FFFFFF); - response_expired (expired[i] & 0x00FFFFFF); + psm->expired[i] & 0x00FFFFFF); + response_expired (psm->expired[i] & 0x00FFFFFF); break; default: - upf_debug ("timeout for unknown id: %u", expired[i] >> 24); + upf_debug ("timeout for unknown id: %u", psm->expired[i] >> 24); break; } } - /* - * Free messages that must be freed after the loop, because this may - * involve stopping timers which already have fired, so they're - * currently collected in the 'expired' vector. If handlers for - * these timers are invoked, they may end up handling messages - * that were freed (put to the msg_pool). - */ + /* Free messages that must be freed after the loop */ /* *INDENT-OFF* */ pool_foreach (msg, psm->msg_pool, @@ -1285,12 +1305,12 @@ static uword hash_unset (psm->request_q, msg->seq_no); mhash_unset (&psm->response_q, msg->request_key, NULL); - upf_pfcp_server_stop_timer (msg->timer); + upf_pfcp_server_stop_msg_timer (msg); pfcp_msg_pool_put (psm, msg); })); /* *INDENT-ON* */ - vec_reset_length (expired); + vec_reset_length (psm->expired); vec_reset_length (event_data); hash_free (psm->free_msgs_by_node); hash_free (psm->free_msgs_by_sidx); diff --git a/upf/upf_pfcp_server.h b/upf/upf_pfcp_server.h index e9140be..9345a2c 100644 --- a/upf/upf_pfcp_server.h +++ b/upf/upf_pfcp_server.h @@ -108,6 +108,7 @@ typedef struct uword *free_msgs_by_node; uword *free_msgs_by_sidx; + u32 *expired; } pfcp_server_main_t; typedef struct @@ -135,7 +136,7 @@ void upf_pfcp_session_start_stop_urr_time (u32 si, urr_time_t * t, u8 start_it); u32 upf_pfcp_server_start_timer (u8 type, u32 id, u32 seconds); -void upf_pfcp_server_stop_timer (u32 handle); +void upf_pfcp_server_stop_msg_timer (pfcp_msg_t * msg); void upf_pfcp_server_deferred_free_msgs_by_node (u32 node); void upf_pfcp_server_deferred_free_msgs_by_sidx (u32 sidx);