Implement userspace notifications

Existing mechanism of userspace notifications (by erroring out pending read requests) is not flexible enough to be used by multipeer. This adds a new OVPN_IOCTL_NOTIFY_EVENT ioctl. When request arrives, we check if there are pending notifications. If yes, then we complete request, writing notification command, peer-id and peer delete reason (if applicable). If there are no pending notifications, request is queued. When notification occurs (such as peer keepalive timeout), we check if there is a pending requests in the queue (see above). If yes, we complete request with notification details. If there are no pending requests, we add notification event to a queue. Events queue is implemented with a C++ class and a kernel linked lists. The queue is a member of device context. Since there is no C++ runtime and constructors are not called for context members, we have to use a separate method for initialization. GitHub: #87 Signed-off-by: Lev Stipakov <[email protected]>
OpenVPN · Oct 22, 2024 · d1c9a6b · d1c9a6b
1 parent 5a38804
commit d1c9a6b
Show file tree

Hide file tree

Showing 7 changed files with 254 additions and 9 deletions.
diff --git a/Driver.cpp b/Driver.cpp
@@ -356,10 +356,18 @@ OvpnStopVPN(_In_ POVPN_DEVICE device)
     WDFREQUEST request;
     while (NT_SUCCESS(WdfIoQueueRetrieveNextRequest(device->PendingReadsQueue, &request))) {
         ULONG_PTR bytesCopied = 0;
-        LOG_INFO("Cancel IO request from manual queue");
+        LOG_INFO("Cancel pending read requests");
         WdfRequestCompleteWithInformation(request, STATUS_CANCELLED, bytesCopied);
     }
 
+    while (NT_SUCCESS(WdfIoQueueRetrieveNextRequest(device->PendingNotificationRequestsQueue, &request))) {
+        ULONG_PTR bytesCopied = 0;
+        LOG_INFO("Cancel pending notifications");
+        WdfRequestCompleteWithInformation(request, STATUS_CANCELLED, bytesCopied);
+    }
+
+    device->PendingNotificationsQueue.FlushEvents();
+
     LOG_EXIT();
 
     return STATUS_SUCCESS;
@@ -422,6 +430,37 @@ OvpnMPStartVPN(POVPN_DEVICE device, WDFREQUEST request, ULONG_PTR* bytesReturned
     return status;
 }
 
+NTSTATUS
+OvpnNotifyEvent(POVPN_DEVICE device, WDFREQUEST request, _Out_ ULONG_PTR* bytesReturned) {
+    LOG_ENTER();
+
+    NTSTATUS status = STATUS_SUCCESS;
+
+    *bytesReturned = 0;
+
+    // do we have pending notifications?
+    auto event = device->PendingNotificationsQueue.GetEvent();
+    if (event != nullptr) {
+        OVPN_NOTIFY_EVENT* evt;
+        LOG_IF_NOT_NT_SUCCESS(status = WdfRequestRetrieveOutputBuffer(request, sizeof(OVPN_NOTIFY_EVENT), (PVOID*)&evt, nullptr));
+        if (NT_SUCCESS(status)) {
+            evt->Cmd = evt->Cmd;
+            evt->PeerId = evt->PeerId;
+            evt->DelPeerReason = evt->DelPeerReason;
+            *bytesReturned = sizeof(OVPN_NOTIFY_EVENT);
+        }
+        device->PendingNotificationsQueue.FreeEvent(event);
+    }
+    else {
+        LOG_IF_NOT_NT_SUCCESS(WdfRequestForwardToIoQueue(request, device->PendingNotificationRequestsQueue));
+        status = STATUS_PENDING;
+    }
+
+    LOG_EXIT();
+
+    return status;
+}
+
 EVT_WDF_IO_QUEUE_IO_DEVICE_CONTROL OvpnEvtIoDeviceControl;
 
 _Use_decl_annotations_
@@ -510,6 +549,10 @@ OvpnEvtIoDeviceControl(WDFQUEUE queue, WDFREQUEST request, size_t outputBufferLe
         ExReleaseSpinLockExclusive(&device->SpinLock, kirql);
         break;
 
+    case OVPN_IOCTL_NOTIFY_EVENT:
+        status = OvpnNotifyEvent(device, request, &bytesReturned);
+        break;
+
     default:
         LOG_WARN("Unknown <ioControlCode>", TraceLoggingValue(ioControlCode, "ioControlCode"));
         status = STATUS_INVALID_DEVICE_REQUEST;
@@ -707,12 +750,19 @@ OvpnEvtDeviceAdd(WDFDRIVER wdfDriver, PWDFDEVICE_INIT deviceInit) {
     WDF_IO_QUEUE_CONFIG_INIT(&queueConfig, WdfIoQueueDispatchManual);
     GOTO_IF_NOT_NT_SUCCESS(done, status, WdfIoQueueCreate(wdfDevice, &queueConfig, WDF_NO_OBJECT_ATTRIBUTES, &device->PendingNewPeerQueue));
 
+    // create manual queue which handles userspace notification requests
+    WDF_IO_QUEUE_CONFIG_INIT(&queueConfig, WdfIoQueueDispatchManual);
+    GOTO_IF_NOT_NT_SUCCESS(done, status, WdfIoQueueCreate(wdfDevice, &queueConfig, WDF_NO_OBJECT_ATTRIBUTES, &device->PendingNotificationRequestsQueue));
+
     GOTO_IF_NOT_NT_SUCCESS(done, status, OvpnTxBufferPoolCreate(&device->TxBufferPool, device));
     GOTO_IF_NOT_NT_SUCCESS(done, status, OvpnRxBufferPoolCreate(&device->RxBufferPool));
 
     GOTO_IF_NOT_NT_SUCCESS(done, status, OvpnBufferQueueCreate(&device->ControlRxBufferQueue));
     GOTO_IF_NOT_NT_SUCCESS(done, status, OvpnBufferQueueCreate(&device->DataRxBufferQueue));
 
+    // constructors are not called for the members of WDF object context, so we use Init() method
+    device->PendingNotificationsQueue.Init();
+
     GOTO_IF_NOT_NT_SUCCESS(done, status, OvpnCryptoInitAlgHandles(&device->AesAlgHandle, &device->ChachaAlgHandle));
 
     // Initialize peers tables

diff --git a/Driver.h b/Driver.h
@@ -31,6 +31,7 @@
 #include "adapter.h"
 #include "bufferpool.h"
 #include "crypto.h"
+#include "notifyqueue.h"
 #include "socket.h"
 #include "uapi\ovpn-dco.h"
 
@@ -60,6 +61,7 @@ struct OVPN_DEVICE {
 
     WDFQUEUE PendingReadsQueue;
     WDFQUEUE PendingWritesQueue;
+    WDFQUEUE PendingNotificationRequestsQueue;
 
     // NEW_PEER request may be enqueued here if TCP connect doesn't finish immediatelly
     WDFQUEUE PendingNewPeerQueue;
@@ -76,6 +78,9 @@ struct OVPN_DEVICE {
     // buffer pool for encrypted data channel and control channel packets to be sent
     OVPN_TX_BUFFER_POOL TxBufferPool;
 
+    // queue to store pending userspace notifications
+    NotifyQueue PendingNotificationsQueue;
+
     OVPN_STATS Stats;
 
     BCRYPT_ALG_HANDLE AesAlgHandle;

diff --git a/notifyqueue.cpp b/notifyqueue.cpp
@@ -0,0 +1,86 @@
+/*
+ *  ovpn-dco-win OpenVPN protocol accelerator for Windows
+ *
+ *  Copyright (C) 2024- OpenVPN Inc <[email protected]>
+ *
+ *  Author:	Lev Stipakov <[email protected]>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License version 2
+ *  as published by the Free Software Foundation.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with this program; if not, write to the Free Software Foundation, Inc.,
+ *  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#include "notifyqueue.h"
+
+#include "trace.h"
+
+VOID
+NotifyQueue::Init()
+{
+    LOG_ENTER();
+
+    InitializeListHead(&Head);
+    KeInitializeSpinLock(&Lock);
+
+    LOG_EXIT();
+}
+
+NTSTATUS
+NotifyQueue::AddEvent(OVPN_NOTIFY_CMD cmd, int peerId, OVPN_DEL_PEER_REASON delPeerReason)
+{
+    NotifyEvent* event = (NotifyEvent*)ExAllocatePool2(POOL_FLAG_NON_PAGED, sizeof(NotifyEvent), 'ovpn');
+    if (!event) {
+        return STATUS_MEMORY_NOT_ALLOCATED;
+    }
+
+    RtlZeroMemory(event, sizeof(NotifyEvent));
+
+    event->Cmd = cmd;
+    event->PeerId = peerId;
+    event->DelPeerReason = delPeerReason;
+
+    ExInterlockedInsertTailList(&Head, &event->ListEntry, &Lock);
+
+    return STATUS_SUCCESS;
+}
+
+NotifyEvent*
+NotifyQueue::GetEvent()
+{
+    PLIST_ENTRY entry = ExInterlockedRemoveHeadList(&Head, &Lock);
+    if (entry == nullptr) {
+        return nullptr;
+    }
+
+    return CONTAINING_RECORD(entry, NotifyEvent, ListEntry);
+}
+
+VOID
+NotifyQueue::FreeEvent(NotifyEvent* event)
+{
+    if (event != nullptr) {
+        ExFreePoolWithTag(event, 'ovpn');
+    }
+}
+
+VOID
+NotifyQueue::FlushEvents()
+{
+    LOG_ENTER();
+
+    NotifyEvent* event = nullptr;
+    while ((event = GetEvent()) != nullptr) {
+        FreeEvent(event);
+    }
+
+    LOG_EXIT();
+}
diff --git a/notifyqueue.h b/notifyqueue.h
@@ -0,0 +1,53 @@
+/*
+ *  ovpn-dco-win OpenVPN protocol accelerator for Windows
+ *
+ *  Copyright (C) 2024- OpenVPN Inc <[email protected]>
+ *
+ *  Author:	Lev Stipakov <[email protected]>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License version 2
+ *  as published by the Free Software Foundation.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with this program; if not, write to the Free Software Foundation, Inc.,
+ *  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#pragma once
+
+#include <ntddk.h>
+
+#include "uapi/ovpn-dco.h"
+
+struct NotifyEvent {
+    LIST_ENTRY ListEntry;
+
+    OVPN_NOTIFY_CMD Cmd;
+    int PeerId;
+    OVPN_DEL_PEER_REASON DelPeerReason;
+};
+
+class NotifyQueue {
+private:
+    LIST_ENTRY Head;
+    KSPIN_LOCK Lock;
+
+public:
+    NotifyQueue() = delete;
+
+    VOID Init();
+
+    NTSTATUS AddEvent(OVPN_NOTIFY_CMD cmd, int peerId, OVPN_DEL_PEER_REASON delPeerReason=OVPN_DEL_PEER_REASON_EXPIRED);
+
+    NotifyEvent* GetEvent();
+
+    VOID FreeEvent(NotifyEvent* event);
+
+    VOID FlushEvents();
+};
diff --git a/ovpn-dco-win.vcxproj b/ovpn-dco-win.vcxproj
@@ -73,6 +73,7 @@
     <ClCompile Include="crypto.cpp" />
     <ClCompile Include="driver.cpp" />
     <ClCompile Include="mss.cpp" />
+    <ClCompile Include="notifyqueue.cpp" />
     <ClCompile Include="peer.cpp" />
     <ClCompile Include="pktid.cpp" />
     <ClCompile Include="rxqueue.cpp" />
@@ -88,6 +89,7 @@
     <ClInclude Include="driver.h" />
     <ClInclude Include="mss.h" />
     <ClInclude Include="netringiterator.h" />
+    <ClInclude Include="notifyqueue.h" />
     <ClInclude Include="peer.h" />
     <ClInclude Include="pktid.h" />
     <ClInclude Include="rxqueue.h" />

diff --git a/timer.cpp b/timer.cpp
@@ -107,18 +107,46 @@ static BOOLEAN OvpnTimerRecv(WDFTIMER timer)
 {
     POVPN_DEVICE device = OvpnGetDeviceContext(WdfTimerGetParentObject(timer));
 
+    POVPN_PEER_TIMER_CONTEXT timerCtx = OvpnGetPeerTimerContext(timer);
+    auto peerId = timerCtx->Peer->PeerId;
+    LOG_INFO("Keepalive timeout", TraceLoggingValue(peerId, "peer-id"));
+
     WDFREQUEST request;
-    NTSTATUS status = WdfIoQueueRetrieveNextRequest(device->PendingReadsQueue, &request);
-    if (!NT_SUCCESS(status)) {
-        LOG_WARN("No pending request for keepalive timeout notification");
-        return FALSE;
-    }
-    else {
-        LOG_INFO("Notify userspace about keepalive timeout");
+    NTSTATUS status = STATUS_SUCCESS;
+
+    if (device->Mode == OVPN_MODE_P2P) {
+        status = WdfIoQueueRetrieveNextRequest(device->PendingReadsQueue, &request);
+        if (!NT_SUCCESS(status)) {
+            LOG_WARN("No pending request for keepalive timeout notification");
+            return FALSE;
+        }
+
         ULONG_PTR bytesSent = 0;
         WdfRequestCompleteWithInformation(request, STATUS_CONNECTION_DISCONNECTED, bytesSent);
-        return TRUE;
     }
+    else {
+        status = WdfIoQueueRetrieveNextRequest(device->PendingNotificationRequestsQueue, &request);
+        if (!NT_SUCCESS(status)) {
+            LOG_WARN("Adding keepalive timeout notification to the queue");
+            return NT_SUCCESS(device->PendingNotificationsQueue.AddEvent(OVPN_NOTIFY_DEL_PEER, peerId, OVPN_DEL_PEER_REASON_EXPIRED));
+        }
+        else {
+            OVPN_NOTIFY_EVENT *evt;
+            ULONG_PTR bytesSent = 0;
+            LOG_IF_NOT_NT_SUCCESS(status = WdfRequestRetrieveOutputBuffer(request, sizeof(OVPN_NOTIFY_EVENT), (PVOID*)&evt, nullptr));
+            if (NT_SUCCESS(status)) {
+                evt->Cmd = OVPN_NOTIFY_DEL_PEER;
+                evt->PeerId = peerId;
+                evt->DelPeerReason = OVPN_DEL_PEER_REASON_EXPIRED;
+                bytesSent = sizeof(OVPN_NOTIFY_EVENT);
+            }
+            WdfRequestCompleteWithInformation(request, status, bytesSent);
+
+            // TODO: remove peer
+        }
+    }
+
+    return NT_SUCCESS(status);
 }
 
 _Use_decl_annotations_

diff --git a/uapi/ovpn-dco.h b/uapi/ovpn-dco.h
@@ -154,6 +154,25 @@ typedef struct _OVPN_MP_START_VPN {
     } ListenAddress;
 } OVPN_MP_START_VPN, * POVPN_MP_START_VPN;
 
+typedef enum {
+    OVPN_NOTIFY_DEL_PEER,
+    OVPN_NOTIFY_ROTATE_KEY
+} OVPN_NOTIFY_CMD;
+
+typedef enum {
+    OVPN_DEL_PEER_REASON_TEARDOWN,
+    OVPN_DEL_PEER_REASON_USERSPACE,
+    OVPN_DEL_PEER_REASON_EXPIRED,
+    OVPN_DEL_PEER_REASON_TRANSPORT_ERROR,
+    OVPN_DEL_PEER_REASON_TRANSPORT_DISCONNECT
+} OVPN_DEL_PEER_REASON;
+
+typedef struct _OVPN_NOTIFY_EVENT {
+    OVPN_NOTIFY_CMD Cmd;
+    int PeerId;
+    OVPN_DEL_PEER_REASON DelPeerReason;
+} OVPN_NOTIFY_EVENT, * POVPN_NOTIFY_EVENT;
+
 #define OVPN_IOCTL_NEW_PEER     CTL_CODE(FILE_DEVICE_UNKNOWN, 1, METHOD_BUFFERED, FILE_ANY_ACCESS)
 #define OVPN_IOCTL_GET_STATS    CTL_CODE(FILE_DEVICE_UNKNOWN, 2, METHOD_BUFFERED, FILE_ANY_ACCESS)
 #define OVPN_IOCTL_NEW_KEY      CTL_CODE(FILE_DEVICE_UNKNOWN, 3, METHOD_BUFFERED, FILE_ANY_ACCESS)
@@ -168,3 +187,5 @@ typedef struct _OVPN_MP_START_VPN {
 #define OVPN_IOCTL_MP_START_VPN   CTL_CODE(FILE_DEVICE_UNKNOWN, 11, METHOD_BUFFERED, FILE_ANY_ACCESS)
 #define OVPN_IOCTL_MP_NEW_PEER    CTL_CODE(FILE_DEVICE_UNKNOWN, 12, METHOD_BUFFERED, FILE_ANY_ACCESS)
 #define OVPN_IOCTL_MP_SET_PEER    CTL_CODE(FILE_DEVICE_UNKNOWN, 13, METHOD_BUFFERED, FILE_ANY_ACCESS)
+
+#define OVPN_IOCTL_NOTIFY_EVENT   CTL_CODE(FILE_DEVICE_UNKNOWN, 14, METHOD_BUFFERED, FILE_ANY_ACCESS)