Skip to content

Commit

Permalink
mod_discord: Add self health check to prevent relay stall.
Browse files Browse the repository at this point in the history 
Occasionally, it seems that the underlying libdiscord library
will stop receiving messages from Discord (sending messages
still works). This results in the relay only working in one
direction, with IRC messages relaying to Discord, but not
vice versa. This has happened several times and is usually
remedied by someone noticing eventually and manually reloading
mod_discord to start everything fresh.

Since we can detect this, until the underlying bug is fixed,
this adds the ability to periodically send test posts to a
test channel and trigger a full module reload if a test post
is not echoed back.
  • Loading branch information
@InterLinked1
InterLinked1 committed Dec 23, 2023
1 parent 5eedc9c commit eec0c75
Show file tree
Hide file tree
Showing 2 changed files with 155 additions and 1 deletion.
9 changes: 9 additions & 0 deletions configs/mod_discord.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,15 @@
; Some settings can only be provided to concord through the config file.
; If a concord config file is provided, the token field here will be ignored (you must specify it in the concord config file).
;exposemembers=yes ; Whether or not to expose Discord channel members in NAMES, WHO, and WHOIS replies on IRC. Default is 'yes'.
;echochannel=912345678907654321,test ; Comma-separated guild ID, channel name, used for testing liveness.
; The guild must have at least one relay channel, but the channel does not have to be configured for a relay,
; and since the channel will be posted to automatically, it should be dedicated exclusively to this testing.
; This is a workaround for a KNOWN BUG. libdiscord periodically will fail to receive messages *from* Discord,
; but messages from IRC can still be relayed to Discord.
; To detect this, you can configure the module to periodically send test messages to a test channel.
; If the module fails to receive an echo of the test post, it will conclude something has broken
; and automatically reload the module, minimizing relay downtime.
; This option may be removed in the future if the bug this works around is fixed.

; Define one or more channel mappings, each of which provides a 1:1 relay between the specified channels:
; You must set up a mapping for reach relay you want to configure.
Expand Down
147 changes: 146 additions & 1 deletion modules/mod_discord.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,8 @@
#include "include/startup.h"
#include "include/cli.h"

#define RELAY_RCV_EVENTUALLY_FAILS

#include "include/net_irc.h"

static int discord_ready = 0;
Expand All @@ -46,6 +48,24 @@ static int expose_members = 1;
static char token[84];
static char configfile[84];

#ifdef RELAY_RCV_EVENTUALLY_FAILS
static pthread_t monitor_thread = 0;
static char echochannel[84] = "";
static u64snowflake echoguildid = 0;
static u64snowflake echochanid = 0;

enum monitor_status {
MONITOR_IDLE = 0, /* Idle */
MONITOR_WAITING = 1, /* Sent test ping, waiting for echo ("pong") */
MONITOR_RECEIVED = 2, /* Got echo */
};

static enum monitor_status monitor_status = MONITOR_IDLE;
static u64snowflake monitor_msgid = 0;

#define PING_TEST_MESSAGE "This is an autogenerated ping by LBBS for liveness check"
#endif

/* Note: only a single client is supported (multiple guilds and multiple channels within the guild are supported) */

struct u64snowflake_entry {
Expand Down Expand Up @@ -657,6 +677,21 @@ static void fetch_channels(struct discord *client, u64snowflake guild_id, u64sno
CCORDcode code;
struct discord_ret_channels ret = { .sync = &channels };

#ifdef RELAY_RCV_EVENTUALLY_FAILS
char *echochan;

if (!echoguildid && !s_strlen_zero(echochannel)) {
char *echoguild;
echochan = echochannel;
echoguild = strsep(&echochan, ",");
if (strlen_zero(echochan)) {
bbs_error("echochannel setting requires guild,channel\n");
} else {
echoguildid = (unsigned long) atol(echoguild);
}
}
#endif

code = discord_get_guild_channels(client, guild_id, &ret);
if (code != CCORD_OK) {
bbs_error("Couldn't fetch channels from guild %lu: %s\n", guild_id, discord_strerror(code, client));
Expand All @@ -668,6 +703,15 @@ static void fetch_channels(struct discord *client, u64snowflake guild_id, u64sno
struct discord_channel *channel = &channels.array[i];
/* channel->member_count is always 0 and channel->recipients is NULL */

#ifdef RELAY_RCV_EVENTUALLY_FAILS
if (echoguildid && echoguildid == guild_id) {
if (!strcmp(echochan, channel->name)) {
echochanid = channel->id;
bbs_debug(1, "Channel %lu/%lu configured as echo channel\n", echoguildid, echochanid);
}
}
#endif

RWLIST_TRAVERSE(&mappings, cp, entry) {
if (guild_id != cp->guild_id) {
continue;
Expand Down Expand Up @@ -1161,7 +1205,24 @@ static void on_message_create(struct discord *client, const struct discord_messa
* back and forth... similar to how NOTICE messages should not be autoresponded
* to on IRC (unlike PRIVMSG), bot messages should not be responded to. */
if (event->author->bot) {
bbs_debug(3, "Ignoring message from bot: %s\n", event->content);
#ifdef RELAY_RCV_EVENTUALLY_FAILS
if (monitor_status == MONITOR_WAITING) {
/* The whole point of the monitor is to regularly
* generate traffic so we can test if reception works.
* For testing purposes, it doesn't actually matter if
* this is the same message we sent, or something else.
* Receiving anything constitutes "success". */
monitor_status = MONITOR_RECEIVED;
/* If, however, it is the same message, don't even bother
* emitting the debug message below. */
if (!strlen_zero(event->content) && !strcmp(event->content, PING_TEST_MESSAGE)) {
monitor_msgid = event->id;
return;
}
}
#endif

bbs_debug(3, "Ignoring message %lu from bot: %s\n", event->id, event->content);
return;
}

Expand Down Expand Up @@ -1266,6 +1327,11 @@ static int load_config(void)

res |= !bbs_config_val_set_str(cfg, "discord", "token", token, sizeof(token));
res |= !bbs_config_val_set_str(cfg, "discord", "concordconfig", configfile, sizeof(configfile));

#ifdef RELAY_RCV_EVENTUALLY_FAILS
bbs_config_val_set_str(cfg, "discord", "echochannel", echochannel, sizeof(echochannel));
#endif

if (!res) {
bbs_error("Missing token in mod_discord.conf, and no JSON config specified, declining to load\n");
return -1; /* Things won't work without the token */
Expand Down Expand Up @@ -1325,6 +1391,73 @@ static int load_config(void)
return 0;
}

#ifdef RELAY_RCV_EVENTUALLY_FAILS
static void generate_test_ping(void)
{
char msg[64] = PING_TEST_MESSAGE;
struct discord_create_message params = {
.content = msg,
.message_reference = &(struct discord_message_reference) {
.message_id = 0,
.channel_id = echochanid,
.guild_id = echoguildid,
.fail_if_not_exists = false, /* Send as a normal message, not an in-thread reply */
},
.components = NULL,
};
monitor_status = MONITOR_WAITING;
discord_create_message(discord_client, echochanid, &params, NULL);
}

static void *monitor_relay(void *unused)
{
UNUSED(unused);

while (!discord_ready) {
usleep(5000000);
}

for (;;) {
bbs_pthread_disable_cancel();
generate_test_ping();
bbs_pthread_enable_cancel();

/* 2 seconds ought to be plenty of time */
usleep(2000000);

/* Check if the post echoed */
bbs_pthread_disable_cancel();
if (monitor_status == MONITOR_RECEIVED) {
/* Be nice and delete the test message we created.
* It serves no further useful purpose. */
if (monitor_msgid) {
char reason[2] = "";
struct discord_delete_message params = {
.reason = reason,
};
discord_delete_message(discord_client, echochanid, monitor_msgid, &params, NULL);
} else {
bbs_warning("Got message, but received message ID unavailable?\n");
}
monitor_msgid = 0;
monitor_status = MONITOR_IDLE;
bbs_pthread_enable_cancel();
} else {
bbs_error("Failed to receive echo of test post from Discord... forcing module reload\n");
/* Completely unload and load the module,
* to ensure libdiscord gets a fresh start. */
bbs_request_module_unload("mod_discord", 1);
bbs_pthread_enable_cancel();
break;
}

sleep(600); /* Check every 10 minutes */
}

return NULL;
}
#endif

static int start_discord_relay(void)
{
discord_add_intents(discord_client, DISCORD_GATEWAY_MESSAGE_CONTENT | DISCORD_GATEWAY_GUILD_MESSAGES | DISCORD_GATEWAY_GUILD_PRESENCES | DISCORD_GATEWAY_GUILDS | DISCORD_GATEWAY_GUILD_MEMBERS | DISCORD_GATEWAY_DIRECT_MESSAGES | DISCORD_GATEWAY_PRESENCE_UPDATE);
Expand All @@ -1349,7 +1482,15 @@ static int start_discord_relay(void)
ccord_global_cleanup();
return -1;
}

irc_relay_register(discord_send, nicklist, privmsg);

#ifdef RELAY_RCV_EVENTUALLY_FAILS
if (!s_strlen_zero(echochannel)) {
bbs_pthread_create(&monitor_thread, NULL, monitor_relay, NULL);
}
#endif

return 0;
}

Expand Down Expand Up @@ -1381,6 +1522,10 @@ static int unload_module(void)
bbs_cli_unregister_multiple(cli_commands_discord);
irc_relay_unregister(discord_send);

if (monitor_thread) {
bbs_pthread_cancel_kill(monitor_thread);
}

ccord_shutdown_async();
bbs_debug(3, "Waiting for Discord thread to exit...\n"); /* This may take a moment */
bbs_pthread_join(discord_thread, NULL);
Expand Down

0 comments on commit eec0c75

Please sign in to comment.