Pausing voice efforts to take a look at the database understructure problem...

.gitignore

@@ -1,2 +1,4 @@
 node_modules
 .env
+tools/profanity_filter/bin/Words.json
+tools/profanity_filter/src/Words.json

package.json

@@ -14,7 +14,12 @@
   "dependencies": {
     "discord.js": "^14.20.0",
     "dotenv": "^16.5.0",
-    "sqlite3": "^5.1.7"
+    "sqlite3": "^5.1.7",
+    "@discordjs/opus": "^0.9.0",
+    "@discordjs/voice": "^0.16.0",
+    "libsodium-wrappers": "^0.7.13",
+    "node-crc": "1.3.2",
+    "prism-media": "^2.0.0-alpha.0"
   },
   "devDependencies": {
     "tsup": "^8.5.0",

src/breadbot.ts

@@ -9,7 +9,8 @@ export const client : Client = new Client({
         GatewayIntentBits.Guilds,
         GatewayIntentBits.GuildMessages,
         GatewayIntentBits.GuildMembers,
-        GatewayIntentBits.MessageContent
+        GatewayIntentBits.MessageContent,
+        GatewayIntentBits.GuildVoiceStates
     ]
 })
 

src/utilties/discord/channels.ts

@@ -30,17 +30,16 @@ export async function insertChannel(db: SQLCommon, channel: GuildBasedChannel |
     try {
         if (channel.isDMBased()) {
             await db.runParameterized(
-                "INSERT INTO channels VALUES (?, ?, ?, ?, ?)",
-                [channel.id, null, channel.recipient?.username, channel.isThread(), channel.isDMBased()]
+                "INSERT INTO channels VALUES (?, ?, ?, ?, ?, ?)",
+                [channel.id, null, channel.recipient?.username, channel.isThread(), channel.isDMBased(), channel.isVoiceBased()]
             )
         } else {
             await db.runParameterized(
-                "INSERT INTO channels VALUES (?, ?, ?, ?, ?)",
-                [channel.id, channel.guild.id, channel.name, channel.isThread(), channel.isDMBased()]
+                "INSERT INTO channels VALUES (?, ?, ?, ?, ?, ?)",
+                [channel.id, channel.guild.id, channel.name, channel.isThread(), channel.isDMBased(), channel.isVoiceBased()]
             )
         }
         
-
         return SQLResult.CREATED
     } catch (err) {
         //TODO Winston should handle this

src/utilties/discord/regex_matching.ts

@@ -0,0 +1,16 @@
+import { Guild } from "discord.js";
+import { SQLCommon } from "../storage/interfaces";
+
+export async function getRegexesForGuild(db: SQLCommon, guild: Guild): Promise<any[]> {
+    return db.getAllParameterized(
+        "SELECT * FROM message_regexes WHERE server_snowflake = ?",
+        [guild.id]
+    )
+}
+
+export async function getRoleExclusionSnowflakesForGuild(db: SQLCommon, guild: Guild): Promise<string[]> {
+    return (await db.getAllParameterized(
+        "SELECT role_snowflake FROM message_regex_no_role_check WHERE server_snowflake = ?",
+        [guild.id]
+    )).map((o) => (o as any).role_snowflake) 
+}

src/utilties/events/messages.ts

@@ -7,7 +7,7 @@ import { insertAttachment, insertMessage, markMessageDeleted, updateMessageConte
 
 export function setupMessageCapture(client: Client, db: SQLCommon) {
     client.on(Events.MessageCreate, async (message) => {
-        processMessageCreate(db, message)
+        await processMessageCreate(db, message)
     })
 
     client.on(Events.MessageUpdate, async (oldMessage, newMessage) => {

src/utilties/storage/tables.ts

@@ -2,13 +2,20 @@ import { SQLCommon } from "./interfaces";
 
 const tables: string[] = [
     "CREATE TABLE IF NOT EXISTS servers (server_snowflake bigint NOT NULL PRIMARY KEY,server_name text NOT NULL,server_description mediumtext);",
-    "CREATE TABLE IF NOT EXISTS channels (channel_snowflake bigint NOT NULL PRIMARY KEY,server_snowflake bigint,channel_name text,is_thread bit NOT NULL,is_dm bit NOT NULL);",
+    "CREATE TABLE IF NOT EXISTS channels (channel_snowflake bigint NOT NULL PRIMARY KEY,server_snowflake bigint,channel_name text,is_thread bit NOT NULL,is_dm bit NOT NULL,is_voice bit NOT NULL);",
     "CREATE TABLE IF NOT EXISTS users (user_snowflake bigint NOT NULL PRIMARY KEY,user_name text NOT NULL,user_displayname text);",
     "CREATE TABLE IF NOT EXISTS messages (message_snowflake bigint NOT NULL PRIMARY KEY,channel_snowflake bigint NOT NULL,user_snowflake bigint NOT NULL,message_content longtext NOT NULL,message_timestamp datetime NOT NULL,message_deleted bit NOT NULL);",
     "CREATE TABLE IF NOT EXISTS message_content_changes (message_change_id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,message_snowflake bigint NOT NULL,message_change_old_timestamp datetime NOT NULL,message_change_old_content longtext NOT NULL);",
     "CREATE TABLE IF NOT EXISTS message_attachments (attachment_snowflake bigint NOT NULL PRIMARY KEY,message_snowflake bigint NOT NULL,attachment_name text NOT NULL,attachment_description text,attachment_timestamp datetime NOT NULL,attachment_mime_type text,attachment_url text NOT NULL,attachment_downloaded bit NOT NULL);",
     "CREATE TABLE IF NOT EXISTS breadthread_autolock (breadthread_autolock_id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,channel_snowflake bigint NOT NULL,inactivity_seconds bigint NOT NULL,locked bit NOT NULL);",
-    "CREATE TABLE IF NOT EXISTS roles (role_snowflake bigint NOT NULL PRIMARY KEY,server_snowflake bigint NOT NULL,role_name text NOT NULL,is_deleted bit NOT NULL);"
+    "CREATE TABLE IF NOT EXISTS roles (role_snowflake bigint NOT NULL PRIMARY KEY,server_snowflake bigint NOT NULL,role_name text NOT NULL,is_deleted bit NOT NULL);",
+    "CREATE TABLE IF NOT EXISTS message_scan_regex_matches (message_scan_regex_matches_id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,message_snowflake bigint NOT NULL,message_regexes_id bigint NOT NULL);",
+    "CREATE TABLE IF NOT EXISTS message_regex_no_role_check (message_regex_no_role_check_id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,server_snowflake bigint NOT NULL,role_snowflake bigint NOT NULL);",
+    "CREATE TABLE IF NOT EXISTS message_regexes (message_regexes_id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,server_snowflake bigint NOT NULL,regex text NOT NULL,priority int NOT NULL,severity int NOT NULL);",
+    "CREATE TABLE IF NOT EXISTS message_regex_words (message_regex_words_id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,message_regexes_id bigint,word text NOT NULL);",
+    "CREATE TABLE IF NOT EXISTS calls (call_id bigint NOT NULL PRIMARY KEY AUTOINCREMENT, channel_snowflake bigint NOT NULL, call_start_time datetime NOT NULL, call_end_time datetime DEFAULT NULL, call_consolidated INTEGER DEFAULT 0 CHECK(call_consolidated IN (0, 1)), call_transcribed INTEGER DEFAULT 0 CHECK(call_transcribed IN (0, 1)), call_data_cleaned_up INTEGER DEFAULT 0 CHECK(call_data_cleaned_up IN (0, 1)));",
+    "CREATE TABLE IF NOT EXISTS call_transcriptions (transcription_id bitint NOT NULL PRIMARY KEY AUTOINCREMENT, call_id bigint NOT NULL, user_snowflake bigint NOT NULL, speaking_start_time datetime NOT NULL, text TEXT NOT NULL);",
+    "CREATE TABLE IF NOT EXISTS call_users (call_users_id bigint NOT NULL PRIMARY KEY AUTOINCREMENT, call_id bigint NOT NULL, user_snowflake bigint NOT NULL, call_join_time datetime NOT NULL, call_leave_time datetime DEFAULT NULL);"
 ]
 
 const constraints: string[] = [

tools/profanity_filter/.vscode/launch.json

@@ -0,0 +1,25 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "type": "java",
+            "name": "Current File",
+            "request": "launch",
+            "mainClass": "${file}"
+        },
+        {
+            "type": "java",
+            "name": "RegexGenerator",
+            "request": "launch",
+            "mainClass": "RegexGenerator",
+            "projectName": "ProfanityFilter_ceb3bd68",
+            "args": [
+                "ass",
+                "shit"
+            ]
+        }
+    ]
+}

tools/profanity_filter/.vscode/settings.json

@@ -0,0 +1,7 @@
+{
+    "java.project.sourcePaths": ["src"],
+    "java.project.outputPath": "bin",
+    "java.project.referencedLibraries": [
+        "lib/**/*.jar"
+    ]
+}

tools/profanity_filter/README.md

@@ -0,0 +1,18 @@
+## Getting Started
+
+Welcome to the VS Code Java world. Here is a guideline to help you get started to write Java code in Visual Studio Code.
+
+## Folder Structure
+
+The workspace contains two folders by default, where:
+
+- `src`: the folder to maintain sources
+- `lib`: the folder to maintain dependencies
+
+Meanwhile, the compiled output files will be generated in the `bin` folder by default.
+
+> If you want to customize the folder structure, open `.vscode/settings.json` and update the related settings there.
+
+## Dependency Management
+
+The `JAVA PROJECTS` view allows you to manage your dependencies. More details can be found [here](https://github.com/microsoft/vscode-java-dependency#manage-dependencies).

tools/profanity_filter/src/generators/ExceptionGenerator.java

@@ -0,0 +1,34 @@
+package generators;
+
+public class ExceptionGenerator {
+    public static void main(String[] args) {
+        //position 0 should be the base word, the rest should be exceptions
+        String base = args[0].toLowerCase();
+
+        String prefixes = "(?<!";
+        String regex = RegexGenerator.regexGenerator(base);
+        String suffixes = "(?!";
+
+        for (int i = 1; i < args.length; i++) {
+            args[i] = "$" + args[i] + "$";
+
+            String prefix = args[i].toLowerCase().split(base)[0];
+
+            String suffix = args[i].toLowerCase().split(base)[1];
+            
+            if (!prefix.equals("$")) {
+                prefixes += RegexGenerator.regexGenerator(prefix) + "|";
+            }
+
+            if (!suffix.equals("$")) {
+                suffixes += RegexGenerator.regexGenerator(suffix) + "|";
+            }
+        }
+
+        prefixes = prefixes.substring(0, prefixes.length() - 1);
+        suffixes = suffixes.substring(0, suffixes.length() - 1);
+
+
+        System.out.println(prefixes + ")" + regex + suffixes + ")");
+    }
+}

tools/profanity_filter/src/generators/RegexGenerator.java

@@ -0,0 +1,77 @@
+package generators;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+public class RegexGenerator {
+    private static Map<Character, String> dictionary;
+
+    static {
+        Map<Character, String> map = new HashMap<>();
+        map.put('a', "[a@*]");
+        map.put('b', "([b38]|\\\\|3)");
+        map.put('c', "[c(k]");
+        map.put('d', "[d]");
+        map.put('e', "[e3*]");
+        map.put('f', "([f]|ph)");
+        map.put('g', "[g]");
+        map.put('h', "[h]");
+        map.put('i', "[il1!*]");
+        map.put('j', "[j]");
+        map.put('k', "[kc]");
+        map.put('l', "[li]");
+        map.put('m', "([m]|rn)");
+        map.put('n', "[n]");
+        map.put('o', "[o0pq*]");
+        map.put('p', "[p]");
+        map.put('q', "[q]");
+        map.put('r', "[r]");
+        map.put('s', "[sz5$]");
+        map.put('t', "[t7+]");
+        map.put('u', "[uv*]");
+        map.put('v', "[v]");
+        map.put('w', "([w]|vv)");
+        map.put('x', "[x]");
+        map.put('y', "[y]");
+        map.put('z', "[z]");
+        map.put(' ', "");
+        map.put('1', "([1]|one)");
+        map.put('2', "([2]|two)");
+        map.put('3', "([3]|three)");
+        map.put('4', "([4]|four)");
+        map.put('5', "([5]|five)");
+        map.put('6', "([6]|six)");
+        map.put('7', "([7]|seven)");
+        map.put('8', "([8]|eight)");
+        map.put('9', "([9]|nine)");
+        map.put('0', "([0]|zero)");
+        dictionary = Collections.unmodifiableMap(map);
+    }
+
+    public static void main(String[] args) {
+        for (String s : args) {
+            String regex = "(\\\\b|^)";
+
+            for (char c : s.toLowerCase().toCharArray()) {
+                regex += dictionary.get(c) + "+[\\\\s\\\\n\\\\W]*";
+            }
+
+            regex += "(\\\\b|$)";
+
+            System.out.println(regex); 
+        }
+    }
+
+    public static String regexGenerator(String str) {
+        String regex = "(\\\\b|^)";
+
+        for (char c : str.toLowerCase().toCharArray()) {
+            regex += dictionary.get(c) + "+[\\\\s\\\\n\\\\W]*";
+        }
+
+        regex += "(\\\\b|$)";
+
+        return regex;
+    }
+}