DB Storage Re-engineering #1

Open
Bradley wants to merge 4 commits from typescript_db_rework into typescript_refactor
20 changed files with 1567 additions and 196 deletions
Showing only changes of commit 34f57b96dc - Show all commits

2
.gitignore vendored
View File

@ -1,4 +1,6 @@
node_modules
dist
breadbot.db
.env
tools/profanity_filter/bin/Words.json
tools/profanity_filter/src/Words.json

1339
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@ -3,25 +3,27 @@
"version": "1.0.0",
"main": "breadbot.ts",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1",
"dev": "tsx watch src/breadbot.ts",
"start": "node dist/breadbot.js",
"build": "tsup src/breadbot.ts --minify"
"watch": "tsc -w",
"start": "tsc && node dist/breadbot.js",
"typeorm": "./node_modules/.bin/typeorm"
},
"author": "",
"license": "ISC",
"description": "",
"dependencies": {
"@discordjs/opus": "^0.9.0",
"@discordjs/voice": "^0.18.0",
"discord.js": "^14.20.0",
"dotenv": "^16.5.0",
"sqlite3": "^5.1.7",
"@discordjs/opus": "^0.9.0",
"@discordjs/voice": "^0.16.0",
"libsodium-wrappers": "^0.7.13",
"node-crc": "1.3.2",
"prism-media": "^2.0.0-alpha.0"
"prism-media": "^2.0.0-alpha.0",
"reflect-metadata": "^0.2.2",
"sqlite3": "^5.1.7",
"typeorm": "^0.3.27"
},
"devDependencies": {
"@types/node": "^24.10.0",
"tsup": "^8.5.0",
"tsx": "^4.20.3",
"typescript": "^5.8.3"

View File

@ -1,8 +1,26 @@
import { CacheType, ChatInputCommandInteraction, Client, Events, GatewayIntentBits, Guild, GuildBasedChannel, Interaction, Role } from "discord.js"
import { utilities } from "./utilties"
import { commands } from "./commands"
import "reflect-metadata"
import { Client, Events, GatewayIntentBits, Guild, GuildBasedChannel, Role } from "discord.js"
import { config } from "./config"
import { SQLCommon } from "./utilties/storage/interfaces"
import { DataSource } from "typeorm"
import { DBServer } from "./utilties/storage/entities/DBServer"
import path from "path"
import { DBChannel } from "./utilties/storage/entities/DBChannel"
import { DBRole } from "./utilties/storage/entities/DBRole"
import { insertGuild } from "./utilties/discord/guilds"
import { insertChannel } from "./utilties/discord/channels"
import { insertRole } from "./utilties/discord/roles"
import { setupRoleCapture } from "./utilties/events/roles"
import { DBUser } from "./utilties/storage/entities/DBUser"
console.log(__dirname + path.sep + "utilities" + path.sep + "storage" + path.sep + "entities" + path.sep + "*.ts")
export const dataSource = new DataSource({
type: "sqlite",
database: "breadbot.db",
entities: [DBServer, DBChannel, DBRole, DBUser],
synchronize: true,
logging: true
})
export const client : Client = new Client({
intents: [
@ -14,6 +32,35 @@ export const client : Client = new Client({
]
})
client.once(Events.ClientReady, async () => {
await dataSource.initialize()
const serverRepo = dataSource.getRepository(DBServer)
const channelRepo = dataSource.getRepository(DBChannel)
const roleRepo = dataSource.getRepository(DBRole)
client.guilds.cache.forEach(async (guild: Guild) => {
const server: DBServer | null = await insertGuild(serverRepo, guild)
if (server != null) {
guild.channels.cache.forEach(async (channel: GuildBasedChannel) => {
await insertChannel(channelRepo, channel)
})
guild.roles.cache.forEach(async (role: Role) => {
await insertRole(roleRepo, role)
})
}
})
setupRoleCapture(client, serverRepo, roleRepo)
console.log("Breadbot is Ready")
})
client.login(config.DISCORD_TOKEN)
/*
export let db: SQLCommon
if (config.DB_MODE == "sqlite") {
@ -82,4 +129,4 @@ setInterval(async () => {
await utilities.breadthread.breadthreadProcessLocks(db, client)
}, 5000)
client.login(config.DISCORD_TOKEN)
client.login(config.DISCORD_TOKEN)*/

View File

@ -1,3 +1,4 @@
// @ts-nocheck
import { ChannelType, ChatInputCommandInteraction, CommandInteraction, MessageFlags, SlashCommandBuilder } from "discord.js";
import { breadthreadEnsureAutoLock, breadthreadRemoveAutoLock } from "../utilties/breadbot/breadthread";
import { db } from "../breadbot";

View File

@ -1,50 +1,37 @@
import { SQLCommon } from "../storage/interfaces";
import { DMChannel, GuildBasedChannel, PartialDMChannel } from "discord.js";
import { SQLResult } from "../storage/enumerations";
import { Repository } from "typeorm";
import { DBChannel } from "../storage/entities/DBChannel";
export async function doesChannelExistByID(db: SQLCommon, channelID: string) : Promise<boolean> {
const queryResult : Object[] = await db.getAllParameterized(
"SELECT * FROM channels WHERE channel_snowflake = ?",
[channelID]
)
return queryResult.length != 0
export async function doesChannelExistByID(db: Repository<DBChannel>, channelID: string) : Promise<boolean> {
return (await db.findOne({"where": {channel_snowflake: channelID}})) != null
}
export async function doesChannelExist(db: SQLCommon, channel : GuildBasedChannel | DMChannel | PartialDMChannel) : Promise<boolean> {
const queryResult : Object[] = await db.getAllParameterized(
"SELECT * FROM channels WHERE channel_snowflake = ?",
[channel.id]
)
return queryResult.length != 0
export async function doesChannelExist(db: Repository<DBChannel>, channel : GuildBasedChannel | DMChannel | PartialDMChannel) : Promise<boolean> {
return await doesChannelExistByID(db, channel.id)
}
export async function insertChannel(db: SQLCommon, channel: GuildBasedChannel | DMChannel | PartialDMChannel) : Promise<SQLResult> {
export async function insertChannel(db: Repository<DBChannel>, channel: GuildBasedChannel | DMChannel | PartialDMChannel) : Promise<DBChannel | null> {
const alreadyExists: boolean = await doesChannelExist(db, channel)
if(alreadyExists) {
return SQLResult.ALREADYEXISTS
return await db.findOne({"where": {channel_snowflake: channel.id}})
}
try {
if (channel.isDMBased()) {
await db.runParameterized(
"INSERT INTO channels VALUES (?, ?, ?, ?, ?, ?)",
[channel.id, null, channel.recipient?.username, channel.isThread(), channel.isDMBased(), channel.isVoiceBased()]
)
} else {
await db.runParameterized(
"INSERT INTO channels VALUES (?, ?, ?, ?, ?, ?)",
[channel.id, channel.guild.id, channel.name, channel.isThread(), channel.isDMBased(), channel.isVoiceBased()]
)
}
const newChannel : DBChannel = await db.create({
channel_snowflake: channel.id,
channel_name: channel.isDMBased() ? channel.recipient?.username : channel.name,
is_dm: channel.isDMBased(),
is_thread: channel.isThread(),
is_voice: channel.isVoiceBased(),
server: channel.isDMBased() ? null : {server_snowflake: channel.guild.id}
})
return SQLResult.CREATED
return await db.save(newChannel)
} catch (err) {
//TODO Winston should handle this
console.log("CHANNEL INSERT ERROR")
console.log(err)
return SQLResult.FAILED
return null
}
}

View File

@ -1,33 +1,29 @@
import { Guild } from "discord.js";
import { SQLCommon } from "../storage/interfaces";
import { SQLResult } from "../storage/enumerations";
import { Repository } from "typeorm";
import { DBServer } from "../storage/entities/DBServer";
export async function doesGuildExist(db: SQLCommon, guild : Guild) : Promise<boolean> {
const queryResult : Object[] = await db.getAllParameterized(
"SELECT * FROM servers WHERE server_snowflake = ?",
[guild.id]
)
return queryResult.length != 0
export async function doesGuildExist(db: Repository<DBServer>, guild : Guild) : Promise<boolean> {
return (await db.findOne({"where": {server_snowflake: guild.id}})) != null
}
export async function insertGuild(db: SQLCommon, guild: Guild) : Promise<SQLResult> {
export async function insertGuild(db: Repository<DBServer>, guild: Guild) : Promise<DBServer | null> {
const alreadyExists: boolean = await doesGuildExist(db, guild)
if (alreadyExists) {
return SQLResult.ALREADYEXISTS
return await db.findOne({"where": {server_snowflake: guild.id}})
}
try {
await db.runParameterized(
"INSERT INTO servers VALUES (?, ?, ?)",
[guild.id, guild.name, guild.description]
)
const server: DBServer = await db.create({
server_snowflake: guild.id,
server_name: guild.name,
server_description: guild.description ?? ""
})
return SQLResult.CREATED
return await db.save(server)
} catch (err) {
console.log("Insert Failed")
//TODO Winston should handle this
console.log(err)
return SQLResult.FAILED
return null
}
}

View File

@ -1,75 +1,79 @@
import { Role } from "discord.js";
import { SQLCommon } from "../storage/interfaces";
import { SQLResult } from "../storage/enumerations";
import { Repository } from "typeorm";
import { DBRole } from "../storage/entities/DBRole";
export async function doesRoleExist(db: SQLCommon, role : Role) : Promise<boolean> {
const queryResult : Object[] = await db.getAllParameterized(
"SELECT * FROM roles WHERE role_snowflake = ?",
[role.id]
)
return queryResult.length != 0
export async function doesRoleExist(db: Repository<DBRole>, role : Role) : Promise<boolean> {
return (await db.findOne({"where": {role_snowflake: role.id}})) != null
}
export async function insertRole(db: SQLCommon, role: Role) : Promise<SQLResult> {
export async function insertRole(db: Repository<DBRole>, role: Role) : Promise<DBRole | null> {
const alreadyExists: boolean = await doesRoleExist(db, role)
if(alreadyExists) {
return SQLResult.ALREADYEXISTS
return await db.findOne({"where": {role_snowflake: role.id}})
}
try {
await db.runParameterized(
"INSERT INTO roles VALUES (?, ?, ?, 0)",
[role.id, role.guild.id, role.name]
)
const newRole : DBRole = await db.create({
role_snowflake: role.id,
server: {server_snowflake: role.guild.id},
role_name: role.name,
is_deleted: false
})
return SQLResult.CREATED
return await db.save(newRole)
} catch (err) {
console.log("ROLE INSERT ERROR")
console.log(err)
return SQLResult.FAILED
return null
}
}
export async function updateRole(db: SQLCommon, role: Role): Promise<SQLResult> {
export async function updateRole(db: Repository<DBRole>, role: Role): Promise<DBRole | null> {
const roleExists: boolean = await doesRoleExist(db, role)
if(!roleExists) {
return SQLResult.FAILED
return null
}
try {
await db.runParameterized(
"UPDATE roles SET role_name = ? WHERE role_snowflake = ?",
[role.name, role.id]
)
const toUpdate: DBRole | null = await db.findOne({"where": {role_snowflake: role.id}})
if(toUpdate != null) {
toUpdate.role_name = role.name
return await db.save(toUpdate)
} else {
return null
}
return SQLResult.UPDATED
} catch (err) {
console.log("ROLE UPDATE FAILED")
console.log(err)
return SQLResult.FAILED
return null
}
}
export async function markRoleDeleted(db: SQLCommon, role: Role) : Promise<SQLResult> {
export async function markRoleDeleted(db: Repository<DBRole>, role: Role) : Promise<DBRole | null> {
const roleExists: boolean = await doesRoleExist(db, role)
if(!roleExists) {
return SQLResult.FAILED
return null
}
try {
await db.runParameterized(
"UPDATE roles SET is_deleted = 1 WHERE role_snowflake = ?",
[role.id]
)
const toUpdate: DBRole | null = await db.findOne({"where": {role_snowflake: role.id}})
return SQLResult.UPDATED
if(toUpdate != null) {
toUpdate.is_deleted = true
return await db.save(toUpdate)
} else {
return null
}
} catch (err) {
console.log("ROLE DELETE FAILED")
console.log(err)
return SQLResult.FAILED
return null
}
}

View File

@ -0,0 +1,20 @@
import { Guild, VoiceBasedChannel } from "discord.js";
import { SQLCommon } from "../storage/interfaces";
export async function breadbotInCall(db: SQLCommon, channel: VoiceBasedChannel) : Promise<boolean> {
const queryResult: Object[] = await db.getAllParameterized(
"SELECT * FROM calls WHERE channel_snowflake = ? AND call_end_time IS NULL",
[channel.id]
)
return queryResult.length != 0
}
export async function getExistingCallID(db: SQLCommon, channel: VoiceBasedChannel) : Promise<Number> {
const queryResult: any[] = await db.getAllParameterized(
"SELECT * FROM calls WHERE channel_snowflake = ? AND call_end_time IS NULL",
[channel.id]
)
return queryResult.length != 0 ? queryResult[0].call_id : -1
}

View File

@ -1,3 +1,4 @@
// @ts-nocheck
import { Client, Events, Message, OmitPartialGroupDMChannel, PartialMessage } from "discord.js";
import { SQLResult } from "../storage/enumerations";
import { SQLCommon } from "../storage/interfaces";

View File

@ -1,23 +1,24 @@
import { Client, Events } from "discord.js";
import { SQLCommon } from "../storage/interfaces";
import { SQLResult } from "../storage/enumerations";
import { insertRole, markRoleDeleted, updateRole } from "../discord/roles";
import { insertGuild } from "../discord/guilds";
import { Repository } from "typeorm";
import { DBServer } from "../storage/entities/DBServer";
import { DBRole } from "../storage/entities/DBRole";
export function setupRoleCapture(client: Client, db: SQLCommon) {
export function setupRoleCapture(client: Client, guildDB: Repository<DBServer>, roleDB: Repository<DBRole>) {
client.on(Events.GuildRoleCreate, async (role) => {
const serverOk: SQLResult = await insertGuild(db, role.guild)
const serverOk: DBServer | null = await insertGuild(guildDB, role.guild)
if (serverOk == SQLResult.ALREADYEXISTS || serverOk == SQLResult.CREATED) {
await insertRole(db, role)
if (serverOk != null) {
await insertRole(roleDB, role)
}
})
client.on(Events.GuildRoleUpdate, async (role) => {
await updateRole(db, role)
await updateRole(roleDB, role)
})
client.on(Events.GuildRoleDelete, async (role) => {
await markRoleDeleted(db, role)
await markRoleDeleted(roleDB, role)
})
}

View File

@ -0,0 +1,19 @@
import { Client, Events, VoiceState } from "discord.js"
import { SQLCommon } from "../storage/interfaces";
export function setupVoice(client: Client, db: SQLCommon) {
client.on(Events.VoiceStateUpdate, (oldState: VoiceState, newState: VoiceState) => {
if(oldState.channel == null && newState.channel != null) {
// TODO Null Type Safety Risk?
if (newState.member?.id == client.user?.id) {
return
}
}
})
}
async function processCallJoin(db: SQLCommon, voiceState: VoiceState) {
}

View File

@ -0,0 +1,27 @@
import { Column, Entity, ManyToOne, OneToMany, PrimaryColumn } from "typeorm";
import { DBServer } from "./DBServer";
import { DBMessage } from "./DBMessage";
@Entity()
export class DBChannel {
@PrimaryColumn({type: "bigint"})
channel_snowflake: string
@ManyToOne(() => DBServer, (server: DBServer) => server.channels, {nullable: true})
server: DBServer | null
@Column()
channel_name: string
@Column()
is_thread: boolean
@Column()
is_dm: boolean
@Column()
is_voice: boolean
@OneToMany(() => DBMessage, (message: DBMessage) => message.channel)
messages: DBMessage[]
}

View File

@ -0,0 +1,28 @@
import { Column, Entity, ManyToOne, OneToMany, PrimaryColumn } from "typeorm";
import { DBChannel } from "./DBChannel";
import { DBUser } from "./DBUser";
import { DBMessageContentChanges } from "./DBMessageContentChanges";
@Entity()
export class DBMessage {
@PrimaryColumn({type: "bigint"})
message_snowflake: string
@ManyToOne(() => DBChannel, (channel: DBChannel) => channel.messages)
channel: DBChannel
@ManyToOne(() => DBUser, (user: DBUser) => user.messages)
user: DBUser
@Column({type: "longtext"})
message_content: string
@Column({type: "datetime"})
message_timestamp: Date
@Column()
message_deleted: boolean
@OneToMany(() => DBMessageContentChanges, (mcc: DBMessageContentChanges) => mcc.message, {nullable: true})
changes: DBMessageContentChanges | null
}

View File

@ -0,0 +1,17 @@
import { Column, Entity, ManyToOne, PrimaryGeneratedColumn } from "typeorm";
import { DBMessage } from "./DBMessage";
@Entity()
export class DBMessageContentChanges {
@PrimaryGeneratedColumn()
message_change_id: number
@ManyToOne(() => DBMessage, (message: DBMessage) => message.changes)
message: DBMessage
@Column({type: "datetime"})
message_change_old_timestamp: Date
@Column({type: "longtext"})
message_change_old_content: string
}

View File

@ -0,0 +1,17 @@
import { Column, Entity, ManyToOne, PrimaryColumn } from "typeorm";
import { DBServer } from "./DBServer";
@Entity()
export class DBRole {
@PrimaryColumn({type: "bigint"})
role_snowflake: string
@ManyToOne(() => DBServer, (server: DBServer) => server.roles)
server: DBServer
@Column()
role_name: string
@Column()
is_deleted: boolean
}

View File

@ -0,0 +1,21 @@
import { Column, Entity, OneToMany, PrimaryColumn } from "typeorm";
import { DBChannel } from "./DBChannel";
import { DBRole } from "./DBRole";
@Entity()
export class DBServer {
@PrimaryColumn({type: "bigint"})
server_snowflake: string
@Column()
server_name: string
@Column()
server_description: string
@OneToMany(() => DBChannel, (channel: DBChannel) => channel.server)
channels: DBChannel[]
@OneToMany(() => DBRole, (role: DBRole) => role.server)
roles: DBRole[]
}

View File

@ -0,0 +1,17 @@
import { Column, Entity, OneToMany, PrimaryColumn } from "typeorm";
import { DBMessage } from "./DBMessage";
@Entity()
export class DBUser {
@PrimaryColumn({type: "bigint"})
user_snowflake: string
@Column()
user_name: string
@Column()
user_displayname: string
@OneToMany(() => DBMessage, (message: DBMessage) => message.user)
messages: DBMessage[]
}

View File

@ -1,14 +1,10 @@
import { SQLCommon } from "./interfaces";
const tables: string[] = [
"CREATE TABLE IF NOT EXISTS servers (server_snowflake bigint NOT NULL PRIMARY KEY,server_name text NOT NULL,server_description mediumtext);",
"CREATE TABLE IF NOT EXISTS channels (channel_snowflake bigint NOT NULL PRIMARY KEY,server_snowflake bigint,channel_name text,is_thread bit NOT NULL,is_dm bit NOT NULL,is_voice bit NOT NULL);",
"CREATE TABLE IF NOT EXISTS users (user_snowflake bigint NOT NULL PRIMARY KEY,user_name text NOT NULL,user_displayname text);",
"CREATE TABLE IF NOT EXISTS messages (message_snowflake bigint NOT NULL PRIMARY KEY,channel_snowflake bigint NOT NULL,user_snowflake bigint NOT NULL,message_content longtext NOT NULL,message_timestamp datetime NOT NULL,message_deleted bit NOT NULL);",
"CREATE TABLE IF NOT EXISTS message_content_changes (message_change_id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,message_snowflake bigint NOT NULL,message_change_old_timestamp datetime NOT NULL,message_change_old_content longtext NOT NULL);",
"CREATE TABLE IF NOT EXISTS message_attachments (attachment_snowflake bigint NOT NULL PRIMARY KEY,message_snowflake bigint NOT NULL,attachment_name text NOT NULL,attachment_description text,attachment_timestamp datetime NOT NULL,attachment_mime_type text,attachment_url text NOT NULL,attachment_downloaded bit NOT NULL);",
"CREATE TABLE IF NOT EXISTS breadthread_autolock (breadthread_autolock_id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,channel_snowflake bigint NOT NULL,inactivity_seconds bigint NOT NULL,locked bit NOT NULL);",
"CREATE TABLE IF NOT EXISTS roles (role_snowflake bigint NOT NULL PRIMARY KEY,server_snowflake bigint NOT NULL,role_name text NOT NULL,is_deleted bit NOT NULL);",
"CREATE TABLE IF NOT EXISTS message_scan_regex_matches (message_scan_regex_matches_id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,message_snowflake bigint NOT NULL,message_regexes_id bigint NOT NULL);",
"CREATE TABLE IF NOT EXISTS message_regex_no_role_check (message_regex_no_role_check_id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,server_snowflake bigint NOT NULL,role_snowflake bigint NOT NULL);",
"CREATE TABLE IF NOT EXISTS message_regexes (message_regexes_id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,server_snowflake bigint NOT NULL,regex text NOT NULL,priority int NOT NULL,severity int NOT NULL);",

View File

@ -1,6 +1,7 @@
{
"compilerOptions": {
"target": "ES2020",
"lib": ["es5", "es6"],
"target": "es6",
"module": "commonjs",
"rootDir": "./src",
"outDir": "./dist",
@ -10,6 +11,10 @@
"forceConsistentCasingInFileNames": true,
"strict": true,
"strictNullChecks": true,
"skipLibCheck": true
"skipLibCheck": true,
"experimentalDecorators": true,
"emitDecoratorMetadata": true,
"strictPropertyInitialization": false,
"sourceMap": true
}
}