diff --git a/.gitignore b/.gitignore index 1bf5102..3e36220 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ node_modules dist +media breadbot.db .env tools/profanity_filter/bin/Words.json diff --git a/src/breadbot.ts b/src/breadbot.ts index add93b4..c5ffe09 100644 --- a/src/breadbot.ts +++ b/src/breadbot.ts @@ -21,6 +21,7 @@ import { DBCall } from "./utilties/storage/entities/DBCall" import { DBCallTranscriptions } from "./utilties/storage/entities/DBCallTranscriptions" import { DBCallUsers } from "./utilties/storage/entities/DBCallUsers" import { DBMessageRegex } from "./utilties/storage/entities/DBMessageRegex" +import { DBMessageRegexMatches } from "./utilties/storage/entities/DBMessageRegexMatches" console.log(__dirname + path.sep + "utilities" + path.sep + "storage" + path.sep + "entities" + path.sep + "*.ts") @@ -38,10 +39,11 @@ export const dataSource = new DataSource({ DBCall, DBCallTranscriptions, DBCallUsers, - DBMessageRegex + DBMessageRegex, + DBMessageRegexMatches ], synchronize: true, - logging: true + logging: false }) export const client : Client = new Client({ diff --git a/src/config.ts b/src/config.ts index 95ecc54..b23f1bc 100644 --- a/src/config.ts +++ b/src/config.ts @@ -2,9 +2,9 @@ import dotenv from "dotenv" dotenv.config() -const { DISCORD_TOKEN, DISCORD_CLIENT_ID, DB_MODE, MEDIA_VOICE_FOLDER } = process.env +const { DISCORD_TOKEN, DISCORD_CLIENT_ID, DB_MODE, MEDIA_VOICE_FOLDER, MEDIA_ATTACHMENT_FOLDER } = process.env -if (!DISCORD_TOKEN || !DISCORD_CLIENT_ID || !DB_MODE || !MEDIA_VOICE_FOLDER) { +if (!DISCORD_TOKEN || !DISCORD_CLIENT_ID || !DB_MODE || !MEDIA_VOICE_FOLDER || !MEDIA_ATTACHMENT_FOLDER) { throw new Error("Missing environment variables") } @@ -12,5 +12,6 @@ export const config = { DISCORD_TOKEN, DISCORD_CLIENT_ID, DB_MODE, - MEDIA_VOICE_FOLDER + MEDIA_VOICE_FOLDER, + MEDIA_ATTACHMENT_FOLDER } \ No newline at end of file diff --git a/src/utilties/discord/messages.ts b/src/utilties/discord/messages.ts index 629dcd0..6bc44a3 100644 --- a/src/utilties/discord/messages.ts +++ b/src/utilties/discord/messages.ts @@ -68,6 +68,8 @@ export async function updateMessageContentHistory(messageDB: Repository { diff --git a/src/utilties/events/messages.ts b/src/utilties/events/messages.ts index f0fc219..e675398 100644 --- a/src/utilties/events/messages.ts +++ b/src/utilties/events/messages.ts @@ -8,6 +8,11 @@ import { DBMessageContentChanges } from "../storage/entities/DBMessageContentCha import { DBMessageAttachments } from "../storage/entities/DBMessageAttachment"; import { DBChannel } from "../storage/entities/DBChannel"; import { DBUser } from "../storage/entities/DBUser"; +import { mkdirSync, createWriteStream } from "fs"; +import { config } from "../../config"; +import path from "path"; +import { Readable } from "stream" +import { finished } from "stream/promises"; export function setupMessageCapture(client: Client, channelDB: Repository, @@ -17,16 +22,57 @@ export function setupMessageCapture(client: Client, maDB: Repository ) { client.on(Events.MessageCreate, async (message) => { + console.log("MESSAGE CREATE") await processMessageCreate(channelDB, userDB, messageDB, maDB, message) }) client.on(Events.MessageUpdate, async (oldMessage, newMessage) => { + console.log("MESSAGE EDITED") + console.log(`Old Message ID: ${oldMessage.id}`) + console.log(`New Message ID: ${newMessage.id}`) await processMessageModify(messageDB, mccDB, maDB, newMessage) }) client.on(Events.MessageDelete, async (deletedMessage) => { await processMessageDeleted(messageDB, deletedMessage) }) + + setInterval(async () => { + console.log("STARTING DOWNLOAD CYCLE") + mkdirSync(config.MEDIA_ATTACHMENT_FOLDER, {recursive: true}) + + let attachmentsToProcess: DBMessageAttachments[] | null = await maDB.find({ + relations: { + message: true + }, + where: { + attachment_downloaded: false + } + }) + + if (attachmentsToProcess != null) { + attachmentsToProcess.forEach(async (attachment: DBMessageAttachments) => { + + mkdirSync(config.MEDIA_ATTACHMENT_FOLDER + path.sep + attachment.message.message_snowflake, {recursive: true}) + + const response = await fetch(attachment.attachment_url) + + if (response.body !== null) { + const fileStream = createWriteStream( + config.MEDIA_ATTACHMENT_FOLDER + path.sep + attachment.message.message_snowflake + + path.sep + attachment.attachment_snowflake + '_' + attachment.attachment_name, + { flags: "wx" } + ) + + await finished(Readable.fromWeb(response.body).pipe(fileStream)) + + attachment.attachment_downloaded = true + + maDB.save(attachment) + } + }) + } + }, 30000) } async function processMessageCreate( diff --git a/src/utilties/storage/entities/DBChannel.ts b/src/utilties/storage/entities/DBChannel.ts index f2fdf02..e3c1038 100644 --- a/src/utilties/storage/entities/DBChannel.ts +++ b/src/utilties/storage/entities/DBChannel.ts @@ -5,7 +5,7 @@ import { DBCall } from "./DBCall"; @Entity() export class DBChannel { - @PrimaryColumn({type: "bigint"}) + @PrimaryColumn({type: "text"}) channel_snowflake: string @ManyToOne(() => DBServer, (server: DBServer) => server.channels, {nullable: true}) diff --git a/src/utilties/storage/entities/DBMessage.ts b/src/utilties/storage/entities/DBMessage.ts index b35ea69..161aafe 100644 --- a/src/utilties/storage/entities/DBMessage.ts +++ b/src/utilties/storage/entities/DBMessage.ts @@ -7,7 +7,7 @@ import { DBMessageRegexMatches } from "./DBMessageRegexMatches"; @Entity() export class DBMessage { - @PrimaryColumn({type: "bigint"}) + @PrimaryColumn({type: "text"}) message_snowflake: string @ManyToOne(() => DBChannel, (channel: DBChannel) => channel.messages) @@ -16,7 +16,7 @@ export class DBMessage { @ManyToOne(() => DBUser, (user: DBUser) => user.messages) user: DBUser - @Column({type: "longtext"}) + @Column() message_content: string @Column({type: "datetime"}) @@ -28,7 +28,7 @@ export class DBMessage { @OneToMany(() => DBMessageContentChanges, (mcc: DBMessageContentChanges) => mcc.message, {nullable: true}) changes: DBMessageContentChanges[] | null - @OneToMany(() => DBMessageAttachments, (ma: DBMessageAttachments) => ma.attachment_snowflake, {nullable: true}) + @OneToMany(() => DBMessageAttachments, (ma: DBMessageAttachments) => ma.attachment_snowflake, {nullable: true, cascade: true}) attachments: DBMessageAttachments[] | null @OneToOne(() => DBMessageRegexMatches, (mrm: DBMessageRegexMatches) => mrm.message, {nullable: true}) diff --git a/src/utilties/storage/entities/DBMessageAttachment.ts b/src/utilties/storage/entities/DBMessageAttachment.ts index 6f3a9b7..22c8e19 100644 --- a/src/utilties/storage/entities/DBMessageAttachment.ts +++ b/src/utilties/storage/entities/DBMessageAttachment.ts @@ -3,7 +3,7 @@ import { DBMessage } from "./DBMessage"; @Entity() export class DBMessageAttachments { - @PrimaryColumn({"type": "bigint"}) + @PrimaryColumn({"type": "text"}) attachment_snowflake: string @ManyToOne(() => DBMessage, (message: DBMessage) => message.attachments) @@ -12,18 +12,18 @@ export class DBMessageAttachments { @Column() attachment_name: string - @Column({nullable: true}) + @Column({type: 'text', nullable: true}) attachment_description: string | null @Column({type: "datetime"}) attachment_timestamp: Date - @Column({nullable: true}) + @Column({type: 'text', nullable: true}) attachment_mime_type: string | null @Column() attachment_url: string @Column({default: false}) - attachment_download: boolean + attachment_downloaded: boolean } \ No newline at end of file diff --git a/src/utilties/storage/entities/DBMessageContentChanges.ts b/src/utilties/storage/entities/DBMessageContentChanges.ts index 5a1e199..87cd6f7 100644 --- a/src/utilties/storage/entities/DBMessageContentChanges.ts +++ b/src/utilties/storage/entities/DBMessageContentChanges.ts @@ -12,6 +12,6 @@ export class DBMessageContentChanges { @Column({type: "datetime"}) message_change_old_timestamp: Date - @Column({type: "longtext"}) + @Column() message_change_old_content: string } \ No newline at end of file diff --git a/src/utilties/storage/entities/DBRole.ts b/src/utilties/storage/entities/DBRole.ts index 75f439d..85696e5 100644 --- a/src/utilties/storage/entities/DBRole.ts +++ b/src/utilties/storage/entities/DBRole.ts @@ -3,7 +3,7 @@ import { DBServer } from "./DBServer"; @Entity() export class DBRole { - @PrimaryColumn({type: "bigint"}) + @PrimaryColumn({type: "text"}) role_snowflake: string @ManyToOne(() => DBServer, (server: DBServer) => server.roles) diff --git a/src/utilties/storage/entities/DBServer.ts b/src/utilties/storage/entities/DBServer.ts index b32c70c..f00dc6b 100644 --- a/src/utilties/storage/entities/DBServer.ts +++ b/src/utilties/storage/entities/DBServer.ts @@ -5,7 +5,7 @@ import { DBMessageRegex } from "./DBMessageRegex"; @Entity() export class DBServer { - @PrimaryColumn({type: "bigint"}) + @PrimaryColumn({type: "text"}) server_snowflake: string @Column() diff --git a/src/utilties/storage/entities/DBUser.ts b/src/utilties/storage/entities/DBUser.ts index bce4cc3..da1d642 100644 --- a/src/utilties/storage/entities/DBUser.ts +++ b/src/utilties/storage/entities/DBUser.ts @@ -5,7 +5,7 @@ import { DBCallUsers } from "./DBCallUsers"; @Entity() export class DBUser { - @PrimaryColumn({type: "bigint"}) + @PrimaryColumn({type: "text"}) user_snowflake: string @Column()