A various number of fixes related to ensuring message content is captured, and adding downloading of attachments

.gitignore

@@ -1,5 +1,6 @@
 node_modules
 dist
+media
 breadbot.db
 .env
 tools/profanity_filter/bin/Words.json

src/breadbot.ts

@@ -21,6 +21,7 @@ import { DBCall } from "./utilties/storage/entities/DBCall"
 import { DBCallTranscriptions } from "./utilties/storage/entities/DBCallTranscriptions"
 import { DBCallUsers } from "./utilties/storage/entities/DBCallUsers"
 import { DBMessageRegex } from "./utilties/storage/entities/DBMessageRegex"
+import { DBMessageRegexMatches } from "./utilties/storage/entities/DBMessageRegexMatches"
 
 console.log(__dirname + path.sep + "utilities" + path.sep + "storage" + path.sep + "entities" + path.sep + "*.ts")
 
@@ -38,10 +39,11 @@ export const dataSource = new DataSource({
         DBCall,
         DBCallTranscriptions,
         DBCallUsers,
-        DBMessageRegex
+        DBMessageRegex,
+        DBMessageRegexMatches
     ],
     synchronize: true,
-    logging: true
+    logging: false
 })
 
 export const client : Client = new Client({

src/config.ts

@@ -2,9 +2,9 @@ import dotenv from "dotenv"
 
 dotenv.config()
 
-const { DISCORD_TOKEN, DISCORD_CLIENT_ID, DB_MODE, MEDIA_VOICE_FOLDER } = process.env
+const { DISCORD_TOKEN, DISCORD_CLIENT_ID, DB_MODE, MEDIA_VOICE_FOLDER, MEDIA_ATTACHMENT_FOLDER } = process.env
 
-if (!DISCORD_TOKEN || !DISCORD_CLIENT_ID || !DB_MODE || !MEDIA_VOICE_FOLDER) {
+if (!DISCORD_TOKEN || !DISCORD_CLIENT_ID || !DB_MODE || !MEDIA_VOICE_FOLDER || !MEDIA_ATTACHMENT_FOLDER) {
     throw new Error("Missing environment variables")
 }
 
@@ -12,5 +12,6 @@ export const config = {
     DISCORD_TOKEN,
     DISCORD_CLIENT_ID,
     DB_MODE,
-    MEDIA_VOICE_FOLDER
+    MEDIA_VOICE_FOLDER,
+    MEDIA_ATTACHMENT_FOLDER
 }

src/utilties/discord/messages.ts

@@ -68,6 +68,8 @@ export async function updateMessageContentHistory(messageDB: Repository<DBMessag
         dbMessage.message_content = message.content
         dbMessage.message_timestamp = message.editedAt ?? message.createdAt
 
+        console.log(dbMessage)
+
         // TODO This should really be a transaction
         // TODO Changes to attachments aren't captured
         return await mccDB.save(contentChange).then(async (dbmcc: DBMessageContentChanges) => {

src/utilties/events/messages.ts

@@ -8,6 +8,11 @@ import { DBMessageContentChanges } from "../storage/entities/DBMessageContentCha
 import { DBMessageAttachments } from "../storage/entities/DBMessageAttachment";
 import { DBChannel } from "../storage/entities/DBChannel";
 import { DBUser } from "../storage/entities/DBUser";
+import { mkdirSync, createWriteStream } from "fs";
+import { config } from "../../config";
+import path from "path";
+import { Readable } from "stream"
+import { finished } from "stream/promises";
 
 export function setupMessageCapture(client: Client, 
     channelDB: Repository<DBChannel>,
@@ -17,16 +22,57 @@ export function setupMessageCapture(client: Client,
     maDB: Repository<DBMessageAttachments>
 ) {
     client.on(Events.MessageCreate, async (message) => {
+        console.log("MESSAGE CREATE")
         await processMessageCreate(channelDB, userDB, messageDB, maDB, message)
     })
 
     client.on(Events.MessageUpdate, async (oldMessage, newMessage) => {
+        console.log("MESSAGE EDITED")
+        console.log(`Old Message ID: ${oldMessage.id}`)
+        console.log(`New Message ID: ${newMessage.id}`)
         await processMessageModify(messageDB, mccDB, maDB, newMessage)
     })
 
     client.on(Events.MessageDelete, async (deletedMessage) => {
         await processMessageDeleted(messageDB, deletedMessage)
     })
+
+    setInterval(async () => {
+        console.log("STARTING DOWNLOAD CYCLE")
+        mkdirSync(config.MEDIA_ATTACHMENT_FOLDER, {recursive: true})
+
+        let attachmentsToProcess: DBMessageAttachments[] | null = await maDB.find({
+            relations: {
+                message: true
+            },
+            where: {
+                attachment_downloaded: false
+            }
+        })
+
+        if (attachmentsToProcess != null) {
+            attachmentsToProcess.forEach(async (attachment: DBMessageAttachments) => {
+                
+                mkdirSync(config.MEDIA_ATTACHMENT_FOLDER + path.sep + attachment.message.message_snowflake, {recursive: true})
+
+                const response = await fetch(attachment.attachment_url)
+
+                if (response.body !== null) {
+                    const fileStream = createWriteStream(
+                        config.MEDIA_ATTACHMENT_FOLDER + path.sep + attachment.message.message_snowflake +
+                            path.sep + attachment.attachment_snowflake + '_' + attachment.attachment_name, 
+                        { flags: "wx" }
+                    )
+
+                    await finished(Readable.fromWeb(response.body).pipe(fileStream))
+
+                    attachment.attachment_downloaded = true
+
+                    maDB.save(attachment)
+                }
+            })
+        }
+    }, 30000)
 }
 
 async function processMessageCreate(

src/utilties/storage/entities/DBChannel.ts

@@ -5,7 +5,7 @@ import { DBCall } from "./DBCall";
 
 @Entity()
 export class DBChannel {
-    @PrimaryColumn({type: "bigint"})
+    @PrimaryColumn({type: "text"})
     channel_snowflake: string
 
     @ManyToOne(() => DBServer, (server: DBServer) => server.channels, {nullable: true})

src/utilties/storage/entities/DBMessage.ts

@@ -7,7 +7,7 @@ import { DBMessageRegexMatches } from "./DBMessageRegexMatches";
 
 @Entity()
 export class DBMessage {
-    @PrimaryColumn({type: "bigint"})
+    @PrimaryColumn({type: "text"})
     message_snowflake: string
 
     @ManyToOne(() => DBChannel, (channel: DBChannel) => channel.messages)
@@ -16,7 +16,7 @@ export class DBMessage {
     @ManyToOne(() => DBUser, (user: DBUser) => user.messages)
     user: DBUser
 
-    @Column({type: "longtext"})
+    @Column()
     message_content: string
 
     @Column({type: "datetime"})
@@ -28,7 +28,7 @@ export class DBMessage {
     @OneToMany(() => DBMessageContentChanges, (mcc: DBMessageContentChanges) => mcc.message, {nullable: true})
     changes: DBMessageContentChanges[] | null
 
-    @OneToMany(() => DBMessageAttachments, (ma: DBMessageAttachments) => ma.attachment_snowflake, {nullable: true})
+    @OneToMany(() => DBMessageAttachments, (ma: DBMessageAttachments) => ma.attachment_snowflake, {nullable: true, cascade: true})
     attachments: DBMessageAttachments[] | null
 
     @OneToOne(() => DBMessageRegexMatches, (mrm: DBMessageRegexMatches) => mrm.message, {nullable: true})

src/utilties/storage/entities/DBMessageAttachment.ts

@@ -3,7 +3,7 @@ import { DBMessage } from "./DBMessage";
 
 @Entity()
 export class DBMessageAttachments {
-    @PrimaryColumn({"type": "bigint"})
+    @PrimaryColumn({"type": "text"})
     attachment_snowflake: string
 
     @ManyToOne(() => DBMessage, (message: DBMessage) => message.attachments)
@@ -12,18 +12,18 @@ export class DBMessageAttachments {
     @Column()
     attachment_name: string
 
-    @Column({nullable: true})
+    @Column({type: 'text', nullable: true})
     attachment_description: string | null
 
     @Column({type: "datetime"})
     attachment_timestamp: Date
 
-    @Column({nullable: true})
+    @Column({type: 'text', nullable: true})
     attachment_mime_type: string | null
 
     @Column()
     attachment_url: string
 
     @Column({default: false})
-    attachment_download: boolean
+    attachment_downloaded: boolean
 }

src/utilties/storage/entities/DBMessageContentChanges.ts

@@ -12,6 +12,6 @@ export class DBMessageContentChanges {
     @Column({type: "datetime"})
     message_change_old_timestamp: Date
 
-    @Column({type: "longtext"})
+    @Column()
     message_change_old_content: string
 }

src/utilties/storage/entities/DBRole.ts

@@ -3,7 +3,7 @@ import { DBServer } from "./DBServer";
 
 @Entity()
 export class DBRole {
-    @PrimaryColumn({type: "bigint"})
+    @PrimaryColumn({type: "text"})
     role_snowflake: string
 
     @ManyToOne(() => DBServer, (server: DBServer) => server.roles)

src/utilties/storage/entities/DBServer.ts

@@ -5,7 +5,7 @@ import { DBMessageRegex } from "./DBMessageRegex";
 
 @Entity()
 export class DBServer {
-    @PrimaryColumn({type: "bigint"})
+    @PrimaryColumn({type: "text"})
     server_snowflake: string
 
     @Column()

src/utilties/storage/entities/DBUser.ts

@@ -5,7 +5,7 @@ import { DBCallUsers } from "./DBCallUsers";
 
 @Entity()
 export class DBUser {
-    @PrimaryColumn({type: "bigint"})
+    @PrimaryColumn({type: "text"})
     user_snowflake: string
 
     @Column()