commit e6c26b203d0b9a65d403ef266d8adffb2c115741
parent ad4e0f8842f7eb551e035af9d28b24905ac603bb
Author: MTRNord <mtrnord1@gmail.com>
Date: Sun, 9 Jun 2024 18:22:54 +0200
make sender and room_id filterable and filter edited messages to not cause too many duplicated messages when backfilling
Diffstat:
3 files changed, 93 insertions(+), 28 deletions(-)
diff --git a/src/bot.ts b/src/bot.ts
@@ -2,6 +2,7 @@ import { MatrixClient, RustSdkCryptoStorageProvider, SimpleFsStorageProvider, Ru
import Indexer from "./indexer.js";
import Config from "./config.js";
import { GetRoomMessagesRequest, GetRoomMessagesResponse, Received, RoomEventFilter, Event } from "./matrix.js";
+import fs from "node:fs";
const directions = { forward: "f", reverse: "b" } as const;
// Pending https://github.com/turt2live/matrix-bot-sdk/issues/250
@@ -10,7 +11,7 @@ async function* getRoomEvents(
room: string,
direction: "forward" | "reverse",
filter?: RoomEventFilter
-): AsyncGenerator<Received<Event>[], void, void> {
+): AsyncGenerator<Received<any>[], void, void> {
const path = `/_matrix/client/v3/rooms/${encodeURIComponent(room)}/messages`;
const base: GetRoomMessagesRequest = {
...(direction && { dir: directions[direction] }),
@@ -35,6 +36,61 @@ function cleanContent(content: any) {
return content;
}
+type BackfillState = {
+ rooms: string[];
+};
+
+async function backfill(client: MatrixClient, indexer: Indexer) {
+ // Check if we have a state file
+ const filePath = "./storage/backfillState.json";
+ if (!fs.existsSync(filePath)) {
+ console.info("No backfill state found, writing empty state file");
+ fs.writeFileSync(filePath, "{\"rooms\":[]}");
+ }
+
+ const stateJson = fs.readFileSync(filePath, "utf-8");
+ const state: BackfillState = JSON.parse(stateJson);
+
+ console.info("Indexing existing messages")
+ const joined_rooms = await client.getJoinedRooms();
+
+ // Remvoe rooms we've already indexed.
+ // The already indexed are a list of strings in the rooms object on our state
+ const newRooms = joined_rooms.filter(room => !state.rooms.includes(room));
+
+ // Save the state file
+ fs.writeFileSync(filePath, JSON.stringify(state));
+
+ let editedEvents: string[] = [];
+
+ for (const room of newRooms) {
+ const pages = getRoomEvents(client, room, "reverse", { types: ["m.room.message"] });
+ for await (const page of pages) {
+ for (const event of page) {
+ if (editedEvents.includes(event["event_id"])) {
+ continue;
+ }
+ if (event.content["m.relates_to"]) {
+ if (event.content["m.relates_to"].rel_type === "m.replace") {
+ editedEvents.push(event.content["m.relates_to"].event_id);
+ }
+ }
+
+ if (await client.crypto.isRoomEncrypted(room)) {
+ await client.crypto.onRoomEvent(room, event);
+ } else {
+ const res = await indexer.insert({ id: event["event_id"].replace("$", "").replace(":", "_").replace(".", "_"), sender: event["sender"], content: cleanContent(event["content"]), room_id: room });
+ console.info(`Indexed message:`, res);
+ }
+ }
+ }
+
+ // Add the room to the state and save it
+ state.rooms.push(room);
+ fs.writeFileSync(filePath, JSON.stringify(state));
+ }
+}
+
async function run() {
const storageProvider = new SimpleFsStorageProvider("./storage/bot.json");
@@ -59,22 +115,7 @@ async function run() {
await client.start();
console.info("Bot started!");
- // TODO: Only do this on the first run
- console.info("Indexing existing messages")
- const joined_rooms = await client.getJoinedRooms();
- for (const room of joined_rooms) {
- const pages = getRoomEvents(client, room, "reverse", { types: ["m.room.message"] });
- for await (const page of pages) {
- for (const event of page) {
- if (await client.crypto.isRoomEncrypted(room)) {
- await client.crypto.onRoomEvent(room, event);
- } else {
- const res = await indexer.insert({ id: event["event_id"].replace("$", "").replace(":", "_").replace(".", "_"), sender: event["sender"], content: cleanContent(event["content"]), room_id: room });
- console.info(`Indexed message:`, res);
- }
- }
- }
- }
+ await backfill(client, indexer);
}
diff --git a/src/indexer.ts b/src/indexer.ts
@@ -9,28 +9,52 @@ import fs from 'node:fs';
*
*/
export default class Indexer {
- private client = new MeiliSearch({
+ private readonly client = new MeiliSearch({
host: 'http://localhost:7700',
apiKey: 'aSampleMasterKey'
});
+ private readonly textIndex = this.client.index('text');
- constructor() { }
+ constructor() {
+ }
// TODO: Properly type the data
public async insert(data: any) {
- const index = this.client.index('text');
- return await index.addDocuments([data], { primaryKey: 'id' });
+ return await this.textIndex.addDocuments([data], { primaryKey: 'id' });
}
- public async search(query: string) {
- const index = this.client.index('text');
+ public async search(query: string, room_id?: string, sender?: string) {
+
+ await this.textIndex.updateSearchableAttributes(['content.body', "sender", "content.m.mentions.user_id"]);
+ await this.textIndex.updateFilterableAttributes([
+ 'id',
+ 'sender',
+ 'room_id'
+ ])
+
let results: Hits<Record<string, any>> = [];
let page = 1;
+
+ let filter = "";
+ if (room_id) {
+ filter += `room_id = "${room_id}"`;
+ }
+ if (sender) {
+ if (filter != "") {
+ filter += " AND ";
+ }
+ filter += `sender = "${sender}"`;
+ }
+ let filters;
+ if (filter != "") {
+ filters = [filter]
+ }
+
// Keep searching until we got everything
- let response = await index.search(query, { page: page });
+ let response = await this.textIndex.search(query, { page: page, filter: filters });
results = results.concat(response.hits);
while (page < response.totalPages) {
- response = await index.search(query, { page: page });
+ response = await this.textIndex.search(query, { page: page, filter: filters });
results = results.concat(response.hits);
page++;
}
diff --git a/src/renderer.tsx b/src/renderer.tsx
@@ -60,11 +60,11 @@ const MessageDocument = ({ query, queryResults }: PropsWithChildren<{ query: str
)
}
-export async function renderPDFToDisk(query: string) {
- const queryResults = await indexer.search(query);
+export async function renderPDFToDisk(query: string, room_id?: string, sender?: string) {
+ const queryResults = await indexer.search(query, room_id, sender);
//console.log(`Search results:`, queryResults);
ReactPDF.renderToFile(<MessageDocument query={query} queryResults={queryResults} />, `output.pdf`);
}
-await renderPDFToDisk("test");
+await renderPDFToDisk("test", undefined, '@mtrnord:midnightthoughts.space');
console.log("PDF rendered to disk");
\ No newline at end of file
