commit cd5df6a05aabb2f33b2027a8ee4243ae97227004
parent 6700402830ee3f6378c973ab912a66ddad3278d9
Author: MTRNord <mtrnord1@gmail.com>
Date: Tue, 15 Oct 2024 01:51:12 +0200
Try to fix various sync issues
Diffstat:
9 files changed, 384 insertions(+), 163 deletions(-)
diff --git a/config.example.yaml b/config.example.yaml
@@ -1,5 +1,6 @@
homeserverUrl: "https://matrix.midnightthoughts.space"
accessToken: ""
+deviceID: "ABCDEFGHIJKL"
meilisearch:
host: "http://localhost:7700"
masterKey: "masterKey"
\ No newline at end of file
diff --git a/package.json b/package.json
@@ -24,12 +24,13 @@
},
"dependencies": {
"@matrix-org/matrix-sdk-crypto-nodejs": "0.2.0-beta.1",
- "@react-pdf/renderer": "^3.4.4",
+ "@react-pdf/renderer": "^4.0.0",
"@sinclair/typebox": "^0.32.31",
"got": "^14.4.1",
"meilisearch": "^0.40.0",
"prompts": "^2.4.2",
"react": "^18.3.1",
"yaml": "^2.4.5"
- }
-}
-\ No newline at end of file
+ },
+ "packageManager": "pnpm@9.12.1+sha512.e5a7e52a4183a02d5931057f7a0dbff9d5e9ce3161e33fa68ae392125b79282a8a8a470a51dfc8a0ed86221442eb2fb57019b0990ed24fab519bf0e1bc5ccfc4"
+}
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
@@ -12,8 +12,8 @@ importers:
specifier: 0.2.0-beta.1
version: 0.2.0-beta.1
'@react-pdf/renderer':
- specifier: ^3.4.4
- version: 3.4.4(react@18.3.1)
+ specifier: ^4.0.0
+ version: 4.0.0(react@18.3.1)
'@sinclair/typebox':
specifier: ^0.32.31
version: 0.32.31
@@ -123,43 +123,43 @@ packages:
resolution: {integrity: sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==}
engines: {node: '>= 8'}
- '@react-pdf/fns@2.2.1':
- resolution: {integrity: sha512-s78aDg0vDYaijU5lLOCsUD+qinQbfOvcNeaoX9AiE7+kZzzCo6B/nX+l48cmt9OosJmvZvE9DWR9cLhrhOi2pA==}
+ '@react-pdf/fns@3.0.0':
+ resolution: {integrity: sha512-ICbIWR93PE6+xf2Xd/fXYO1dAuiOAJaszEuGGv3wp5lLSeeelDXlEYLh6R05okxh28YqMzc0Qd85x6n6MtaLUQ==}
- '@react-pdf/font@2.5.1':
- resolution: {integrity: sha512-Hyb2zBb92Glc3lvhmJfy4dO2Mj29KB26Uk12Ua9EhKAdiuCTLBqgP8Oe1cGwrvDI7xA4OOcwvBMdYh0vhOUHzA==}
+ '@react-pdf/font@3.0.0':
+ resolution: {integrity: sha512-/SYEud06maGQiAD0H6J5xnqigKm/FpLKqFYH6+2OjQ2tb32nnKqwqL3pS7glDTFYBJ11B6fVgSNex6tc1V8UDA==}
- '@react-pdf/image@2.3.6':
- resolution: {integrity: sha512-7iZDYZrZlJqNzS6huNl2XdMcLFUo68e6mOdzQeJ63d5eApdthhSHBnkGzHfLhH5t8DCpZNtClmklzuLL63ADfw==}
+ '@react-pdf/image@3.0.0':
+ resolution: {integrity: sha512-l4HV5NutPw52Wbuvxol0BiFc6TkKXK/kJNwVIElphw47nwfueLEjIfBxRTuOeHmpdibvEh3a5STwYmMTQjXSxg==}
- '@react-pdf/layout@3.12.1':
- resolution: {integrity: sha512-BxSeykDxvADlpe4OGtQ7NH46QXq3uImAYsTHOPLCwbXMniQ1O3uCBx7H+HthxkCNshgYVPp9qS3KyvQv/oIZwg==}
+ '@react-pdf/layout@4.0.0':
+ resolution: {integrity: sha512-BNipDwULk9ehvf0V5HLrP2QIujkfj2LzHs8yfCQ/Z8TkAX6dEt1AgwDttBpBLfzcu8w8HFjgpLR1TNOTXiOsAQ==}
- '@react-pdf/pdfkit@3.1.10':
- resolution: {integrity: sha512-P/qPBtCFo2HDJD0i6NfbmoBRrsOVO8CIogYsefwG4fklTo50zNgnMM5U1WLckTuX8Qt1ThiQuokmTG5arheblA==}
+ '@react-pdf/pdfkit@4.0.0':
+ resolution: {integrity: sha512-HaaAoBpoRGJ6c1ZOANNQZ3q6Ehmagqa8n40x+OZ5s9HcmUviZ34SCm+QBa42s1o4299M+Lgw3UoqpW7sHv3/Hg==}
- '@react-pdf/png-js@2.3.1':
- resolution: {integrity: sha512-pEZ18I4t1vAUS4lmhvXPmXYP4PHeblpWP/pAlMMRkEyP7tdAeHUN7taQl9sf9OPq7YITMY3lWpYpJU6t4CZgZg==}
+ '@react-pdf/png-js@3.0.0':
+ resolution: {integrity: sha512-eSJnEItZ37WPt6Qv5pncQDxLJRK15eaRwPT+gZoujP548CodenOVp49GST8XJvKMFt9YqIBzGBV/j9AgrOQzVA==}
- '@react-pdf/primitives@3.1.1':
- resolution: {integrity: sha512-miwjxLwTnO3IjoqkTVeTI+9CdyDggwekmSLhVCw+a/7FoQc+gF3J2dSKwsHvAcVFM0gvU8mzCeTofgw0zPDq0w==}
+ '@react-pdf/primitives@4.0.0':
+ resolution: {integrity: sha512-yp4E0rDL03NaUp/CnDBz3HQNfH2Mzdlgku57yhTMGNzetwB0NJusXcjYg5XsTGIXnR7Tv80JKI4O4ajj+oaLeQ==}
- '@react-pdf/render@3.4.4':
- resolution: {integrity: sha512-CfGxWmVgrY3JgmB1iMnz2W6Ck+8pisZeFt8vGlxP+JfT+0onr208pQvGSV5KwA9LGhAdABxqc/+y17V3vtKdFA==}
+ '@react-pdf/render@4.0.0':
+ resolution: {integrity: sha512-gvrw4HM9RocuDLd+19cvP2xaSs3h4OgYn2N6VDXzj6LYQnoHBAazRV9qMpGi8FuNlQ3Va+s82R1ynXYMr0FXIg==}
- '@react-pdf/renderer@3.4.4':
- resolution: {integrity: sha512-j1TWMHHXDeHdoQE3xjhBh0MZ2rn7wHIlP/uglr/EJZXqnPbfg6bfLzRJCM6bs+XJV3d8+zLQjHf6sF/fWcBDfg==}
+ '@react-pdf/renderer@4.0.0':
+ resolution: {integrity: sha512-yDObqmnF08Mwc24W8axmT/y+JPxJkweVQ/EM3GOXh0qyZN5VP5JvKDiCqmftI6QEKfvTFNsquuEQm13GBjkbMg==}
peerDependencies:
react: ^16.8.0 || ^17.0.0 || ^18.0.0
- '@react-pdf/stylesheet@4.2.5':
- resolution: {integrity: sha512-XnmapeCW+hDuNdVwpuvO04WKv71wAs8aH+saIq29Bo2fp1SxznHTcQArTZtK6Wgr/E9BHXeB2iAPpUZuI6G+xA==}
+ '@react-pdf/stylesheet@5.0.0':
+ resolution: {integrity: sha512-FrJXUMsvSGee13gpL82HOhhk16y3IKxLmYvmJU1ZUo9Jm9pLydnHDPlSUDK+rKmxdSD2X+twUR7sv6FlrA5i+A==}
- '@react-pdf/textkit@4.4.1':
- resolution: {integrity: sha512-Jl9wdTqIvJ5pX+vAGz0EOhP7ut5Two9H6CzTKo/YYPeD79cM2yTXF3JzTERBC28y7LR0Waq9D2LHQjI+b/EYUQ==}
+ '@react-pdf/textkit@5.0.0':
+ resolution: {integrity: sha512-+K6zkF6lDXHIZZ9nNzplJ48IrjXNGQqVfO5v73AsutHbvx44E62t46EeoGLHsmjGeMG70TooSI1Mwq/7f/5tLw==}
- '@react-pdf/types@2.5.0':
- resolution: {integrity: sha512-XsVRkt0hQ60I4e3leAVt+aZR3KJCaJd179BfJHAv4F4x6Vq3yqkry8lcbUWKGKDw1j3/8sW4FsgGR41SFvsG9A==}
+ '@react-pdf/types@2.7.0':
+ resolution: {integrity: sha512-7KrPPCpgRPKR+g+T127PE4bpw9Q84ZiY07EYRwXKVtTEVW9wJ5BZiF9smT9IvH19s+MQaDLmYRgjESsnqlyH0Q==}
'@sec-ant/readable-stream@0.4.1':
resolution: {integrity: sha512-831qok9r2t8AlxLko40y2ebgSDhenenCatLVeW/uBtnHPyhHOvG0C7TvfgecV+wHzIm5KUICgzmVpWS+IMEAeg==}
@@ -604,8 +604,8 @@ packages:
isexe@2.0.0:
resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==}
- jay-peg@1.0.2:
- resolution: {integrity: sha512-fyV3NVvv6pTys/3BTapBUGAWAuU9rM2gRcgijZHzptd5KKL+s+S7hESFN+wOsbDH1MzFwdlRAXi0aGxS6uiMKg==}
+ jay-peg@1.1.0:
+ resolution: {integrity: sha512-WhyKySfx5CEFoKDnpmHyJUrpX5fUrr/X3kqVHISmiO9jrJC73RQBOAZJB8bDrWT4PHEkl0QgNZLlWJfAWAIFew==}
jju@1.4.0:
resolution: {integrity: sha512-8wb9Yw966OSxApiCt0K3yNJL8pnNeIv+OEq2YMidz4FKP6nonSRoOXc80iXY4JaN2FC11B9qsNmDsm+ZOfMROA==}
@@ -969,8 +969,8 @@ packages:
resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==}
engines: {node: '>=10'}
- yoga-layout@2.0.1:
- resolution: {integrity: sha512-tT/oChyDXelLo2A+UVnlW9GU7CsvFMaEnd9kVFsaiCQonFAXd3xrHhkLYu+suwwosrAEQ746xBU+HvYtm1Zs2Q==}
+ yoga-layout@3.1.0:
+ resolution: {integrity: sha512-auzJ8lEovThZIpR8wLGWNo/JEj4VTO79q9/gOJ0dWb3shAYPFdX3t9VN0fC0v+jeQF77STUdCzebLwRMqzn5gQ==}
snapshots:
@@ -1043,84 +1043,84 @@ snapshots:
'@nodelib/fs.scandir': 2.1.5
fastq: 1.17.1
- '@react-pdf/fns@2.2.1':
+ '@react-pdf/fns@3.0.0':
dependencies:
'@babel/runtime': 7.24.7
- '@react-pdf/font@2.5.1':
+ '@react-pdf/font@3.0.0':
dependencies:
'@babel/runtime': 7.24.7
- '@react-pdf/types': 2.5.0
+ '@react-pdf/types': 2.7.0
cross-fetch: 3.1.8
fontkit: 2.0.2
is-url: 1.2.4
transitivePeerDependencies:
- encoding
- '@react-pdf/image@2.3.6':
+ '@react-pdf/image@3.0.0':
dependencies:
'@babel/runtime': 7.24.7
- '@react-pdf/png-js': 2.3.1
+ '@react-pdf/png-js': 3.0.0
cross-fetch: 3.1.8
- jay-peg: 1.0.2
+ jay-peg: 1.1.0
transitivePeerDependencies:
- encoding
- '@react-pdf/layout@3.12.1':
+ '@react-pdf/layout@4.0.0':
dependencies:
'@babel/runtime': 7.24.7
- '@react-pdf/fns': 2.2.1
- '@react-pdf/image': 2.3.6
- '@react-pdf/pdfkit': 3.1.10
- '@react-pdf/primitives': 3.1.1
- '@react-pdf/stylesheet': 4.2.5
- '@react-pdf/textkit': 4.4.1
- '@react-pdf/types': 2.5.0
+ '@react-pdf/fns': 3.0.0
+ '@react-pdf/image': 3.0.0
+ '@react-pdf/pdfkit': 4.0.0
+ '@react-pdf/primitives': 4.0.0
+ '@react-pdf/stylesheet': 5.0.0
+ '@react-pdf/textkit': 5.0.0
+ '@react-pdf/types': 2.7.0
cross-fetch: 3.1.8
emoji-regex: 10.3.0
queue: 6.0.2
- yoga-layout: 2.0.1
+ yoga-layout: 3.1.0
transitivePeerDependencies:
- encoding
- '@react-pdf/pdfkit@3.1.10':
+ '@react-pdf/pdfkit@4.0.0':
dependencies:
'@babel/runtime': 7.24.7
- '@react-pdf/png-js': 2.3.1
+ '@react-pdf/png-js': 3.0.0
browserify-zlib: 0.2.0
crypto-js: 4.2.0
fontkit: 2.0.2
- jay-peg: 1.0.2
+ jay-peg: 1.1.0
vite-compatible-readable-stream: 3.6.1
- '@react-pdf/png-js@2.3.1':
+ '@react-pdf/png-js@3.0.0':
dependencies:
browserify-zlib: 0.2.0
- '@react-pdf/primitives@3.1.1': {}
+ '@react-pdf/primitives@4.0.0': {}
- '@react-pdf/render@3.4.4':
+ '@react-pdf/render@4.0.0':
dependencies:
'@babel/runtime': 7.24.7
- '@react-pdf/fns': 2.2.1
- '@react-pdf/primitives': 3.1.1
- '@react-pdf/textkit': 4.4.1
- '@react-pdf/types': 2.5.0
+ '@react-pdf/fns': 3.0.0
+ '@react-pdf/primitives': 4.0.0
+ '@react-pdf/textkit': 5.0.0
+ '@react-pdf/types': 2.7.0
abs-svg-path: 0.1.1
color-string: 1.9.1
normalize-svg-path: 1.1.0
parse-svg-path: 0.1.2
svg-arc-to-cubic-bezier: 3.2.0
- '@react-pdf/renderer@3.4.4(react@18.3.1)':
+ '@react-pdf/renderer@4.0.0(react@18.3.1)':
dependencies:
'@babel/runtime': 7.24.7
- '@react-pdf/font': 2.5.1
- '@react-pdf/layout': 3.12.1
- '@react-pdf/pdfkit': 3.1.10
- '@react-pdf/primitives': 3.1.1
- '@react-pdf/render': 3.4.4
- '@react-pdf/types': 2.5.0
+ '@react-pdf/font': 3.0.0
+ '@react-pdf/layout': 4.0.0
+ '@react-pdf/pdfkit': 4.0.0
+ '@react-pdf/primitives': 4.0.0
+ '@react-pdf/render': 4.0.0
+ '@react-pdf/types': 2.7.0
events: 3.3.0
object-assign: 4.1.1
prop-types: 15.8.1
@@ -1130,25 +1130,25 @@ snapshots:
transitivePeerDependencies:
- encoding
- '@react-pdf/stylesheet@4.2.5':
+ '@react-pdf/stylesheet@5.0.0':
dependencies:
'@babel/runtime': 7.24.7
- '@react-pdf/fns': 2.2.1
- '@react-pdf/types': 2.5.0
+ '@react-pdf/fns': 3.0.0
+ '@react-pdf/types': 2.7.0
color-string: 1.9.1
hsl-to-hex: 1.0.0
media-engine: 1.0.3
postcss-value-parser: 4.2.0
- '@react-pdf/textkit@4.4.1':
+ '@react-pdf/textkit@5.0.0':
dependencies:
'@babel/runtime': 7.24.7
- '@react-pdf/fns': 2.2.1
+ '@react-pdf/fns': 3.0.0
bidi-js: 1.0.3
hyphen: 1.10.4
unicode-properties: 1.4.1
- '@react-pdf/types@2.5.0': {}
+ '@react-pdf/types@2.7.0': {}
'@sec-ant/readable-stream@0.4.1': {}
@@ -1651,7 +1651,7 @@ snapshots:
isexe@2.0.0: {}
- jay-peg@1.0.2:
+ jay-peg@1.1.0:
dependencies:
restructure: 3.0.1
@@ -1952,4 +1952,4 @@ snapshots:
yocto-queue@0.1.0: {}
- yoga-layout@2.0.1: {}
+ yoga-layout@3.1.0: {}
diff --git a/src/bot.ts b/src/bot.ts
@@ -2,7 +2,7 @@ import Indexer from "./indexer.js";
import Config from "./config.js";
import fs from "node:fs";
import { MatrixClient } from "./tiny-sdk.js";
-import { RoomId } from "@matrix-org/matrix-sdk-crypto-nodejs";
+import { renderPDFToBufferWithData } from "./renderer.js";
// Ensures no matrix IDs are in the content.body
function cleanContent(content: Record<string, any>) {
@@ -54,13 +54,8 @@ async function backfill(client: MatrixClient, indexer: Indexer) {
}
}
- if (event["type"] === "m.room.encrypted") {
- const rawEvent = await client.olmMachine?.decryptRoomEvent(JSON.stringify(event), new RoomId(room));
- if (rawEvent) {
- event = JSON.parse(rawEvent.event);
- }
- }
- void handleMessages(indexer, client, room, event)
+ event = await client.decryptEvent(event, room);
+ void handleMessages(indexer, client, room, event, true);
}
}
@@ -97,8 +92,35 @@ function convertEventToDocument(event: any, roomId: string, room_name?: string)
return doc
}
-async function handleMessages(indexer: Indexer, client: MatrixClient, roomId: string, event: any) {
+async function handleMessages(indexer: Indexer, client: MatrixClient, roomId: string, event: any, historical = false) {
if (event["content"]["msgtype"] === "m.text") {
+ const user = await client.whoami();
+ // If command and sender is bot user then handle as a command and not index
+ if (event["content"]["body"].startsWith("!") && event["sender"] === user.user_id && !historical) {
+ // Get the command directly after the ! (e.g. !search -> search)
+ const command = event["content"]["body"].split(" ")[0].substring(1);
+ const args: string[] = event["content"]["body"].split(" ").slice(1);
+
+ // Render the amount of events given in the argument (e.g. !last 5) as a pdf and send it into the room
+ if (command === "last") {
+ console.info(`Received "last" command in room ${roomId}`);
+
+ const amount = parseInt(args[0]);
+ console.info(`Rendering last ${amount} messages as PDF`);
+ const results = await indexer.search("", roomId, undefined, amount);
+ console.info(`Got ${results.hits.length} results`);
+ const pdf = await renderPDFToBufferWithData(results, "", roomId, undefined);
+
+ const event_id = event["event_id"] as string;
+ await client.sendFile(roomId, event_id, "Here is the PDF with the last " + amount + " messages", pdf);
+
+ // Redact the original command message
+ await client.redactEvent(roomId, event_id, "Redacted by bot");
+ return;
+ }
+ }
+
+
//console.info(`Received message in room ${roomId}`);
if (event.content["m.relates_to"]) {
if (event.content["m.relates_to"].rel_type === "m.replace") {
@@ -108,11 +130,13 @@ async function handleMessages(indexer: Indexer, client: MatrixClient, roomId: st
}
const room_name = await client.getRoomName(roomId);
- await indexer.insert(convertEventToDocument(event, roomId, room_name));
+
+ void indexer.insert(convertEventToDocument(event, roomId, room_name));
}
}
async function handleSync(client: MatrixClient, indexer: Indexer) {
+ console.info("Starting sync");
const events = client.sync();
for await (const [roomId, event] of events) {
void handleMessages(indexer, client, roomId, event)
diff --git a/src/config.ts b/src/config.ts
@@ -7,6 +7,7 @@ import YAML from 'yaml';
const ConfigSchema = Type.Object({
homeserverUrl: Type.String(),
accessToken: Type.String(),
+ masMode: Type.Boolean(),
meilisearch: Type.Object({
host: Type.String(),
masterKey: Type.String()
@@ -25,6 +26,7 @@ export default class Config {
const config = {
homeserverUrl: process.env.HOMESERVER_URL,
accessToken: process.env.ACCESS_TOKEN,
+ masMode: process.env.MAS_MODE === 'true',
};
// load the config file async and merge it with the env
@@ -50,6 +52,11 @@ export default class Config {
return this.config.accessToken;
}
+
+ public get masMode(): boolean {
+ return this.config.masMode;
+ }
+
public set homeserverUrl(value: string) {
this.config.homeserverUrl = value;
this.save();
@@ -60,6 +67,11 @@ export default class Config {
this.save();
}
+ public set masMode(value: boolean) {
+ this.config.masMode = value;
+ this.save();
+ }
+
public get meiliSearchHost(): string {
return this.config.meilisearch.host;
}
diff --git a/src/indexer.ts b/src/indexer.ts
@@ -25,8 +25,7 @@ export default class Indexer {
return await this.textIndex.addDocuments([data], { primaryKey: 'id' });
}
- public async search(query: string, room_id?: string, sender?: string) {
-
+ public async search(query: string, room_id?: string, sender?: string, limit?: number) {
await this.textIndex.updateSearchableAttributes(['content.body', "sender", "content.m.mentions.user_id", "room_name"]);
await this.textIndex.updateFilterableAttributes([
'id',
@@ -62,6 +61,10 @@ export default class Indexer {
page++;
}
+ if (limit) {
+ results = results.slice(0, limit);
+ }
+
return { hits: results, estimatedTotalHits: results.length };
}
diff --git a/src/renderer.tsx b/src/renderer.tsx
@@ -1,8 +1,10 @@
import React, { PropsWithChildren } from 'react';
-import ReactPDF, { Font, Page, Text, View, Document, StyleSheet, Link } from '@react-pdf/renderer';
+import ReactPDF, { Font, Page, Text, View, Document, StyleSheet, Link, renderToBuffer } from '@react-pdf/renderer';
import Indexer from './indexer.js';
import { Hits } from 'meilisearch';
import prompts, { PromptObject } from 'prompts';
+import { fileURLToPath } from 'node:url';
+import { resolve } from 'node:path';
Font.registerEmojiSource({
format: 'png',
@@ -23,10 +25,10 @@ const styles = StyleSheet.create({
const indexer = new Indexer();
-const borderRadius = 5;
+const borderRadius = 2;
// A pdf renderer
-const MessageDocument = ({ query, room_id, sender, queryResults }: PropsWithChildren<{ query: string, room_id?: string, sender?: string, queryResults: { hits: Hits<Record<string, any>>; estimatedTotalHits: number; } }>) => {
+export const MessageDocument = ({ query, room_id, sender, queryResults }: PropsWithChildren<{ query: string, room_id?: string, sender?: string, queryResults: { hits: Hits<Record<string, any>>; estimatedTotalHits: number; } }>) => {
return (
<Document
title={`Search results for: "${query}"`}
@@ -57,8 +59,25 @@ const MessageDocument = ({ query, room_id, sender, queryResults }: PropsWithChil
</View>
<View>
{queryResults.hits.map((result, index) => (
- <View wrap={false} key={index} style={{ flexDirection: 'column', margin: 10, padding: 5, backgroundColor: '#e6e6e6', gap: 25, borderTopLeftRadius: borderRadius, borderTopRightRadius: borderRadius, borderBottomLeftRadius: borderRadius, borderBottomRightRadius: borderRadius, borderStyle: 'solid', borderWidth: 1, borderColor: '#00000' }}>
- <View style={{ flexDirection: 'row', justifyContent: 'space-between', fontSize: 10 }} >
+ <View
+ wrap={result.content.body.length > 500}
+ key={index}
+ style={{
+ flexDirection: 'column',
+ margin: 8,
+ padding: 12,
+ backgroundColor: '#e6e6e6',
+ gap: 16,
+ borderTopLeftRadius: borderRadius,
+ borderTopRightRadius: borderRadius,
+ borderBottomLeftRadius: borderRadius,
+ borderBottomRightRadius: borderRadius,
+ borderStyle: 'solid',
+ borderWidth: 1,
+ borderColor: '#00000'
+ }}>
+ <View
+ style={{ flexDirection: 'row', justifyContent: 'space-between', fontSize: 10 }}>
<Text>{result.room_name ?? result.room_id}</Text>
<View style={{ flexDirection: 'row' }}>
<Text>{result.sender}</Text>
@@ -69,21 +88,21 @@ const MessageDocument = ({ query, room_id, sender, queryResults }: PropsWithChil
</View>
))}
</View>
- </View>
- <View style={{ flexDirection: 'row', justifyContent: 'space-between', marginTop: 10, fontSize: 10 }} fixed>
- <Text render={({ pageNumber, totalPages }) => (
- `${pageNumber} / ${totalPages}`
- )} />
- <Text>
- Matrix Search Engine
- </Text>
+ <View style={{ flexDirection: 'row', justifyContent: 'space-between', marginTop: 10, fontSize: 10 }} fixed>
+ <Text render={({ pageNumber, totalPages }) => (
+ `${pageNumber} / ${totalPages}`
+ )} />
+ <Text>
+ Matrix Search Engine
+ </Text>
+ </View>
</View>
</Page>
</Document >
)
}
-export async function renderPDFToDisk(query: string, room_id?: string, sender?: string) {
+export async function renderPDFToDisk(query: string, room_id?: string, sender?: string, includeRedactions: boolean = false, includeEdits: boolean = false) {
let cleanedRoomId = room_id?.trim();
let cleanedSender = sender?.trim();
if (cleanedRoomId === "") {
@@ -93,33 +112,67 @@ export async function renderPDFToDisk(query: string, room_id?: string, sender?:
cleanedSender = undefined;
}
const queryResults = await indexer.search(query, cleanedRoomId, cleanedSender);
+
+ // Filter out redactions if the user does not want them
+ if (!includeRedactions) {
+ queryResults.hits = queryResults.hits.filter((hit) => !hit.redacted);
+ }
+ // Filter out edits if the user does not want them
+ if (!includeEdits) {
+ queryResults.hits = queryResults.hits.filter((hit) => !hit.edited);
+ }
+
//console.log(`Search results:`, queryResults);
await ReactPDF.renderToFile(<MessageDocument query={query} room_id={cleanedRoomId} sender={cleanedSender} queryResults={queryResults} />, `output.pdf`);
}
+export async function renderPDFToBufferWithData(queryResults: { hits: Hits<Record<string, any>>; estimatedTotalHits: number; }, query?: string, room_id?: string, sender?: string,): Promise<Buffer> {
+ return await renderToBuffer(<MessageDocument query={query ?? ""} room_id={room_id} sender={sender} queryResults={queryResults} />);
+}
-const questions: PromptObject<string>[] = [
- {
- type: 'text',
- name: 'query',
- initial: "",
- message: 'Enter your search query:',
- },
- {
- type: 'text',
- name: 'room_id',
- initial: undefined,
- message: 'Enter the Room ID:',
- },
- {
- type: 'text',
- name: 'sender',
- initial: undefined,
- message: 'Enter the sender:',
- }
-]
-const response = await prompts(questions);
+const pathToThisFile = resolve(fileURLToPath(import.meta.url))
+const pathPassedToNode = resolve(process.argv[1])
+const isThisFileBeingRunViaCLI = pathToThisFile.includes(pathPassedToNode)
+
+if (isThisFileBeingRunViaCLI) {
+ const questions: PromptObject<string>[] = [
+ {
+ type: 'text',
+ name: 'query',
+ initial: "",
+ message: 'Enter your search query:',
+ },
+ {
+ type: 'text',
+ name: 'room_id',
+ initial: undefined,
+ message: 'Enter the Room ID:',
+ },
+ {
+ type: 'text',
+ name: 'sender',
+ initial: undefined,
+ message: 'Enter the sender:',
+ },
+ // Ask if the redactions should be included
+ {
+ type: 'confirm',
+ name: 'include_redactions',
+ initial: false,
+ message: 'Include redactions in the search results?',
+ },
+ // Ask if the edits should be included
+ {
+ type: 'confirm',
+ name: 'include_edits',
+ initial: false,
+ message: 'Include edits in the search results?',
+ }
+ ]
+
+ const response = await prompts(questions);
-await renderPDFToDisk(response.query as string, response.room_id as string | undefined, response.sender as string | undefined);
-console.log("PDF rendered to disk");
-\ No newline at end of file
+ await renderPDFToDisk(response.query as string, response.room_id as string | undefined, response.sender as string | undefined, response.include_redactions as boolean, response.include_edits as boolean);
+ console.log("PDF rendered to disk");
+}
+\ No newline at end of file
diff --git a/src/tiny-sdk.ts b/src/tiny-sdk.ts
@@ -18,7 +18,7 @@ import {
ToDeviceRequest,
UserId
} from "@matrix-org/matrix-sdk-crypto-nodejs";
-import fs from 'node:fs/promises';
+import { readFileSync, writeFileSync } from "node:fs";
export const directions = { forward: "f", reverse: "b" } as const;
@@ -38,6 +38,7 @@ export class MatrixClient {
constructor(private homeserver: string, private token: string) { }
async start() {
+ console.log("Starting client");
const whoami = await this.whoami();
console.log(`Logged in as ${whoami.user_id}`);
@@ -48,6 +49,7 @@ export class MatrixClient {
// 2. if we don't, create a random string
// 3. create a new DeviceId object
const device_id = whoami.device_id ?? Math.random().toString(36).substring(7);
+ console.log(`Device ID: ${device_id}`);
this.deviceId = new DeviceId(device_id);
this.olmMachine = await OlmMachine.initialize(this.userId, this.deviceId, "./storage/crypto", undefined, StoreType.Sqlite);
@@ -57,11 +59,13 @@ export class MatrixClient {
// Reads the storage/bot.json for the syncToken field.
// Also handle the case where the file doesn't exist.
- async getSyncTokenFromStorage(): Promise<string | undefined> {
+ getSyncTokenFromStorage(): { syncToken?: string, previous?: string } | undefined {
try {
- const file = await fs.readFile("./storage/bot.json", "utf-8");
+ const file = readFileSync("./storage/bot.json", "utf-8");
const json = JSON.parse(file);
- return json.syncToken as string | undefined;
+ const syncToken = json.syncToken as string | undefined;
+ const previous = json.previous as string | undefined;
+ return { syncToken, previous };
} catch (e) {
return undefined;
}
@@ -69,27 +73,36 @@ export class MatrixClient {
// Writes the syncToken to the storage/bot.json
// Ensure we dont fail if the file doesn't exist
- async setSyncTokenToStorage(syncToken: string) {
+ setSyncTokenToStorage(syncToken: string, previous?: string) {
console.log("Setting sync token to", syncToken);
- await fs.writeFile("./storage/bot.json", JSON.stringify({ syncToken }));
+ try {
+ writeFileSync("./storage/bot.json", JSON.stringify({ syncToken, previous }), { flush: true });
+ } catch (e) {
+ console.error("Failed to write sync token", e);
+ process.exit(1);
+ }
}
async * sync(): AsyncGenerator<[string, Record<string, any>], void, void> {
if (!this.olmMachine || !this.token) {
throw new Error("Client not started");
}
- do {
+ while (this.syncRunning) {
let url = "/_matrix/client/v3/sync";
- const syncToken = await this.getSyncTokenFromStorage();
- console.log("Syncing with token", syncToken);
- if (syncToken) {
- url += `?since=${syncToken}`;
+ const tokens = this.getSyncTokenFromStorage();
+ console.log("Syncing with token", tokens?.syncToken);
+ if (tokens?.syncToken) {
+ url += `?since=${tokens.syncToken}`;
}
let syncResp: Record<string, any>;
try {
syncResp = await this.getRequest(url, {}, 30 * 1000);
- } catch { continue }
+ console.log("Got sync response");
+ } catch (e) {
+ console.error("Failed to sync", e);
+ continue;
+ }
const toDeviceEvents: Record<string, any>[] | undefined = syncResp.to_device?.events;
const oneTimeKeyCounts: Record<string, number> = syncResp.device_one_time_keys_count;
@@ -106,7 +119,6 @@ export class MatrixClient {
unusedFallbackKeys,
);
-
const outgoingRequests = await this.olmMachine.outgoingRequests();
// Send outgoing requests
for (const request of outgoingRequests) {
@@ -164,30 +176,19 @@ export class MatrixClient {
continue;
}
}
- console.log("Synced", syncResp.next_batch as string)
// Save the sync token
- await this.setSyncTokenToStorage(syncResp.next_batch as string);
+ console.log("Next:", syncResp.next_batch as string, "Current:", tokens?.syncToken);
+ this.setSyncTokenToStorage(syncResp.next_batch as string, tokens?.syncToken);
+
+ if (syncResp.next_batch === tokens?.previous || syncResp.next_batch === tokens?.syncToken) {
+ // We already synced this token
+ console.log("Already synced this token, waiting for new events");
+ continue;
+ }
// decrypt all events here. for our usecase we only care about joined rooms
- await Promise.all(Object.entries(syncResp.rooms.join as Record<string, any>).map(async ([roomId, room]: [string, any]) => {
- await Promise.all(room.timeline.events.map(async (event: Record<string, any>) => {
- if (event.type === "m.room.encrypted") {
- try {
- const rawEvent = await this.olmMachine?.decryptRoomEvent(JSON.stringify(event), new RoomId(roomId));
- if (rawEvent) {
- event = JSON.parse(rawEvent.event);
- } else {
- //console.warn("Failed to decrypt event", event);
- return event;
- }
- } catch (e) {
- //console.warn("Failed to decrypt event", e);
- return event;
- }
- }
- }));
- }));
+ syncResp.rooms.join = await this.decryptTimeline(syncResp);
// parse new events from the sync response
const joinedRooms = syncResp.rooms.join as Record<string, any>;
@@ -199,13 +200,93 @@ export class MatrixClient {
yield [roomId, event];
}
}
- } while (this.syncRunning);
+ }
+ }
+
+ private async decryptTimeline(syncResp: any) {
+ const decryptedEvents: Record<string, Record<string, any>[]> = {};
+ for (const [roomId, room] of Object.entries(syncResp.rooms.join as Record<string, any>)) {
+ const decryptedRoom = await this.decryptListOfEvents(room.timeline.events as Record<string, any>[], roomId);
+ decryptedEvents[roomId] = decryptedRoom;
+ }
+ return decryptedEvents;
+ }
+
+ async decryptListOfEvents(events: Record<string, any>[], roomId: string): Promise<Record<string, any>[]> {
+ return await Promise.all(events.map(async (event) => {
+ return await this.decryptEvent(event, roomId);
+ }));
+ }
+
+ async decryptEvent(event: Record<string, any>, roomId: string): Promise<Record<string, any>> {
+ if (event.type === "m.room.encrypted") {
+ const members = await this.getRoomMembers(roomId, ['join', 'invite']);
+ await this.addTrackedUsers(members.map(e => e["state_key"] as string))
+ try {
+ const rawEvent = await this.olmMachine?.decryptRoomEvent(JSON.stringify(event), new RoomId(roomId));
+ if (rawEvent) {
+ event = JSON.parse(rawEvent.event);
+ return event;
+ } else {
+ console.warn("Failed to decrypt event1", event);
+ return event;
+ }
+ } catch (e) {
+ console.warn("Failed to decrypt event2", e);
+ return event;
+ }
+ }
+ return event;
}
+ private async getRoomMembers(roomId: string, membership: string[]) {
+ const resp: { chunk: Record<string, any>[]; } = await this.getRequest(`/_matrix/client/v3/rooms/${encodeURIComponent(roomId)}/members`, { membership }, 30 * 1000);
+ return resp.chunk;
+ }
+
async whoami(): Promise<WhoAmIResponse> {
return await this.getRequest("/_matrix/client/v3/account/whoami", {}, 30 * 1000) as WhoAmIResponse;
}
+ async sendFile(roomId: string, event_id: string, body: string, file: Buffer) {
+ // Upload with authenticated media endpoint
+ const mediaResponse = await this.postRequest(`/_matrix/media/v3/upload?filename=${encodeURIComponent(roomId.replaceAll("!", "").replaceAll(":", "_"))}.pdf`, {}, file, 30 * 1000, "application/pdf");
+ const mediaUrl = mediaResponse.content_uri;
+
+ // Send a notice to the room telling the user the file is ready
+ const content = {
+ msgtype: "m.notice",
+ body: body,
+ "m.relates_to": {
+ "m.in_reply_to": {
+ event_id: event_id
+ }
+ }
+ };
+
+ const txnId = `${Date.now()}-${Math.random()}`;
+
+ await this.putRequest(`/_matrix/client/v3/rooms/${encodeURIComponent(roomId)}/send/m.room.message/${encodeURIComponent(txnId)}`, {}, JSON.stringify(content).toString(), 30 * 1000);
+
+ // Send file to the room
+ const txnId2 = `${Date.now()}-${Math.random()}`;
+
+ const fileContent = {
+ msgtype: "m.file",
+ body: `${roomId.replaceAll("!", "").replaceAll(":", "_")}.pdf`,
+ url: mediaUrl,
+ info: {
+ mimetype: "application/pdf",
+ size: file.length
+ }
+ };
+ await this.putRequest(`/_matrix/client/v3/rooms/${encodeURIComponent(roomId)}/send/m.room.message/${encodeURIComponent(txnId2)}`, {}, JSON.stringify(fileContent), 30 * 1000);
+ }
+
+ async redactEvent(roomId: string, eventId: string, reason?: string) {
+ await this.postRequest(`/_matrix/client/v3/rooms/${encodeURIComponent(roomId)}/redact/${encodeURIComponent(eventId)}`, {}, JSON.stringify({ reason }), 30 * 1000);
+ }
+
// Fetches the room name from the cache or the server
async getRoomName(roomId: string): Promise<string | undefined> {
if (typeof this.roomNameCache[roomId] === "string") {
@@ -231,32 +312,32 @@ export class MatrixClient {
if (!this.token) {
throw new Error("Client not started");
}
- return await got.get(`${this.homeserver}${path}`, {
+ return got.get(`${this.homeserver}${path}`, {
searchParams: query, timeout: {
request: timeout
}, headers: { "Content-Type": "application/json", "Authorization": `Bearer ${this.token}` }
}).json();
}
- async postRequest(path: string, query: Record<string, any>, body: string, timeout: number): Promise<any> {
+ async postRequest(path: string, query: Record<string, any>, body: string | Buffer, timeout: number, content_type = "application/json"): Promise<any> {
if (!this.token) {
throw new Error("Client not started");
}
return await got.post(`${this.homeserver}${path}`, {
searchParams: query, body, timeout: {
request: timeout
- }, headers: { "Content-Type": "application/json", "Authorization": `Bearer ${this.token}` }
+ }, headers: { "Content-Type": content_type, "Authorization": `Bearer ${this.token}` }
}).json();
}
- async putRequest(path: string, query: Record<string, any>, body: string, timeout: number): Promise<any> {
+ async putRequest(path: string, query: Record<string, any>, body: string, timeout: number, content_type = "application/json"): Promise<any> {
if (!this.token) {
throw new Error("Client not started");
}
return await got.put(`${this.homeserver}${path}`, {
searchParams: query, body, timeout: {
request: timeout
- }, headers: { "Content-Type": "application/json", "Authorization": `Bearer ${this.token}` }
+ }, headers: { "Content-Type": content_type, "Authorization": `Bearer ${this.token}` }
}).json();
}
@@ -301,4 +382,48 @@ export class MatrixClient {
yield response.chunk;
} while (from);
}
+
+ private async addTrackedUsers(users: string[]) {
+ const uids = users.map((u) => new UserId(u));
+ await this.olmMachine?.updateTrackedUsers(uids);
+
+ const keysClaim = await this.olmMachine?.getMissingSessions(uids);
+ if (keysClaim) {
+ const keysClaimRequest = keysClaim;
+ const resp = await this.postRequest("/_matrix/client/v3/keys/claim", {}, keysClaimRequest.body, 30 * 1000);
+ await this.olmMachine?.markRequestAsSent(keysClaim.id, keysClaim.type, JSON.stringify(resp));
+ }
+ }
+
+ // // MAS specific login
+ // async masLogin() {
+ // const auth_issuer_resp: { issuer: string } = await got.get(`${this.homeserver}/_matrix/client/unstable/org.matrix.msc2965/auth_issuer`).json();
+ // const issuer = auth_issuer_resp.issuer;
+
+ // const openid_config: { token_endpoint: string; device_authorization_endpoint: string; registration_endpoint: string; } = await got.get(issuer + "/.well-known/openid-configuration").json();
+ // const token_endpoint = openid_config.token_endpoint;
+ // const device_authorization_endpoint = openid_config.device_authorization_endpoint;
+ // const registration_endpoint = openid_config.registration_endpoint;
+
+ // // Register a client with `urn:ietf:params:oauth:grant-type:device_code` and `refresh_token` grant types
+ // const request = {
+ // application_type: "native",
+ // client_name: "Matrix Search",
+ // grant_types: ["urn:ietf:params:oauth:grant-type:device_code", "refresh_token"],
+ // response_types: [],
+ // token_endpoint_auth_method: "none"
+ // };
+ // const client_data: { client_id: string; } = await got.post(registration_endpoint, {
+ // json: request
+ // }).json();
+
+ // // Get a device code
+ // const deviceID = Math.random().toString(36).substring(7);
+ // const device_code_resp = await got.post(device_authorization_endpoint, {
+ // json: {
+ // client_id: client_data.client_id,
+ // scope: `urn:matrix:org.matrix.msc2967.client:api:* urn:matrix:org.matrix.msc2967.client:device:${deviceID}`
+ // }
+ // }).json();
+ // }
}
\ No newline at end of file
diff --git a/tsconfig.json b/tsconfig.json
@@ -1,4 +1,7 @@
{
+ "include": [
+ "src/**/*"
+ ],
"compilerOptions": {
"jsx": "react",
/* Visit https://aka.ms/tsconfig to read more about this file */
