msc-viewer

A possibly better way to display the Matrix Spec Changes
git clone git://archive.git.mtrnord.blog/MTRNord/msc-viewer.git
Log | Files | Refs | README | LICENSE

index.ts (11853B)

      1 import { MeiliSearch } from 'meilisearch'
      2 import { Octokit } from '@octokit/core';
      3 import { throttling } from '@octokit/plugin-throttling'
      4 import { paginateRest } from "@octokit/plugin-paginate-rest";
      5 import { MultiBar, Presets } from 'cli-progress';
      6 
      7 //const INDEX = "MSCs";
      8 const INDEX = "MSC_development";
      9 
     10 const OWNER = "matrix-org";
     11 const REPO = "matrix-spec-proposals";
     12 
     13 function wait(ms: number) {
     14     return new Promise(function (resolve, reject) {
     15         setTimeout(resolve, ms);
     16     });
     17 }
     18 
     19 const multibar = new MultiBar({
     20     clearOnComplete: false,
     21     hideCursor: true,
     22     format: ' {name} {bar} | {percentage} - {value}/{total} - {eta_formatted}',
     23 }, Presets.shades_classic);
     24 const prsBar = multibar.create(1, 0, { name: "PRs" });
     25 const commentsBar = multibar.create(1, 0, { name: "Comments" });
     26 const indexingBar = multibar.create(1, 0, { name: "Indexing" });
     27 
     28 interface Comment {
     29     url: string;
     30     diff_hunk: string;
     31     path: string;
     32     position?: number;
     33     commit_id: string;
     34     in_reply_to_id?: number;
     35     user?: {
     36         name?: string;
     37         login?: string;
     38         avatar_url?: string;
     39         url: string;
     40     };
     41     body: string;
     42     created_at: string;
     43     updated_at: string;
     44     author_association: string;
     45     _links: {
     46         self: {
     47             href: string;
     48         };
     49         html: {
     50             href: string;
     51         };
     52         pull_request: {
     53             href: string;
     54         };
     55     };
     56     reactions?: {
     57         url: string;
     58         total_count: number;
     59         "+1": number;
     60         "-1": number;
     61         laugh: number;
     62         confused: number;
     63         heart: number;
     64         hooray: number;
     65         eyes: number;
     66         rocket: number;
     67     };
     68 };
     69 
     70 interface Threads {
     71     [key: string]: Comment[];
     72 };
     73 
     74 interface Document {
     75     uid: number;
     76     author: string;
     77     author_url: string;
     78     body: string;
     79     closedAt: number;
     80     createdAt: number;
     81     mergedAt: number;
     82     updatedAt: number;
     83     permalink: string;
     84     title: string;
     85     state: string;
     86     threads?: Threads;
     87     comments?: Comment[];
     88     labels: { name: string; color: string; }[];
     89 }
     90 
     91 
     92 const client = new MeiliSearch({
     93     host: process.env.MEILI_HOST ?? 'http://127.0.0.1:7700',
     94     apiKey: process.env.APIKey,
     95     requestConfig: {
     96         headers: {
     97             Authorization: `Bearer ${process.env.APIKey}` ?? ''
     98         },
     99     }
    100 })
    101 
    102 const MyOctokit = Octokit.plugin(throttling, paginateRest);
    103 
    104 const octokit = new MyOctokit({
    105     auth: process.env.GIT_SECRET,
    106     throttle: {
    107         onRateLimit: (retryAfter, options, octokit, retryCount) => {
    108             console.warn(
    109                 `Request quota exhausted for request ${options.method} ${options.url}`,
    110             );
    111 
    112             if (retryCount <= 5) {
    113                 // retry 5 times
    114                 console.info(`Retrying after ${retryAfter} seconds!`);
    115                 return true;
    116             }
    117         },
    118         onSecondaryRateLimit: (retryAfter, options, octokit) => {
    119             // does not retry, only logs a warning
    120             console.warn(
    121                 `SecondaryRateLimit detected for request ${options.method} ${options.url}`,
    122             );
    123         },
    124 
    125     },
    126 });
    127 
    128 await client.deleteIndex(INDEX)
    129 await client.updateIndex(INDEX, { primaryKey: 'uid' });
    130 
    131 await client.index(INDEX).updateDisplayedAttributes([
    132     'uid',
    133     'author',
    134     'author_url',
    135     'body',
    136     "closedAt",
    137     "createdAt",
    138     "mergedAt",
    139     "updatedAt",
    140     "permalink",
    141     "title",
    142     "state",
    143     "threads",
    144     "comments",
    145     "labels",
    146 ]);
    147 await client.index(INDEX).updateSearchableAttributes([
    148     "title",
    149     'author',
    150     'body',
    151     "state",
    152     "threads",
    153     "comments",
    154 ]);
    155 
    156 const synonyms = {
    157     'ara4n': ['Matthew'],
    158     'turt2live': ["Travis", "TravisR"]
    159 };
    160 await client.index(INDEX).updateSynonyms(synonyms)
    161 
    162 await client.index(INDEX).updateFilterableAttributes([
    163     'author',
    164     'state',
    165     'closedAt',
    166     'createdAt',
    167     'mergedAt',
    168     'updatedAt',
    169     "labels",
    170 ])
    171 await client.index(INDEX).updateSortableAttributes(['closedAt', 'createdAt', 'mergedAt', 'updatedAt'])
    172 
    173 async function wait_for_rate_limit() {
    174     // Wait for github to reset the rate limit by fetching the rate limit endpoint and waiting until the reset time
    175     const rate_limit = await octokit.request('GET /rate_limit');
    176     const reset_time = rate_limit.data.resources.core.reset;
    177     const current_time = Math.floor(Date.now() / 1000);
    178     const wait_time = reset_time - current_time;
    179     multibar.log(`Waiting for ${wait_time} seconds until the rate limit is reset.`)
    180     await wait(wait_time * 1000);
    181 }
    182 
    183 async function get_documents(): Promise<Document[]> {
    184     let prs = 0;
    185     let documents: Document[] = [];
    186     const prIterator = octokit.paginate.iterator(
    187         "GET /repos/{owner}/{repo}/pulls",
    188         {
    189             owner: OWNER,
    190             repo: REPO,
    191             per_page: 100,
    192             state: "all",
    193         }
    194     );
    195 
    196     let last_added = 0;
    197     for await (const response of prIterator) {
    198         const nodes = response.data;
    199         prs += nodes.length;
    200         prsBar.increment(last_added);
    201         prsBar.setTotal(prs);
    202 
    203         const new_documents = await Promise.allSettled(nodes.map(async (node: any): Promise<Document> => {
    204             const author = node.user ? node.user.login : "unknown author";
    205             const author_url = node.user ? node.user.url : undefined;
    206             const closedAt = dateToTimestamp(new Date(node.closed_at));
    207             const createdAt = dateToTimestamp(new Date(node.created_at));
    208             const mergedAt = dateToTimestamp(new Date(node.merged_at));
    209             const updatedAt = dateToTimestamp(new Date(node.updated_at));
    210 
    211             const labels = await get_labels(node.labels);
    212             return {
    213                 uid: node.number,
    214                 author: author,
    215                 author_url: author_url,
    216                 body: node.body,
    217                 closedAt: closedAt,
    218                 createdAt: createdAt,
    219                 mergedAt: mergedAt,
    220                 permalink: node.url,
    221                 title: node.title,
    222                 state: node.state,
    223                 updatedAt: updatedAt,
    224                 labels: labels
    225             }
    226         }));
    227         documents = documents.concat(new_documents.filter((x) => x.status === "fulfilled").map((x) => (x as PromiseFulfilledResult<Document>).value));
    228         last_added = new_documents.length;
    229     }
    230 
    231     prsBar.increment(last_added);
    232     commentsBar.setTotal(documents.length);
    233     indexingBar.setTotal(documents.length);
    234 
    235     for (let i = 0; i < documents.length; i++) {
    236         const document = documents[i];
    237         const { threads, comments } = await get_comments(document.uid);
    238         document.threads = threads;
    239         document.comments = comments;
    240         commentsBar.increment();
    241     }
    242     return documents;
    243 }
    244 
    245 function dateToTimestamp(date: Date): number {
    246     return date.getTime() / 1000;
    247 }
    248 
    249 async function get_labels(labels: { id: number, node_id: string, url: string, name: string, description: string, color: string, default: boolean }[]): Promise<{ name: string; color: string; }[]> {
    250     return labels.map((label: any) => {
    251         return {
    252             name: label.name,
    253             color: label.color
    254         }
    255     });
    256 }
    257 
    258 async function get_comments(pr_id: number): Promise<{ threads: Threads, comments: Comment[] }> {
    259     let threads: Threads = {};
    260     let comments_aggregated: Comment[] = [];
    261     const commentIterator = octokit.paginate.iterator(
    262         "GET /repos/{owner}/{repo}/pulls/{pull_number}/comments",
    263         {
    264             owner: OWNER,
    265             repo: REPO,
    266             pull_number: pr_id,
    267             per_page: 100,
    268         }
    269     );
    270     for await (const response of commentIterator) {
    271         const data = response.data;
    272         for (const comment of data) {
    273             const clean_comment: Comment = {
    274                 url: comment.url,
    275                 diff_hunk: comment.diff_hunk,
    276                 path: comment.path,
    277                 position: comment.position,
    278                 commit_id: comment.commit_id,
    279                 in_reply_to_id: comment.in_reply_to_id,
    280                 user: comment.user ? {
    281                     name: comment.user.name ?? undefined,
    282                     login: comment.user.login,
    283                     avatar_url: comment.user.avatar_url,
    284                     url: comment.user.url,
    285                 } : undefined,
    286                 body: comment.body,
    287                 created_at: comment.created_at,
    288                 updated_at: comment.updated_at,
    289                 author_association: comment.author_association,
    290                 _links: {
    291                     self: {
    292                         href: comment._links.self.href,
    293                     },
    294                     html: {
    295                         href: comment._links.html.href,
    296                     },
    297                     pull_request: {
    298                         href: comment._links.pull_request.href,
    299                     },
    300                 },
    301                 reactions: comment.reactions ? {
    302                     url: comment.reactions.url,
    303                     total_count: comment.reactions.total_count,
    304                     "+1": comment.reactions["+1"],
    305                     "-1": comment.reactions["-1"],
    306                     laugh: comment.reactions.laugh,
    307                     confused: comment.reactions.confused,
    308                     heart: comment.reactions.heart,
    309                     hooray: comment.reactions.hooray,
    310                     eyes: comment.reactions.eyes,
    311                     rocket: comment.reactions.rocket,
    312                 } : undefined,
    313             };
    314             if (comment.pull_request_review_id) {
    315                 if (threads[comment.pull_request_review_id] == undefined) {
    316                     threads[comment.pull_request_review_id] = [];
    317                 }
    318                 threads[comment.pull_request_review_id].push(clean_comment)
    319             } else {
    320                 comments_aggregated.push(clean_comment)
    321             }
    322         }
    323         await wait(150);
    324     }
    325     return {
    326         threads: threads,
    327         comments: comments_aggregated
    328     }
    329 }
    330 
    331 /**
    332  * We want to wait until the status of the task is either `succeeded` or until
    333  * the `error` field has an error. If we got an error we print it and exit.
    334  * 
    335  * @param taskID The id of a meilisearch task
    336  */
    337 async function waitForTaskToFailOrComplete(taskID: number, pr_number: number): Promise<number | undefined> {
    338     const task = await client.getTask(taskID);
    339     if (task.status === 'failed') {
    340         console.error(`Task failed for PR ${pr_number}:`, JSON.stringify(task, null, 2));
    341         //process.exit(1);
    342         return pr_number;
    343     } else if (task.status === 'succeeded') {
    344         return;
    345     } else {
    346         await wait(1000);
    347         await waitForTaskToFailOrComplete(taskID, pr_number);
    348     }
    349 }
    350 
    351 /**
    352  * We add documents one by one.
    353  * 
    354  * @param documents The documents to add to the index
    355  * @param primaryKey The primary key of the documents
    356  */
    357 async function addDocuments(documents: Document[], primaryKey: string): Promise<number[]> {
    358     let errors: number[] = [];
    359     for (const document of documents) {
    360         const task = await client.index(INDEX).addDocuments([document], { primaryKey });
    361         //const error_pr = await waitForTaskToFailOrComplete(task.taskUid, document.number);
    362         indexingBar.increment();
    363         //if (error_pr) {
    364         //    errors.push(error_pr);
    365         //}
    366     }
    367     return errors;
    368 }
    369 
    370 async function main() {
    371     await wait_for_rate_limit();
    372     const documents = await get_documents();
    373 
    374     const errors = await addDocuments(documents, 'uid');
    375 
    376     multibar.stop();
    377     if (errors.length > 0) {
    378         console.error(`Failed to index the following PRs: ${errors.join(", ")}`);
    379     }
    380     const stats = await client.getStats();
    381     console.log(`Stats:`, JSON.stringify(stats, null, 2));
    382 }
    383 
    384 await main();