commit e82943292892ebffa509360b1002549c44233a59
parent b1ad13e0f4c543fd7814e0234eb6cdbbe8f662fc
Author: MTRNord <mtrnord1@gmail.com>
Date: Sun, 17 Dec 2023 03:15:37 +0100
Massively improve the indexer code, fix casing of the checkboxes and add missing date filters
Diffstat:
5 files changed, 252 insertions(+), 126 deletions(-)
diff --git a/scripts/indexer/package-lock.json b/scripts/indexer/package-lock.json
@@ -10,12 +10,13 @@
"license": "AGPL-3.0-or-later",
"dependencies": {
"@octokit/core": "^5.0.2",
- "@octokit/plugin-paginate-graphql": "^4.0.0",
"@octokit/plugin-paginate-rest": "^9.1.5",
"@octokit/plugin-throttling": "^8.1.3",
+ "cli-progress": "^3.12.0",
"meilisearch": "^0.36.0"
},
"devDependencies": {
+ "@types/cli-progress": "^3.11.5",
"@types/node": "^20.10.3",
"typescript": "^5.3.2"
}
@@ -75,17 +76,6 @@
"resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-19.1.0.tgz",
"integrity": "sha512-6G+ywGClliGQwRsjvqVYpklIfa7oRPA0vyhPQG/1Feh+B+wU0vGH1JiJ5T25d3g1JZYBHzR2qefLi9x8Gt+cpw=="
},
- "node_modules/@octokit/plugin-paginate-graphql": {
- "version": "4.0.0",
- "resolved": "https://registry.npmjs.org/@octokit/plugin-paginate-graphql/-/plugin-paginate-graphql-4.0.0.tgz",
- "integrity": "sha512-7HcYW5tP7/Z6AETAPU14gp5H5KmCPT3hmJrS/5tO7HIgbwenYmgw4OY9Ma54FDySuxMwD+wsJlxtuGWwuZuItA==",
- "engines": {
- "node": ">= 18"
- },
- "peerDependencies": {
- "@octokit/core": ">=5"
- }
- },
"node_modules/@octokit/plugin-paginate-rest": {
"version": "9.1.5",
"resolved": "https://registry.npmjs.org/@octokit/plugin-paginate-rest/-/plugin-paginate-rest-9.1.5.tgz",
@@ -150,6 +140,15 @@
"@octokit/openapi-types": "^19.1.0"
}
},
+ "node_modules/@types/cli-progress": {
+ "version": "3.11.5",
+ "resolved": "https://registry.npmjs.org/@types/cli-progress/-/cli-progress-3.11.5.tgz",
+ "integrity": "sha512-D4PbNRbviKyppS5ivBGyFO29POlySLmA2HyUFE4p5QGazAMM3CwkKWcvTl8gvElSuxRh6FPKL8XmidX873ou4g==",
+ "dev": true,
+ "dependencies": {
+ "@types/node": "*"
+ }
+ },
"node_modules/@types/node": {
"version": "20.10.3",
"resolved": "https://registry.npmjs.org/@types/node/-/node-20.10.3.tgz",
@@ -159,6 +158,14 @@
"undici-types": "~5.26.4"
}
},
+ "node_modules/ansi-regex": {
+ "version": "5.0.1",
+ "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
+ "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==",
+ "engines": {
+ "node": ">=8"
+ }
+ },
"node_modules/before-after-hook": {
"version": "2.2.3",
"resolved": "https://registry.npmjs.org/before-after-hook/-/before-after-hook-2.2.3.tgz",
@@ -169,6 +176,17 @@
"resolved": "https://registry.npmjs.org/bottleneck/-/bottleneck-2.19.5.tgz",
"integrity": "sha512-VHiNCbI1lKdl44tGrhNfU3lup0Tj/ZBMJB5/2ZbNXRCPuRCO7ed2mgcK4r17y+KB2EfuYuRaVlwNbAeaWGSpbw=="
},
+ "node_modules/cli-progress": {
+ "version": "3.12.0",
+ "resolved": "https://registry.npmjs.org/cli-progress/-/cli-progress-3.12.0.tgz",
+ "integrity": "sha512-tRkV3HJ1ASwm19THiiLIXLO7Im7wlTuKnvkYaTkyoAPefqjNg7W7DHKUlGRxy9vxDvbyCYQkQozvptuMkGCg8A==",
+ "dependencies": {
+ "string-width": "^4.2.3"
+ },
+ "engines": {
+ "node": ">=4"
+ }
+ },
"node_modules/cross-fetch": {
"version": "3.1.8",
"resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.8.tgz",
@@ -182,6 +200,19 @@
"resolved": "https://registry.npmjs.org/deprecation/-/deprecation-2.3.1.tgz",
"integrity": "sha512-xmHIy4F3scKVwMsQ4WnVaS8bHOx0DmVwRywosKhaILI0ywMDWPtBSku2HNxRvF7jtwDRsoEwYQSfbxj8b7RlJQ=="
},
+ "node_modules/emoji-regex": {
+ "version": "8.0.0",
+ "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
+ "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="
+ },
+ "node_modules/is-fullwidth-code-point": {
+ "version": "3.0.0",
+ "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
+ "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==",
+ "engines": {
+ "node": ">=8"
+ }
+ },
"node_modules/meilisearch": {
"version": "0.36.0",
"resolved": "https://registry.npmjs.org/meilisearch/-/meilisearch-0.36.0.tgz",
@@ -217,6 +248,30 @@
"wrappy": "1"
}
},
+ "node_modules/string-width": {
+ "version": "4.2.3",
+ "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",
+ "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==",
+ "dependencies": {
+ "emoji-regex": "^8.0.0",
+ "is-fullwidth-code-point": "^3.0.0",
+ "strip-ansi": "^6.0.1"
+ },
+ "engines": {
+ "node": ">=8"
+ }
+ },
+ "node_modules/strip-ansi": {
+ "version": "6.0.1",
+ "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz",
+ "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==",
+ "dependencies": {
+ "ansi-regex": "^5.0.1"
+ },
+ "engines": {
+ "node": ">=8"
+ }
+ },
"node_modules/tr46": {
"version": "0.0.3",
"resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
diff --git a/scripts/indexer/package.json b/scripts/indexer/package.json
@@ -8,14 +8,15 @@
"author": "",
"license": "AGPL-3.0-or-later",
"devDependencies": {
+ "@types/cli-progress": "^3.11.5",
"@types/node": "^20.10.3",
"typescript": "^5.3.2"
},
"dependencies": {
"@octokit/core": "^5.0.2",
- "@octokit/plugin-paginate-graphql": "^4.0.0",
"@octokit/plugin-paginate-rest": "^9.1.5",
"@octokit/plugin-throttling": "^8.1.3",
+ "cli-progress": "^3.12.0",
"meilisearch": "^0.36.0"
}
}
diff --git a/scripts/indexer/src/index.ts b/scripts/indexer/src/index.ts
@@ -1,9 +1,8 @@
import { MeiliSearch } from 'meilisearch'
import { Octokit } from '@octokit/core';
import { throttling } from '@octokit/plugin-throttling'
-import type { GraphQlQueryResponseData } from "@octokit/graphql";
-import { paginateGraphql } from "@octokit/plugin-paginate-graphql";
import { paginateRest } from "@octokit/plugin-paginate-rest";
+import { MultiBar, Presets } from 'cli-progress';
//const INDEX = "MSCs";
const INDEX = "MSC_development";
@@ -16,6 +15,15 @@ function wait(ms: number) {
});
}
+const multibar = new MultiBar({
+ clearOnComplete: false,
+ hideCursor: true,
+ format: ' {name} {bar} | {percentage} - {value}/{total} - {eta_formatted}',
+}, Presets.shades_classic);
+const prsBar = multibar.create(1, 0, { name: "PRs" });
+const commentsBar = multibar.create(1, 0, { name: "Comments" });
+const indexingBar = multibar.create(1, 0, { name: "Indexing" });
+
interface Comment {
url: string;
pull_request_review_id?: number;
@@ -73,19 +81,18 @@ interface Threads {
interface Document {
uid: number;
author: string;
+ author_url: string;
body: string;
- closed: boolean;
closedAt: number;
createdAt: number;
- merged: boolean;
mergedAt: number;
number: number;
permalink: string;
title: string;
state: string;
updatedAt: number;
- threads: Threads;
- comments: Comment[];
+ threads?: Threads;
+ comments?: Comment[];
labels: { name: string; color: string; }[];
}
@@ -100,71 +107,39 @@ const client = new MeiliSearch({
}
})
-const MyOctokit = Octokit.plugin(throttling, paginateGraphql, paginateRest);
+const MyOctokit = Octokit.plugin(throttling, paginateRest);
const octokit = new MyOctokit({
auth: process.env.GIT_SECRET,
throttle: {
onRateLimit: (retryAfter, options, octokit, retryCount) => {
- octokit.log.warn(
+ console.warn(
`Request quota exhausted for request ${options.method} ${options.url}`,
);
- if (retryCount < 25) {
- // only retries once
- octokit.log.info(`Retrying after ${retryAfter} seconds!`);
+ if (retryCount <= 5) {
+ // retry 5 times
+ console.info(`Retrying after ${retryAfter} seconds!`);
return true;
}
},
onSecondaryRateLimit: (retryAfter, options, octokit) => {
// does not retry, only logs a warning
- octokit.log.warn(
+ console.warn(
`SecondaryRateLimit detected for request ${options.method} ${options.url}`,
);
},
- },
-});
-const prIterator = await octokit.graphql.paginate.iterator<GraphQlQueryResponseData>(
- `query paginate($cursor: String) {
- repository(owner: "matrix-org", name: "matrix-spec-proposals") {
- pullRequests(first: 100, after: $cursor) {
- nodes {
- author {
- login
- }
- body
- closed
- closedAt
- createdAt
- merged
- mergedAt
- number
- permalink
- title
- state
- updatedAt
- }
- pageInfo {
- hasNextPage
- endCursor
- }
- }
- }
- }`, {
- cursor: process.env.PREV_CURSOR ?? undefined
+ },
});
-let prs = 0;
-let lastCursor = "unknown";
-// client.deleteIndex(INDEX)
+//client.deleteIndex(INDEX)
await client.index(INDEX).updateDisplayedAttributes([
'author',
+ 'author_url',
'body',
- 'closed',
"closedAt",
"createdAt",
- "merged",
"mergedAt",
"number",
"permalink",
@@ -189,8 +164,6 @@ await client.index(INDEX).updateSynonyms(synonyms)
await client.index(INDEX).updateFilterableAttributes([
'author',
'state',
- 'merged',
- 'closed',
'closedAt',
'createdAt',
'mergedAt',
@@ -198,66 +171,86 @@ await client.index(INDEX).updateFilterableAttributes([
"labels"])
await client.index(INDEX).updateSortableAttributes(['closedAt', 'createdAt', 'mergedAt', 'updatedAt'])
-for await (const response of prIterator) {
- const resp: GraphQlQueryResponseData = response;
- const nodes = resp.repository.pullRequests.nodes;
- prs += nodes.length;
- console.log(`${prs} prs found.`);
-
- const documents = await Promise.allSettled(nodes.map(async (node: any): Promise<Document> => {
- const author = node.author ? node.author.login : "unknown author";
- const closedAt = dateToTimestamp(new Date(node.closedAt));
- const createdAt = dateToTimestamp(new Date(node.createdAt));
- const mergedAt = dateToTimestamp(new Date(node.mergedAt));
- const updatedAt = dateToTimestamp(new Date(node.updatedAt));
+async function wait_for_rate_limit() {
+ // Wait for github to reset the rate limit by fetching the rate limit endpoint and waiting until the reset time
+ const rate_limit = await octokit.request('GET /rate_limit');
+ const reset_time = rate_limit.data.resources.core.reset;
+ const current_time = Math.floor(Date.now() / 1000);
+ const wait_time = reset_time - current_time;
+ multibar.log(`Waiting for ${wait_time} seconds until the rate limit is reset.`)
+ await wait(wait_time * 1000);
+}
- const { threads, comments } = await get_comments(node.number);
- const labels = await get_labels(node.number);
- console.log(`PR ${node.number} has ${labels.length} labels.`);
- return {
- uid: node.number,
- author: author,
- body: node.body,
- closed: node.closed,
- closedAt: closedAt,
- createdAt: createdAt,
- merged: node.merged,
- mergedAt: mergedAt,
- number: node.number,
- permalink: node.permalink,
- title: node.title,
- state: node.state,
- updatedAt: updatedAt,
- threads: threads,
- comments: comments,
- labels: labels
+async function get_documents(): Promise<Document[]> {
+ let prs = 0;
+ let documents: Document[] = [];
+ const prIterator = octokit.paginate.iterator(
+ "GET /repos/{owner}/{repo}/pulls",
+ {
+ owner: OWNER,
+ repo: REPO,
+ per_page: 100,
+ state: "all",
}
- }));
- await client.index(INDEX).addDocuments(documents);
+ );
+
+ let last_added = 0;
+ for await (const response of prIterator) {
+ const nodes = response.data;
+ prs += nodes.length;
+ prsBar.increment(last_added);
+ prsBar.setTotal(prs);
+ const new_documents = await Promise.allSettled(nodes.map(async (node: any): Promise<Document> => {
+ const author = node.user ? node.user.login : "unknown author";
+ const author_url = node.user ? node.user.url : undefined;
+ const closedAt = dateToTimestamp(new Date(node.closed_at));
+ const createdAt = dateToTimestamp(new Date(node.created_at));
+ const mergedAt = dateToTimestamp(new Date(node.merged_at));
+ const updatedAt = dateToTimestamp(new Date(node.updated_at));
- lastCursor = resp.repository.pullRequests.pageInfo.endCursor;
- // Dont overload meilisearch
- await wait(5000);
+ const labels = await get_labels(node.labels);
+ return {
+ uid: node.number,
+ author: author,
+ author_url: author_url,
+ body: node.body,
+ closedAt: closedAt,
+ createdAt: createdAt,
+ mergedAt: mergedAt,
+ number: node.number,
+ permalink: node.url,
+ title: node.title,
+ state: node.state,
+ updatedAt: updatedAt,
+ labels: labels
+ }
+ }));
+ documents = documents.concat(new_documents.filter((x) => x.status === "fulfilled").map((x) => (x as PromiseFulfilledResult<Document>).value));
+ last_added = new_documents.length;
+ prsBar.updateETA();
+ }
+
+ prsBar.increment(last_added);
+ commentsBar.setTotal(documents.length);
+ indexingBar.setTotal(documents.length);
+
+ for (let i = 0; i < documents.length; i++) {
+ const document = documents[i];
+ const { threads, comments } = await get_comments(document.number);
+ document.threads = threads;
+ document.comments = comments;
+ commentsBar.increment();
+ commentsBar.updateETA();
+ }
+ return documents;
}
-console.log(`Last Cursor:`, lastCursor);
-const stats = await client.getStats();
-console.log(`Stats:`, JSON.stringify(stats, null, 2));
function dateToTimestamp(date: Date): number {
return date.getTime() / 1000;
}
-async function get_labels(pr_id: number): Promise<{ name: string; color: string; }[]> {
- const labels = await octokit.paginate(
- "GET /repos/{owner}/{repo}/issues/{issue_number}/labels",
- {
- owner: OWNER,
- repo: REPO,
- issue_number: pr_id,
- per_page: 100,
- }
- );
+async function get_labels(labels: { id: number, node_id: string, url: string, name: string, description: string, color: string, default: boolean }[]): Promise<{ name: string; color: string; }[]> {
return labels.map((label: any) => {
return {
name: label.name,
@@ -339,10 +332,31 @@ async function get_comments(pr_id: number): Promise<{ threads: Threads, comments
comments_aggregated.push(clean_comment)
}
}
+ await wait(150);
}
- console.log(`${Object.keys(threads).length} Threads and ${comments_aggregated.length} free standing comments in PR with number ${pr_id}`)
return {
threads: threads,
comments: comments_aggregated
}
}
+
+async function main() {
+ await wait_for_rate_limit();
+ const documents = await get_documents();
+
+ // Add documents in bulks of 1000
+ for (let i = 0; i < documents.length; i += 1000) {
+ const documents_to_add = documents.slice(i, i + 1000);
+ await client.index(INDEX).addDocuments(documents_to_add, { primaryKey: 'uid' });
+ indexingBar.increment(documents_to_add.length);
+ indexingBar.updateETA();
+ // Dont overload meilisearch
+ await wait(5 * 60 * 1000);
+ }
+
+ multibar.stop();
+ const stats = await client.getStats();
+ console.log(`Stats:`, JSON.stringify(stats, null, 2));
+}
+
+await main();
diff --git a/src/components/Checkbox.astro b/src/components/Checkbox.astro
@@ -3,13 +3,14 @@ interface Props {
name: string;
label: string;
checked?: boolean;
+ disabled?: boolean;
}
-const { name, label, checked = false } = Astro.props;
+const { name, label, checked = false, disabled = false } = Astro.props;
---
-<label class="form-control">
- <input type="checkbox" name={name} checked={checked} />
+<label class={disabled ? "form-control--disabled" : "form-control"}>
+ <input type="checkbox" name={name} checked={checked} disabled={disabled} />
{label}
</label>
@@ -58,6 +59,16 @@ const { name, label, checked = false } = Astro.props;
outline-offset: max(2px, 0.15em);
}
+ input[type="checkbox"]:disabled {
+ color: #4b5563;
+ cursor: not-allowed;
+ }
+
+ .form-control--disabled {
+ color: #4b5563;
+ cursor: not-allowed;
+ }
+
.form-control {
font-size: 18px;
font-weight: 400;
diff --git a/src/pages/results.astro b/src/pages/results.astro
@@ -191,6 +191,51 @@ function filter_to_string(filter: FilterJson): string {
delete filter_copy.start_merged_closed_date;
delete filter_copy.end_merged_closed_date;
+ // Convert the date filters to a string
+ let date_filter = "(";
+ if (start_opened_date !== undefined) {
+ if (date_filter === "(") {
+ date_filter += `created_at >= ${start_opened_date}`;
+ } else {
+ date_filter += ` AND created_at >= ${start_opened_date}`;
+ }
+ }
+
+ if (end_opened_date !== undefined) {
+ if (date_filter === "(") {
+ date_filter += `created_at <= ${end_opened_date}`;
+ } else {
+ date_filter += ` AND created_at <= ${end_opened_date}`;
+ }
+ }
+
+ if (start_merged_closed_date !== undefined) {
+ if (date_filter === "(") {
+ date_filter += `(merged_at >= ${start_merged_closed_date} AND merged_at != null) OR (closed_at >= ${start_merged_closed_date} AND closed_at != null)`;
+ } else {
+ date_filter += ` AND (merged_at >= ${start_merged_closed_date} AND merged_at != null) OR (closed_at >= ${start_merged_closed_date} AND closed_at != null)`;
+ }
+ }
+
+ if (end_merged_closed_date !== undefined) {
+ if (date_filter === "(") {
+ date_filter += `(merged_at <= ${end_merged_closed_date} AND merged_at != null) OR (closed_at <= ${end_merged_closed_date} AND closed_at != null)`;
+ } else {
+ date_filter += ` AND (merged_at <= ${end_merged_closed_date} AND merged_at != null) OR (closed_at <= ${end_merged_closed_date} AND closed_at != null)`;
+ }
+ }
+
+ // Close the date filter
+ if (date_filter !== "") {
+ date_filter += ")";
+ }
+
+ if (filter_string !== "" && date_filter !== "()") {
+ filter_string += ` AND ${date_filter}`;
+ } else if (filter_string === "" && date_filter !== "()") {
+ filter_string += date_filter;
+ }
+
// Convert the rest of the filter to a string
let label_filter = "(";
for (const key in filter_copy) {
@@ -276,21 +321,21 @@ function filter_to_string(filter: FilterJson): string {
<Checkbox name="server_server" label="Server-Server" checked={filter_parsed.server_server} />
<Checkbox name="application_services" label="Application Services" checked={filter_parsed.application_services} />
<Checkbox name="e2e" label="E2EE" checked={filter_parsed.e2e} />
- <Checkbox name="kind_core" label="kind:core" checked={filter_parsed.kind_core} />
- <Checkbox name="kind_feature" label="kind:feature" checked={filter_parsed.kind_feature} />
- <Checkbox name="kind_maintenance" label="kind:maintenance" checked={filter_parsed.kind_maintenance} />
- <Checkbox name="needs_implementation" label="needs-implementation" checked={filter_parsed.needs_implementation} />
- <Checkbox name="proposal" label="proposal" checked={filter_parsed.proposal} />
- <Checkbox name="abandoned" label="abandoned" checked={filter_parsed.abandoned} />
- <Checkbox name="blocked" label="blocked" checked={filter_parsed.blocked} />
+ <Checkbox name="kind_core" label="Kind: Core" checked={filter_parsed.kind_core} />
+ <Checkbox name="kind_feature" label="Kind: Feature" checked={filter_parsed.kind_feature} />
+ <Checkbox name="kind_maintenance" label="Kind: Maintenance" checked={filter_parsed.kind_maintenance} />
+ <Checkbox name="needs_implementation" label="Needs Implementation" checked={filter_parsed.needs_implementation} />
+ <Checkbox name="proposal" label="Proposal" checked={filter_parsed.proposal} />
+ <Checkbox name="abandoned" label="Abandoned" checked={filter_parsed.abandoned} />
+ <Checkbox name="blocked" label="Blocked" checked={filter_parsed.blocked} />
</div>
</div>
<div class="filter-section">
<h1>State</h1>
<div class="filter-options">
- <Checkbox name="merged" label="merged" checked={filter_parsed.state?.merged} />
- <Checkbox name="closed" label="closed" checked={filter_parsed.state?.closed} />
- <Checkbox name="other" label="other" checked={filter_parsed.state?.other} />
+ <Checkbox name="merged" label="Merged" checked={filter_parsed.state?.merged} />
+ <Checkbox name="closed" label="Closed" checked={filter_parsed.state?.closed} />
+ <Checkbox name="other" label="Other" checked={filter_parsed.state?.other} />
</div>
</div>
