|
|
|
/* eslint-disable no-await-in-loop */
|
|
|
|
import { omit } from 'lodash';
|
|
|
|
import { Index, MeiliSearch } from 'meilisearch';
|
|
|
|
import DomainModel from 'hydrooj/src/model/domain';
|
|
|
|
import ProblemModel from 'hydrooj/src/model/problem';
|
|
|
|
import * as system from 'hydrooj/src/model/system';
|
|
|
|
import { iterateAllProblem } from 'hydrooj/src/pipelineUtils';
|
|
|
|
import * as bus from 'hydrooj/src/service/bus';
|
|
|
|
import { sleep } from 'hydrooj/src/utils';
|
|
|
|
|
|
|
|
const client = new MeiliSearch({
|
|
|
|
host: system.get('meilisearch.url') || 'http://127.0.0.1:7700',
|
|
|
|
apiKey: system.get('meilisearch.masterKey'),
|
|
|
|
});
|
|
|
|
|
|
|
|
const index: Index<{ id: string, title?: string, content?: string }> = client.index('problem');
|
|
|
|
const indexOmit = ['_id', 'docType', 'data', 'additional_file', 'config', 'stats', 'assign'];
|
|
|
|
|
|
|
|
bus.on('problem/add', async (doc, docId) => {
|
|
|
|
await index.addDocuments([{
|
|
|
|
id: `${doc.domainId}-${docId}`,
|
|
|
|
...omit(doc, indexOmit),
|
|
|
|
}]);
|
|
|
|
});
|
|
|
|
bus.on('problem/edit', async (pdoc) => {
|
|
|
|
await index.updateDocuments([{
|
|
|
|
id: `${pdoc.domainId}-${pdoc.docId}`,
|
|
|
|
...omit(pdoc, indexOmit),
|
|
|
|
}]);
|
|
|
|
});
|
|
|
|
bus.on('problem/del', async (domainId, docId) => {
|
|
|
|
await index.deleteDocument(`${domainId}-${docId}`);
|
|
|
|
});
|
|
|
|
|
|
|
|
global.Hydro.lib.problemSearch = async (domainId, q, opts) => {
|
|
|
|
const allowedSize = system.get('meilisearch.indexSize') || 10000;
|
|
|
|
const limit = opts?.limit || system.get('pagination.problem');
|
|
|
|
const offset = Math.min(allowedSize - limit, opts?.skip || 0);
|
|
|
|
const union = await DomainModel.getUnion(domainId);
|
|
|
|
const domainIds = [domainId, ...(union?.union || [])];
|
|
|
|
const res = await index.search(q, {
|
|
|
|
offset,
|
|
|
|
limit,
|
|
|
|
filter: domainIds.map((i) => `domainId = ${i}`).join(' OR '),
|
|
|
|
});
|
|
|
|
return {
|
|
|
|
countRelation: 'eq',
|
|
|
|
total: res.estimatedTotalHits,
|
|
|
|
hits: res.hits.map((i) => i.id.replace('-', '/')),
|
|
|
|
};
|
|
|
|
};
|
|
|
|
|
|
|
|
export const description = 'Meilisearch reindex';
|
|
|
|
|
|
|
|
export const validate = {
|
|
|
|
$or: [
|
|
|
|
{ domainId: 'string' },
|
|
|
|
{ domainId: 'undefined' },
|
|
|
|
],
|
|
|
|
};
|
|
|
|
|
|
|
|
async function waitForTask(id: number) {
|
|
|
|
// eslint-disable-next-line no-constant-condition
|
|
|
|
while (true) {
|
|
|
|
const status = await client.getTask(id);
|
|
|
|
if (status.error) throw new Error(status.error.message);
|
|
|
|
if (status.status !== 'processing') break;
|
|
|
|
await sleep(1000);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
export async function run(_, report) {
|
|
|
|
await index.addDocuments([{ id: '1' }]);
|
|
|
|
let task = await index.deleteAllDocuments();
|
|
|
|
await waitForTask(task.taskUid);
|
|
|
|
report({ message: 'old index deleted' });
|
|
|
|
task = await index.updateFilterableAttributes(['domainId', 'tag']);
|
|
|
|
await waitForTask(task.taskUid);
|
|
|
|
report({ message: 'filterable attributes updated' });
|
|
|
|
task = await index.updateDisplayedAttributes(['id', 'title', 'tag']);
|
|
|
|
await waitForTask(task.taskUid);
|
|
|
|
report({ message: 'displayed attributes updated' });
|
|
|
|
task = await index.updateSearchableAttributes(['title', 'tag']);
|
|
|
|
await waitForTask(task.taskUid);
|
|
|
|
report({ message: 'searchable attributes updated' });
|
|
|
|
task = await index.updateSortableAttributes(['domainId']);
|
|
|
|
await waitForTask(task.taskUid);
|
|
|
|
report({ message: 'sortable attributes updated' });
|
|
|
|
let i = 0;
|
|
|
|
let documents: any[] = [];
|
|
|
|
const cb = async (pdoc) => {
|
|
|
|
i++;
|
|
|
|
if (!(i % 1000)) {
|
|
|
|
task = await index.addDocuments(documents);
|
|
|
|
await waitForTask(task.taskUid);
|
|
|
|
documents = [];
|
|
|
|
report({ message: `${i} problems indexed` });
|
|
|
|
}
|
|
|
|
documents.push({
|
|
|
|
id: `${pdoc.domainId}-${pdoc.docId}`,
|
|
|
|
...omit(pdoc, indexOmit),
|
|
|
|
});
|
|
|
|
};
|
|
|
|
await iterateAllProblem(ProblemModel.PROJECTION_PUBLIC, cb);
|
|
|
|
if (documents.length) {
|
|
|
|
task = await index.addDocuments(documents);
|
|
|
|
await waitForTask(task.taskUid);
|
|
|
|
}
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
global.Hydro.script.ensureMeiliSearch = { run, description, validate };
|