From f1e24099912a8f85f27a9673f4bc14e0cbb6aedc Mon Sep 17 00:00:00 2001 From: undefined Date: Sun, 12 Sep 2021 05:28:38 +0800 Subject: [PATCH] core: optimize query speed --- build/prepare.js | 2 +- package.json | 4 +- packages/hydrooj/package.json | 4 +- packages/hydrooj/src/handler/contest.ts | 18 ++--- packages/hydrooj/src/handler/domain.ts | 56 +++------------ packages/hydrooj/src/handler/home.ts | 16 ++--- packages/hydrooj/src/handler/record.ts | 7 +- packages/hydrooj/src/interface.ts | 11 +-- packages/hydrooj/src/model/builtin.ts | 16 ++--- packages/hydrooj/src/model/contest.ts | 2 +- packages/hydrooj/src/model/discussion.ts | 4 +- packages/hydrooj/src/model/document.ts | 73 +++++++++----------- packages/hydrooj/src/model/domain.ts | 12 +++- packages/hydrooj/src/model/record.ts | 3 +- packages/hydrooj/src/model/storage.ts | 14 ++-- packages/hydrooj/src/model/task.ts | 1 - packages/hydrooj/src/script/rating.ts | 2 +- packages/hydrooj/src/service/__mocks__/db.ts | 3 - packages/hydrooj/src/service/db.ts | 12 +++- packages/hydrooj/src/service/server.ts | 3 +- 20 files changed, 116 insertions(+), 147 deletions(-) diff --git a/build/prepare.js b/build/prepare.js index 19d04a19..2fe5b1cf 100644 --- a/build/prepare.js +++ b/build/prepare.js @@ -2,7 +2,7 @@ const fs = require('fs'); const path = require('path'); const compilerOptionsBase = { - target: 'es2019', + target: 'es2020', module: 'commonjs', esModuleInterop: true, moduleResolution: 'node', diff --git a/package.json b/package.json index c4571fec..4e1ed386 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "build": "node build/prepare.js && tsc -b --verbose", "build:watch": "node build/prepare.js && tsc -b --watch", "build:ui": "node packages/ui-default/build", - "build:ui:dev": "node packages/ui-default/build --dev", + "build:ui:dev": "node --trace-deprecation packages/ui-default/build --dev", "build:ui:production": "node packages/ui-default/build --production", "lint": "eslint packages --ext ts --fix", "lint:ci": "eslint packages --ext ts", @@ -48,7 +48,7 @@ "nmls": "^3.0.1", "ora": "5.4.1", "semver": "^7.3.5", - "typedoc": "^0.21.9", + "typedoc": "^0.22.2", "typescript": "4.4.2" } } diff --git a/packages/hydrooj/package.json b/packages/hydrooj/package.json index b1f0b8db..0825ab52 100644 --- a/packages/hydrooj/package.json +++ b/packages/hydrooj/package.json @@ -1,6 +1,6 @@ { "name": "hydrooj", - "version": "2.34.4", + "version": "2.34.5", "bin": "bin/hydrooj.js", "main": "src/loader", "module": "src/loader", @@ -31,7 +31,7 @@ "lodash": "^4.17.21", "lru-cache": "^6.0.0", "mime-types": "^2.1.32", - "minio": "7.0.17", + "minio": "7.0.19", "moment-timezone": "^0.5.33", "mongodb": "^3.7.0", "nanoid": "^3.1.25", diff --git a/packages/hydrooj/src/handler/contest.ts b/packages/hydrooj/src/handler/contest.ts index 6460f812..7af2bdb0 100644 --- a/packages/hydrooj/src/handler/contest.ts +++ b/packages/hydrooj/src/handler/contest.ts @@ -5,13 +5,13 @@ import moment from 'moment-timezone'; import { ObjectID } from 'mongodb'; import { Time } from '@hydrooj/utils/lib/utils'; import { - BadRequestError, - ContestNotAttendedError, ContestNotFoundError, ContestNotLiveError, InvalidTokenError, PermissionError, ProblemNotFoundError, - RecordNotFoundError, - ValidationError } from '../error'; + BadRequestError, ContestNotAttendedError, ContestNotFoundError, + ContestNotLiveError, InvalidTokenError, PermissionError, + ProblemNotFoundError, RecordNotFoundError, ValidationError, +} from '../error'; import { - DomainDoc, - ProblemDoc, Tdoc, User } from '../interface'; + DomainDoc, ProblemDoc, Tdoc, User, +} from '../interface'; import paginate from '../lib/paginate'; import { PERM, PRIV } from '../model/builtin'; import * as contest from '../model/contest'; @@ -24,15 +24,15 @@ import * as system from '../model/system'; import user from '../model/user'; import * as bus from '../service/bus'; import { - Handler, param, - Route, Types } from '../service/server'; + Handler, param, Route, Types, +} from '../service/server'; import storage from '../service/storage'; export class ContestListHandler extends Handler { @param('rule', Types.Range(contest.RULES), true) @param('page', Types.PositiveInt, true) async get(domainId: string, rule = '', page = 1) { - const cursor = contest.getMulti(domainId, rule ? { rule } : undefined).sort({ beginAt: -1 }); + const cursor = contest.getMulti(domainId, rule ? { rule } : undefined); const qs = rule ? `rule=${rule}` : ''; const [tdocs, tpcount] = await paginate(cursor, page, system.get('pagination.contest')); const tids = []; diff --git a/packages/hydrooj/src/handler/domain.ts b/packages/hydrooj/src/handler/domain.ts index dddfaacc..c8f1221c 100644 --- a/packages/hydrooj/src/handler/domain.ts +++ b/packages/hydrooj/src/handler/domain.ts @@ -1,9 +1,9 @@ import { Dictionary } from 'lodash'; import moment from 'moment-timezone'; import { - DomainJoinAlreadyMemberError, DomainJoinForbiddenError, - InvalidJoinInvitationCodeError, - RoleAlreadyExistError, ValidationError } from '../error'; + DomainJoinAlreadyMemberError, DomainJoinForbiddenError, InvalidJoinInvitationCodeError, + RoleAlreadyExistError, ValidationError, +} from '../error'; import type { DomainDoc } from '../interface'; import avatar from '../lib/avatar'; import paginate from '../lib/paginate'; @@ -16,15 +16,15 @@ import { DOMAIN_SETTINGS, DOMAIN_SETTINGS_BY_KEY } from '../model/setting'; import * as system from '../model/system'; import user from '../model/user'; import { - Handler, param, post, - query, Route, Types } from '../service/server'; + Handler, param, post, query, Route, Types, +} from '../service/server'; import { log2 } from '../utils'; class DomainRankHandler extends Handler { @query('page', Types.PositiveInt, true) async get(domainId: string, page = 1) { const [dudocs, upcount, ucount] = await paginate( - domain.getMultiUserInDomain(domainId, { uid: { $nin: [0, 1] } }).sort({ rp: -1 }), + domain.getMultiUserInDomain(domainId, { uid: { $gt: 1 } }).sort({ rp: -1 }), page, 100, ); @@ -33,13 +33,9 @@ class DomainRankHandler extends Handler { udocs.push(user.getById(domainId, dudoc.uid)); } udocs = await Promise.all(udocs); - const path = [ - ['Hydro', 'homepage'], - ['ranking', null], - ]; this.response.template = 'ranking.html'; this.response.body = { - udocs, upcount, ucount, page, path, + udocs, upcount, ucount, page, }; } } @@ -55,13 +51,8 @@ class ManageHandler extends Handler { class DomainEditHandler extends ManageHandler { async get() { - const path = [ - ['Hydro', 'homepage'], - ['domain', null], - ['domain_edit', null], - ]; this.response.template = 'domain_edit.html'; - this.response.body = { current: this.domain, settings: DOMAIN_SETTINGS, path }; + this.response.body = { current: this.domain, settings: DOMAIN_SETTINGS }; } async post(args) { @@ -76,13 +67,8 @@ class DomainEditHandler extends ManageHandler { class DomainDashboardHandler extends ManageHandler { async get() { - const path = [ - ['Hydro', 'homepage'], - ['domain', null], - ['domain_dashboard', null], - ]; this.response.template = 'domain_dashboard.html'; - this.response.body = { domain: this.domain, path }; + this.response.body = { domain: this.domain }; } async postInitDiscussionNode({ domainId }) { @@ -165,9 +151,7 @@ class DomainPermissionHandler extends ManageHandler { const perms = this.request.body[role] instanceof Array ? this.request.body[role] : [this.request.body[role]]; - // @ts-expect-error roles[role] = 0n; - // @ts-expect-error for (const r of perms) roles[role] |= 1n << BigInt(r); } await domain.setRoles(domainId, roles); @@ -178,13 +162,8 @@ class DomainPermissionHandler extends ManageHandler { class DomainRoleHandler extends ManageHandler { async get({ domainId }) { const roles = await domain.getRoles(domainId, true); - const path = [ - ['Hydro', 'homepage'], - ['domain', null], - ['domain_role', null], - ]; this.response.template = 'domain_role.html'; - this.response.body = { roles, domain: this.domain, path }; + this.response.body = { roles, domain: this.domain }; } @param('role', Types.Name) @@ -220,11 +199,6 @@ class DomainJoinApplicationsHandler extends ManageHandler { delete this.response.body.expirations[domain.JOIN_EXPIRATION_KEEP_CURRENT]; } this.response.body.url_prefix = (this.domain.host || [])[0] || system.get('server.url'); - this.response.body.path = [ - ['Hydro', 'homepage'], - ['domain', null], - ['domain_join_applications', null], - ]; this.response.template = 'domain_join_applications.html'; } @@ -255,11 +229,7 @@ class DomainJoinApplicationsHandler extends ManageHandler { class DomainJoinHandler extends Handler { joinSettings: any; - - constructor(ctx) { - super(ctx); - this.noCheckPermView = true; - } + noCheckPermView = true; async prepare() { const r = await domain.getRoles(this.domain); @@ -274,10 +244,6 @@ class DomainJoinHandler extends Handler { this.response.template = 'domain_join.html'; this.response.body.joinSettings = this.joinSettings; this.response.body.code = code; - this.response.body.path = [ - ['Hydro', 'homepage'], - ['domain_join', 'domain_join', { domainId, code }], - ]; } @param('code', Types.Content, true) diff --git a/packages/hydrooj/src/handler/home.ts b/packages/hydrooj/src/handler/home.ts index 73b96df5..8d53e56d 100644 --- a/packages/hydrooj/src/handler/home.ts +++ b/packages/hydrooj/src/handler/home.ts @@ -1,10 +1,10 @@ import yaml from 'js-yaml'; import { ObjectID } from 'mongodb'; import { - BlacklistedError, - DomainAlreadyExistsError, InvalidTokenError, - NotFoundError, PermissionError, - UserAlreadyExistError, UserNotFoundError, ValidationError, VerifyPasswordError } from '../error'; + BlacklistedError, DomainAlreadyExistsError, InvalidTokenError, + NotFoundError, PermissionError, UserAlreadyExistError, + UserNotFoundError, ValidationError, VerifyPasswordError, +} from '../error'; import { DomainDoc, MessageDoc, Setting } from '../interface'; import avatar from '../lib/avatar'; import { md5 } from '../lib/crypto'; @@ -25,7 +25,7 @@ import * as training from '../model/training'; import user from '../model/user'; import * as bus from '../service/bus'; import { - Connection, ConnectionHandler, Handler, param, Route, Types, + Connection, ConnectionHandler, Handler, param, Route, Types, } from '../service/server'; const { geoip, useragent } = global.Hydro.lib; @@ -40,7 +40,7 @@ class HomeHandler extends Handler { async getHomework(domainId: string, limit: number = 5) { if (this.user.hasPerm(PERM.PERM_VIEW_HOMEWORK)) { const tdocs = await contest.getMulti(domainId, {}, document.TYPE_HOMEWORK) - .sort('beginAt', -1).limit(limit).toArray(); + .limit(limit).toArray(); const tsdict = await contest.getListStatus( domainId, this.user._id, tdocs.map((tdoc) => tdoc.docId), document.TYPE_HOMEWORK, @@ -53,7 +53,7 @@ class HomeHandler extends Handler { async getContest(domainId: string, limit: number = 10) { if (this.user.hasPerm(PERM.PERM_VIEW_CONTEST)) { const tdocs = await contest.getMulti(domainId) - .sort('beginAt', -1).limit(limit).toArray(); + .limit(limit).toArray(); const tsdict = await contest.getListStatus( domainId, this.user._id, tdocs.map((tdoc) => tdoc.docId), ); @@ -88,7 +88,7 @@ class HomeHandler extends Handler { async getRanking(domainId: string, limit: number = 50) { if (this.user.hasPerm(PERM.PERM_VIEW_RANKING)) { - const dudocs = await domain.getMultiUserInDomain(domainId, { uid: { $nin: [0, 1] } }) + const dudocs = await domain.getMultiUserInDomain(domainId, { uid: { $gt: 1 } }) .sort({ rp: -1 }).project({ uid: 1 }).limit(limit).toArray(); const uids = dudocs.map((dudoc) => dudoc.uid); this.collectUser(uids); diff --git a/packages/hydrooj/src/handler/record.ts b/packages/hydrooj/src/handler/record.ts index b3f6ceaa..c62d4387 100644 --- a/packages/hydrooj/src/handler/record.ts +++ b/packages/hydrooj/src/handler/record.ts @@ -14,8 +14,8 @@ import TaskModel from '../model/task'; import user from '../model/user'; import * as bus from '../service/bus'; import { - Connection, ConnectionHandler, Handler, param, - Route, Types } from '../service/server'; + Connection, ConnectionHandler, Handler, param, Route, Types, +} from '../service/server'; import { buildProjection } from '../utils'; import { postJudge } from './judge'; @@ -57,9 +57,8 @@ class RecordListHandler extends Handler { if (status) q.status = status; if (all) { this.checkPriv(PRIV.PRIV_MANAGE_ALL_DOMAIN); - q.domainId = { $exists: true }; } - let cursor = record.getMulti(domainId, q).sort('_id', -1); + let cursor = record.getMulti(all ? '' : domainId, q).sort('_id', -1); if (!full) cursor = cursor.project(buildProjection(record.PROJECTION_LIST)); const limit = full ? 10 : system.get('pagination.record'); const rdocs = invalid diff --git a/packages/hydrooj/src/interface.ts b/packages/hydrooj/src/interface.ts index 79a60e85..ae24b3c8 100644 --- a/packages/hydrooj/src/interface.ts +++ b/packages/hydrooj/src/interface.ts @@ -487,11 +487,12 @@ export interface JudgeResultBody { } export interface Task { - _id: ObjectID, - type: string, - executeAfter: Date, - priority: number, - [key: string]: any + _id: ObjectID; + type: string; + subType?: string; + executeAfter: Date; + priority: number; + [key: string]: any; } export interface UploadStream extends Writable { diff --git a/packages/hydrooj/src/model/builtin.ts b/packages/hydrooj/src/model/builtin.ts index 820375e1..37b18a98 100644 --- a/packages/hydrooj/src/model/builtin.ts +++ b/packages/hydrooj/src/model/builtin.ts @@ -1,16 +1,8 @@ -// @ts-nocheck - -/* - * Why nocheck? - * BitInt requires at least ES2020, but we can't use it as - * NodeJS doesn't support some of this new features. - * e.g.: object?.property - */ import { - STATUS, STATUS_CODES, - STATUS_TEXTS, USER_GENDER_FEMALE, USER_GENDER_ICONS, USER_GENDER_MALE, USER_GENDER_OTHER, - USER_GENDER_RANGE, - USER_GENDERS } from '@hydrooj/utils/lib/status'; + STATUS, STATUS_CODES, STATUS_TEXTS, + USER_GENDER_FEMALE, USER_GENDER_ICONS, USER_GENDER_MALE, + USER_GENDER_OTHER, USER_GENDER_RANGE, USER_GENDERS, +} from '@hydrooj/utils/lib/status'; export * from '@hydrooj/utils/lib/status'; diff --git a/packages/hydrooj/src/model/contest.ts b/packages/hydrooj/src/model/contest.ts index b9559d62..c7c3b3a4 100644 --- a/packages/hydrooj/src/model/contest.ts +++ b/packages/hydrooj/src/model/contest.ts @@ -591,7 +591,7 @@ export function count(domainId: string, query: any, type: Type = document.TYPE_C export function getMulti( domainId: string, query: FilterQuery = {}, type?: K, ) { - return document.getMulti(domainId, type || document.TYPE_CONTEST, query); + return document.getMulti(domainId, type || document.TYPE_CONTEST, query).sort({ beginAt: -1 }); } export async function getAndListStatus( diff --git a/packages/hydrooj/src/model/discussion.ts b/packages/hydrooj/src/model/discussion.ts index 7c6c82ae..a9c1c0ea 100644 --- a/packages/hydrooj/src/model/discussion.ts +++ b/packages/hydrooj/src/model/discussion.ts @@ -306,10 +306,10 @@ const t = Math.exp(-0.15); async function updateSort() { const cursor = document.coll.find({ docType: document.TYPE_DISCUSSION }); - // eslint-disable-next-line no-await-in-loop - while (await cursor.hasNext()) { + while (true) { // eslint-disable-next-line no-await-in-loop const data = await cursor.next(); + if (!data) return; // eslint-disable-next-line no-await-in-loop const rCount = await getMultiReply(data.domainId, data.docId).count(); const sort = ((data.sort || 100) + Math.max(rCount - (data.lastRCount || 0), 0) * 10) * t; diff --git a/packages/hydrooj/src/model/document.ts b/packages/hydrooj/src/model/document.ts index 059c77e4..6a1f8b11 100644 --- a/packages/hydrooj/src/model/document.ts +++ b/packages/hydrooj/src/model/document.ts @@ -1,11 +1,10 @@ /* eslint-disable object-curly-newline */ import assert from 'assert'; import { - Cursor, FilterQuery, ObjectID, OnlyFieldsOfType, - UpdateQuery } from 'mongodb'; + Cursor, FilterQuery, ObjectID, OnlyFieldsOfType, UpdateQuery, +} from 'mongodb'; import { - Content, - DiscussionDoc, DiscussionReplyDoc, ProblemDoc, ProblemStatusDoc, Tdoc, TrainingDoc, + Content, DiscussionDoc, DiscussionReplyDoc, ProblemDoc, ProblemStatusDoc, Tdoc, TrainingDoc, } from '../interface'; import * as bus from '../service/bus'; import db from '../service/db'; @@ -402,41 +401,37 @@ export async function revSetStatus( return res.value; } -async function ensureIndexes() { - const ic = coll.createIndex.bind(coll); - const is = collStatus.createIndex.bind(collStatus); - const u = { unique: true }; - const s = { sparse: true }; - await ic({ domainId: 1, docType: 1, docId: 1 }, u); - await ic({ domainId: 1, docType: 1, owner: 1, docId: -1 }); - // For problem - await ic({ domainId: 1, docType: 1, search: 'text', title: 'text' }, s); - await ic({ domainId: 1, docType: 1, tag: 1, docId: 1 }, s); - await ic({ domainId: 1, docType: 1, hidden: 1, tag: 1, docId: 1 }, s); - // For problem solution - await ic({ domainId: 1, docType: 1, parentType: 1, parentId: 1, vote: -1, docId: -1 }, s); - // For discussion - await ic({ domainId: 1, docType: 1, pin: -1, updateAt: -1, docId: -1 }, s); - await ic({ domainId: 1, docType: 1, parentType: 1, parentId: 1, updateAt: -1, docId: -1 }, s); - // Hidden doc - await ic({ domainId: 1, docType: 1, hidden: 1, docId: -1 }, s); - // For contest - await ic({ domainId: 1, docType: 1, pids: 1 }, s); - await ic({ domainId: 1, docType: 1, rule: 1, docId: -1 }, s); - // For training - await ic({ domainId: 1, docType: 1, 'dag.pids': 1 }, s); - await is({ domainId: 1, docType: 1, uid: 1, docId: 1 }, u); - // For rp system - await is({ domainId: 1, docType: 1, docId: 1, status: 1, rid: 1, rp: 1 }, s); - // For contest rule OI - await is({ domainId: 1, docType: 1, docId: 1, score: -1 }, s); - // For contest rule ACM - await is({ domainId: 1, docType: 1, docId: 1, accept: -1, time: 1 }, s); - // For training - await is({ domainId: 1, docType: 1, uid: 1, enroll: 1, docId: 1 }, s); -} - -bus.once('app/started', ensureIndexes); +bus.once('app/started', async () => { + await db.ensureIndexes( + coll, + { key: { domainId: 1, docType: 1, docId: 1 }, name: 'basic', unique: true }, + { key: { domainId: 1, docType: 1, owner: 1, docId: -1 }, name: 'owner' }, + // For problem + { key: { domainId: 1, docType: 1, search: 'text', title: 'text' }, name: 'search', sparse: true }, + { key: { domainId: 1, docType: 1, tag: 1, docId: 1 }, name: 'tag', sparse: true }, + { key: { domainId: 1, docType: 1, hidden: 1, tag: 1, docId: 1 }, name: 'hidden', sparse: true }, + // For problem solution + { key: { domainId: 1, docType: 1, parentType: 1, parentId: 1, vote: -1, docId: -1 }, name: 'solution', sparse: true }, + // For discussion + { key: { domainId: 1, docType: 1, pin: -1, sort: -1, docId: -1 }, name: 'discussionSort', sparse: true }, + { key: { domainId: 1, docType: 1, parentType: 1, parentId: 1, pin: -1, sort: -1, docId: -1 }, name: 'discussionNodeSort', sparse: true }, + // Hidden doc + { key: { domainId: 1, docType: 1, hidden: 1, docId: -1 }, name: 'hiddenDoc', sparse: true }, + // For contest + { key: { domainId: 1, docType: 1, pids: 1 }, name: 'contest', sparse: true }, + { key: { domainId: 1, docType: 1, rule: 1, docId: -1 }, name: 'contestRule', sparse: true }, + // For training + { key: { domainId: 1, docType: 1, 'dag.pids': 1 }, name: 'training', sparse: true }, + ); + await db.ensureIndexes( + collStatus, + { key: { domainId: 1, docType: 1, docId: 1, uid: 1 }, name: 'basic', unique: true }, + { key: { domainId: 1, docType: 1, docId: 1, status: 1, rid: 1, rp: 1 }, name: 'rp', sparse: true }, + { key: { domainId: 1, docType: 1, docId: 1, score: -1 }, name: 'contestRuleOI', sparse: true }, + { key: { domainId: 1, docType: 1, docId: 1, accept: -1, time: 1 }, name: 'contestRuleACM', sparse: true }, + { key: { domainId: 1, docType: 1, uid: 1, enroll: 1, docId: 1 }, name: 'training', sparse: true }, + ); +}); bus.on('domain/delete', (domainId) => Promise.all([ coll.deleteMany({ domainId }), collStatus.deleteMany({ domainId }), diff --git a/packages/hydrooj/src/model/domain.ts b/packages/hydrooj/src/model/domain.ts index 3d03eaef..7a0b7792 100644 --- a/packages/hydrooj/src/model/domain.ts +++ b/packages/hydrooj/src/model/domain.ts @@ -275,6 +275,16 @@ class DomainModel { } } -bus.on('app/started', () => coll.createIndex({ lower: 1 }, { unique: true })); +bus.once('app/started', async () => { + await db.ensureIndexes( + coll, + { key: { lower: 1 }, name: 'lower', unique: true }, + ); + await db.ensureIndexes( + collUser, + { key: { domainId: 1, uid: 1 }, name: 'uid', unique: true }, + { key: { domainId: 1, uid: 1, rp: -1 }, name: 'rp', sparse: true }, + ); +}); export default DomainModel; global.Hydro.model.domain = DomainModel; diff --git a/packages/hydrooj/src/model/record.ts b/packages/hydrooj/src/model/record.ts index 9c8481f4..60c56404 100644 --- a/packages/hydrooj/src/model/record.ts +++ b/packages/hydrooj/src/model/record.ts @@ -143,7 +143,8 @@ class RecordModel { } static getMulti(domainId: string, query: any) { - return RecordModel.coll.find({ domainId, ...query }); + if (domainId) query = { domainId, ...query }; + return RecordModel.coll.find(query); } static async update( diff --git a/packages/hydrooj/src/model/storage.ts b/packages/hydrooj/src/model/storage.ts index 0ef9369d..55a17d7d 100644 --- a/packages/hydrooj/src/model/storage.ts +++ b/packages/hydrooj/src/model/storage.ts @@ -29,7 +29,7 @@ export class StorageModel { static async get(path: string, savePath?: string) { const { value } = await StorageModel.coll.findOneAndUpdate( - { path, autoDelete: { $exists: false } }, + { path, autoDelete: null }, { $set: { lastUsage: new Date() } }, { returnDocument: 'after' }, ); @@ -38,7 +38,7 @@ export class StorageModel { static async rename(path: string, newPath: string) { return await StorageModel.coll.updateOne( - { path, autoDelete: { $exists: false } }, + { path, autoDelete: null }, { $set: { path: newPath } }, ); } @@ -46,7 +46,7 @@ export class StorageModel { static async del(path: string[]) { const autoDelete = moment().add(7, 'day').toDate(); await StorageModel.coll.updateMany( - { path: { $in: path }, autoDelete: { $exists: false } }, + { path: { $in: path }, autoDelete: null }, { $set: { autoDelete } }, ); } @@ -57,11 +57,11 @@ export class StorageModel { const results = recursive ? await StorageModel.coll.find({ path: { $regex: new RegExp(`^${escapeRegExp(target)}`, 'i') }, - autoDelete: { $exists: false }, + autoDelete: null, }).toArray() : await StorageModel.coll.find({ path: { $regex: new RegExp(`^${escapeRegExp(target)}[^/]+$`) }, - autoDelete: { $exists: false }, + autoDelete: null, }).toArray(); return results.map((i) => ({ ...i, name: i.path.split(target)[1], prefix: target, @@ -70,7 +70,7 @@ export class StorageModel { static async getMeta(path: string) { const { value } = await StorageModel.coll.findOneAndUpdate( - { path, autoDelete: { $exists: false } }, + { path, autoDelete: null }, { $set: { lastUsage: new Date() } }, { returnDocument: 'after' }, ); @@ -85,7 +85,7 @@ export class StorageModel { static async signDownloadLink(target: string, filename?: string, noExpire = false, useAlternativeEndpointFor?: 'user' | 'judge') { const res = await StorageModel.coll.findOneAndUpdate( - { path: target, autoDelete: { $exists: false } }, + { path: target, autoDelete: null }, { $set: { lastUsage: new Date() } }, ); return await storage.signDownloadLink(res.value?._id || target, filename, noExpire, useAlternativeEndpointFor); diff --git a/packages/hydrooj/src/model/task.ts b/packages/hydrooj/src/model/task.ts index 8d4bccbc..13482db3 100644 --- a/packages/hydrooj/src/model/task.ts +++ b/packages/hydrooj/src/model/task.ts @@ -93,7 +93,6 @@ class TaskModel { static async add(task: Partial & { type: string }) { const t: Task = { ...task, - count: task.count ?? 1, priority: task.priority ?? 0, executeAfter: task.executeAfter || new Date(), _id: new ObjectID(), diff --git a/packages/hydrooj/src/script/rating.ts b/packages/hydrooj/src/script/rating.ts index 3d715816..1f11ae8c 100644 --- a/packages/hydrooj/src/script/rating.ts +++ b/packages/hydrooj/src/script/rating.ts @@ -131,7 +131,7 @@ async function runInDomain(id: string, isSub: boolean, report: Function) { ], (a, b) => a.uid === b.uid); for (const dudoc of dudocs) deltaudict[dudoc.uid] = dudoc.rpdelta; const contests: Tdoc<30 | 60>[] = await contest.getMulti('', { domainId, rated: true }) - .sort('endAt', -1).toArray() as any; + .toArray() as any; await report({ message: `Found ${contests.length} contests in ${id}` }); for (const i in contests) { const tdoc = contests[i]; diff --git a/packages/hydrooj/src/service/__mocks__/db.ts b/packages/hydrooj/src/service/__mocks__/db.ts index 3f03fb20..995d02f0 100644 --- a/packages/hydrooj/src/service/__mocks__/db.ts +++ b/packages/hydrooj/src/service/__mocks__/db.ts @@ -1,4 +1,3 @@ -// @ts-nocheck import { Collection, Db, MongoClient } from 'mongodb'; import { BaseService, Collections } from '../../interface'; import * as bus from '../bus'; @@ -40,8 +39,6 @@ class MongoService implements BaseService { public async stop() { await this.client.close(); await this.client2.close(); - await this.db.close(); - await this.db2.close(); } } diff --git a/packages/hydrooj/src/service/db.ts b/packages/hydrooj/src/service/db.ts index a3ae1048..b39594d0 100644 --- a/packages/hydrooj/src/service/db.ts +++ b/packages/hydrooj/src/service/db.ts @@ -57,7 +57,17 @@ class MongoService implements BaseService { existed = []; } for (const index of args) { - const i = existed.find(t => t.name == index.name || JSON.stringify(t.key) == JSON.stringify(index.key)); + let i = existed.find(t => t.name == index.name || JSON.stringify(t.key) == JSON.stringify(index.key)); + if (!i && Object.keys(index.key).map(k => index.key[k]).includes('text')) { + i = existed.find(t => t.textIndexVersion); + } + if (i.textIndexVersion) { + const cur = Object.keys(i.key).filter(t => !t.startsWith('_')).map(k => k + ':' + i.key[k]); + for (const key of Object.keys(i.weights)) cur.push(key + ':text'); + const wanted = Object.keys(index.key).map(key => key + ':' + index.key[key]); + if (cur.sort().join(' ') === wanted.sort().join(' ') && i.name === index.name) continue; + } + index.background = true; if (!i) { logger.info('Indexing %s.%s with key %o', coll.collectionName, index.name, index.key); await coll.createIndexes([index]); diff --git a/packages/hydrooj/src/service/server.ts b/packages/hydrooj/src/service/server.ts index 2925c34e..4ad16d64 100644 --- a/packages/hydrooj/src/service/server.ts +++ b/packages/hydrooj/src/service/server.ts @@ -420,7 +420,7 @@ export class Handler extends HandlerCommon { csrfToken: string; loginMethods: any; - noCheckPermView: boolean; + noCheckPermView = false; notUsage: boolean; __param: Record; @@ -467,7 +467,6 @@ export class Handler extends HandlerCommon { const [xff, xhost] = system.getMany(['server.xff', 'server.xhost']); if (xff) this.request.ip = this.request.headers[xff.toLowerCase()] || this.request.ip; if (xhost) this.request.host = this.request.headers[xhost.toLowerCase()] || this.request.host; - this.noCheckPermView = false; } // eslint-disable-next-line class-methods-use-this