From 18fc66ecb7a8dd5b14d9ea772683fdb1d5b78b3a Mon Sep 17 00:00:00 2001 From: undefined Date: Fri, 18 Mar 2022 22:48:23 +0800 Subject: [PATCH] core&ui&elastic: support elastic search and allow search result pagination --- .eslintrc.yaml | 3 + packages/elastic/package.json | 11 ++ packages/elastic/script.ts | 95 +++++++++++++ packages/hydrojudge/src/hosts/index.ts | 1 + packages/hydrojudge/src/log.ts | 1 + packages/hydrooj/bin/commands.ts | 129 ++++++++++++++++++ packages/hydrooj/package.json | 2 +- packages/hydrooj/src/entry/cli.ts | 10 +- packages/hydrooj/src/entry/common.ts | 38 +----- packages/hydrooj/src/entry/worker.ts | 19 +-- packages/hydrooj/src/handler/index.ts | 16 +++ packages/hydrooj/src/handler/problem.ts | 39 ++++-- packages/hydrooj/src/interface.ts | 23 +++- packages/hydrooj/src/lib/index.ts | 16 +++ packages/hydrooj/src/logger.ts | 1 + packages/hydrooj/src/model/contest.ts | 9 +- packages/hydrooj/src/model/index.ts | 19 +++ packages/hydrooj/src/model/task.ts | 2 +- packages/hydrooj/src/script/index.ts | 6 + packages/hydrooj/src/service/__mocks__/db.ts | 43 ------ packages/login-with-osu/README.md | 1 - packages/login-with-osu/lib.ts | 73 ---------- packages/login-with-osu/locales/zh.yaml | 1 - packages/login-with-osu/package.json | 18 --- packages/login-with-osu/setting.yaml | 15 -- packages/migrate-hustoj/script.ts | 41 +++--- packages/sonic/model.ts | 9 +- packages/sonic/package.json | 2 +- packages/ui-default/locales/zh.yaml | 1 + packages/ui-default/package.json | 2 +- .../templates/partials/problem_list.html | 4 +- .../templates/partials/problem_stat.html | 2 +- packages/vjudge/src/providers/uoj.ts | 3 +- 33 files changed, 411 insertions(+), 244 deletions(-) create mode 100644 packages/elastic/package.json create mode 100644 packages/elastic/script.ts create mode 100644 packages/hydrooj/bin/commands.ts create mode 100644 packages/hydrooj/src/handler/index.ts create mode 100644 packages/hydrooj/src/lib/index.ts create mode 100644 packages/hydrooj/src/model/index.ts create mode 100644 packages/hydrooj/src/script/index.ts delete mode 100644 packages/hydrooj/src/service/__mocks__/db.ts delete mode 100644 packages/login-with-osu/README.md delete mode 100644 packages/login-with-osu/lib.ts delete mode 100644 packages/login-with-osu/locales/zh.yaml delete mode 100644 packages/login-with-osu/package.json delete mode 100644 packages/login-with-osu/setting.yaml diff --git a/.eslintrc.yaml b/.eslintrc.yaml index 0cab092f..3019a09e 100644 --- a/.eslintrc.yaml +++ b/.eslintrc.yaml @@ -1,5 +1,6 @@ root: true extends: + - airbnb - airbnb-typescript/base env: jquery: true @@ -16,6 +17,8 @@ ignorePatterns: - public/ - '*.spec.ts' rules: + function-call-argument-newline: 0 + react/function-component-definition: 0 '@typescript-eslint/dot-notation': 0 '@typescript-eslint/no-implied-eval': 0 '@typescript-eslint/no-throw-literal': 0 diff --git a/packages/elastic/package.json b/packages/elastic/package.json new file mode 100644 index 00000000..ae1df2c0 --- /dev/null +++ b/packages/elastic/package.json @@ -0,0 +1,11 @@ +{ + "name": "@hydrooj/elastic-search", + "version": "1.0.0", + "main": "package.json", + "repository": "https://github.com/hydro-dev/Hydro", + "author": "undefined ", + "license": "AGPL-3.0-or-later", + "dependencies": { + "@elastic/elasticsearch": "^8.1.0" + } +} diff --git a/packages/elastic/script.ts b/packages/elastic/script.ts new file mode 100644 index 00000000..3a07edb4 --- /dev/null +++ b/packages/elastic/script.ts @@ -0,0 +1,95 @@ +import { Client } from '@elastic/elasticsearch'; +import { omit } from 'lodash'; +import DomainModel from 'hydrooj/src/model/domain'; +import ProblemModel from 'hydrooj/src/model/problem'; +import * as system from 'hydrooj/src/model/system'; +import { iterateAllProblem, iterateAllProblemInDomain } from 'hydrooj/src/pipelineUtils'; +import * as bus from 'hydrooj/src/service/bus'; + +const client = new Client({ node: system.get('elasticsearch.url') || 'http://192.168.1.82:9200' }); + +const indexOmit = ['_id', 'docType', 'data', 'additional_file', 'config', 'stats', 'assign']; + +bus.on('problem/add', async (doc, docId) => { + await client.index({ + index: 'problem', + id: `${doc.domainId}/${docId}`, + document: omit(doc, indexOmit), + }); +}); +bus.on('problem/edit', async (pdoc) => { + await client.index({ + index: 'problem', + id: `${pdoc.domainId}/${pdoc.docId}`, + document: omit(pdoc, indexOmit), + }); +}); +bus.on('problem/del', async (domainId, docId) => { + await client.delete({ + index: 'problem', + id: `${domainId}/${docId}`, + }); +}); + +global.Hydro.lib.problemSearch = async (domainId, q, opts) => { + const size = opts?.limit || system.get('pagination.problem'); + const from = opts?.skip || 0; + const union = await DomainModel.getUnion(domainId); + const domainIds = [domainId, ...(union?.union || [])]; + const res = await client.search({ + index: 'problem', + size, + from, + query: { + simple_query_string: { + query: q, + fields: ['tag^5', 'title^3', 'content'], + }, + }, + post_filter: { + bool: { + minimum_should_match: 1, + should: domainIds.map((i) => ({ match: { domainId: i } })), + }, + }, + }); + return { + countRelation: typeof res.hits.total === 'number' ? 'eq' : res.hits.total.relation, + total: typeof res.hits.total === 'number' ? res.hits.total : res.hits.total.value, + hits: res.hits.hits.map((i) => i._id), + }; +}; + +export const description = 'Elastic problem search re-index'; + +export async function run({ domainId }, report) { + try { + if (domainId) await client.deleteByQuery({ index: 'problem', query: { match: { domainId } } }); + else await client.deleteByQuery({ index: 'problem', query: { match_all: {} } }); + } catch (e) { + if (!e.message.includes('index_not_found_exception')) throw e; + } + let i = 0; + const cb = async (pdoc) => { + i++; + if (!(i % 1000)) report({ message: `${i} problems indexed` }); + await client.index({ + index: 'problem', + id: `${pdoc.domainId}/${pdoc.docId}`, + document: omit(pdoc, indexOmit), + }); + }; + if (domainId) await iterateAllProblemInDomain(domainId, ProblemModel.PROJECTION_PUBLIC, cb); + else await iterateAllProblem(ProblemModel.PROJECTION_PUBLIC, cb); + await client.indices.refresh({ index: 'problem' }); + return true; +} + +export const validate = { + $or: [ + { domainId: 'string' }, + { domainId: 'undefined' }, + ], +}; + +global.Hydro.script.ensureElasticSearch = { run, description, validate }; diff --git a/packages/hydrojudge/src/hosts/index.ts b/packages/hydrojudge/src/hosts/index.ts index 43f9f622..f8af9b61 100644 --- a/packages/hydrojudge/src/hosts/index.ts +++ b/packages/hydrojudge/src/hosts/index.ts @@ -1,2 +1,3 @@ export { default as vj4 } from './vj4'; +// eslint-disable-next-line no-restricted-exports export { default as hydro, default } from './hydro'; diff --git a/packages/hydrojudge/src/log.ts b/packages/hydrojudge/src/log.ts index 193f384d..611e5da2 100644 --- a/packages/hydrojudge/src/log.ts +++ b/packages/hydrojudge/src/log.ts @@ -68,6 +68,7 @@ export class Logger { public stream: NodeJS.WritableStream = process.stderr; constructor(public name: string, private showDiff = false) { + // eslint-disable-next-line no-constructor-return if (name in instances) return instances[name]; let hash = 0; for (let i = 0; i < name.length; i++) { diff --git a/packages/hydrooj/bin/commands.ts b/packages/hydrooj/bin/commands.ts new file mode 100644 index 00000000..8e416028 --- /dev/null +++ b/packages/hydrooj/bin/commands.ts @@ -0,0 +1,129 @@ +import child from 'child_process'; +import os from 'os'; +import path from 'path'; +import fs from 'fs-extra'; +import arg from '@hydrooj/utils/lib/arg'; + +const argv = arg(); + +const exec = (...args) => { + console.log('Executing: ', args[0], args[1].join(' ')); + const res = child.spawnSync(...args); + if (res.error) throw res.error; + if (res.status) throw new Error(`Error: Exited with code ${res.status}`); + return res; +}; + +function buildUrl(opts) { + let mongourl = `${opts.protocol || 'mongodb'}://`; + if (opts.username) mongourl += `${opts.username}:${opts.password}@`; + mongourl += `${opts.host}:${opts.port}/${opts.name}`; + if (opts.url) mongourl = opts.url; + return mongourl; +} + +const hydroPath = path.resolve(os.homedir(), '.hydro'); +fs.ensureDirSync(hydroPath); +const addonPath = path.resolve(hydroPath, 'addon.json'); +if (!fs.existsSync(addonPath)) fs.writeFileSync(addonPath, '[]'); +let addons = JSON.parse(fs.readFileSync(addonPath).toString()); + +if (argv._[0] === 'db') { + const dbConfig = fs.readFileSync(path.resolve(hydroPath, 'config.json'), 'utf-8'); + const url = buildUrl(JSON.parse(dbConfig)); + child.spawn('mongo', [url], { stdio: 'inherit' }); + process.exit(0); +} + +if (argv._[0] === 'backup') { + const dbConfig = fs.readFileSync(path.resolve(hydroPath, 'config.json'), 'utf-8'); + const url = buildUrl(JSON.parse(dbConfig)); + const dir = `${os.tmpdir()}/${Math.random().toString(36).substring(2)}`; + exec('mongodump', [url, `--out=${dir}/dump`], { stdio: 'inherit' }); + const env = `${os.homedir()}/.hydro/env`; + if (fs.existsSync(env)) fs.copySync(env, `${dir}/env`); + const target = `${process.cwd()}/backup-${new Date().toISOString().replace(':', '-').split(':')[0]}.zip`; + exec('zip', ['-r', target, 'dump'], { cwd: dir, stdio: 'inherit' }); + if (!argv.dbOnly) { + exec('zip', ['-r', target, 'file'], { cwd: '/data', stdio: 'inherit' }); + } + exec('rm', ['-rf', dir]); + console.log(`Database backup saved at ${target}`); + process.exit(0); +} + +if (argv._[0] === 'restore') { + const dbConfig = fs.readFileSync(path.resolve(hydroPath, 'config.json'), 'utf-8'); + const url = buildUrl(JSON.parse(dbConfig)); + const dir = `${os.tmpdir()}/${Math.random().toString(36).substring(2)}`; + if (!fs.existsSync(argv._[1])) { + console.error('Cannot find file'); + process.exit(1); + } + exec('unzip', [argv._[1], '-d', dir], { stdio: 'inherit' }); + exec('mongorestore', [`--uri=${url}`, `--dir=${dir}/dump/${JSON.parse(dbConfig).name}`, '--drop'], { stdio: 'inherit' }); + if (fs.existsSync(`${dir}/file`)) { + exec('rm', ['-rf', '/data/file/*'], { stdio: 'inherit' }); + exec('bash', ['-c', `mv ${dir}/file/* /data/file`], { stdio: 'inherit' }); + } + if (fs.existsSync(`${dir}/env`)) { + fs.copySync(`${dir}/env`, `${os.homedir()}/.hydro/env`, { overwrite: true }); + } + fs.removeSync(dir); + console.log('Successfully restored.'); + process.exit(0); +} + +if (!addons.includes('@hydrooj/ui-default')) { + try { + const ui = argv.ui as string || '@hydrooj/ui-default'; + require.resolve(ui); + addons.push(ui); + } catch (e) { + console.error('Please also install @hydrooj/ui-default'); + } +} + +if (argv._[0] && argv._[0] !== 'cli') { + const operation = argv._[0]; + const arg1 = argv._[1]; + const arg2 = argv._[2]; + if (operation === 'addon') { + if (arg1 === 'create') { + fs.mkdirSync('/root/addon'); + child.execSync('yarn init -y', { cwd: '/root/addon' }); + fs.mkdirSync('/root/addon/templates'); + fs.mkdirSync('/root/addon/locales'); + fs.mkdirSync('/root/addon/public'); + addons.push('/root/addon'); + } else if (arg1 === 'add') { + for (let i = 0; i < addons.length; i++) { + if (addons[i] === arg2) { + addons.splice(i, 1); + break; + } + } + addons.push(arg2); + } else if (arg1 === 'remove') { + for (let i = 0; i < addons.length; i++) { + if (addons[i] === arg2) { + addons.splice(i, 1); + break; + } + } + } + addons = Array.from(new Set(addons)); + console.log('Current Addons: ', addons); + fs.writeFileSync(addonPath, JSON.stringify(addons, null, 2)); + process.exit(0); + } + console.error('Unknown command: ', argv._[0]); +} else { + const hydro = require('../src/loader'); + addons = Array.from(new Set(addons)); + for (const addon of addons) hydro.addon(addon); + (argv._[0] === 'cli' ? hydro.loadCli : hydro.load)().catch((e) => { + console.error(e); + process.exit(1); + }); +} diff --git a/packages/hydrooj/package.json b/packages/hydrooj/package.json index 45c155d4..e1a3133d 100644 --- a/packages/hydrooj/package.json +++ b/packages/hydrooj/package.json @@ -1,6 +1,6 @@ { "name": "hydrooj", - "version": "3.9.7", + "version": "3.10.0", "bin": "bin/hydrooj.js", "main": "src/loader", "module": "src/loader", diff --git a/packages/hydrooj/src/entry/cli.ts b/packages/hydrooj/src/entry/cli.ts index 0ba092ca..e520e293 100644 --- a/packages/hydrooj/src/entry/cli.ts +++ b/packages/hydrooj/src/entry/cli.ts @@ -7,9 +7,7 @@ import options from '../options'; import * as bus from '../service/bus'; import db from '../service/db'; import { - builtinLib, builtinModel, builtinScript, - lib, model, script, - service, + lib, model, script, service, } from './common'; const argv = cac().parse(); @@ -92,14 +90,14 @@ export async function load() { await db.start(opts); const storage = require('../service/storage'); await storage.start(); - for (const i of builtinLib) require(`../lib/${i}`); + require('../lib/index'); await lib(pending, fail); const systemModel = require('../model/system'); await systemModel.runConfig(); await service(pending, fail); - for (const i of builtinModel) require(`../model/${i}`); + require('../model/index'); await model(pending, fail); - for (const i of builtinScript) require(`../script/${i}`); + require('../script/index'); await script(pending, fail, []); await bus.parallel('app/started'); await cli(); diff --git a/packages/hydrooj/src/entry/common.ts b/packages/hydrooj/src/entry/common.ts index 300943ef..5a15bb04 100644 --- a/packages/hydrooj/src/entry/common.ts +++ b/packages/hydrooj/src/entry/common.ts @@ -1,3 +1,4 @@ +/* eslint-disable import/no-dynamic-require */ /* eslint-disable no-await-in-loop */ /* eslint-disable no-eval */ import os from 'os'; @@ -10,31 +11,6 @@ import * as bus from '../service/bus'; const logger = new Logger('common'); -export const builtinLib = [ - 'jwt', 'download', 'i18n', 'mail', 'useragent', - 'crypto', 'misc', 'paginate', 'hash.hydro', 'rank', - 'validator', 'ui', 'testdataConfig', 'difficulty', 'content', - 'avatar', -]; - -export const builtinModel = [ - 'builtin', 'document', 'domain', 'blacklist', 'opcount', - 'setting', 'token', 'user', 'storage', 'problem', - 'record', 'contest', 'message', 'solution', 'training', - 'discussion', 'system', 'oplog', 'blog', -]; - -export const builtinHandler = [ - 'home', 'problem', 'record', 'judge', 'user', - 'contest', 'training', 'discussion', 'manage', 'import', - 'misc', 'homework', 'domain', 'status', 'api', 'blog', -]; - -export const builtinScript = [ - 'rating', 'problemStat', 'blacklist', 'deleteUser', 'storageUsage', - 'checkUpdate', -]; - function getFiles(folder: string, base = ''): string[] { const files = []; const f = fs.readdirSync(folder); @@ -53,7 +29,7 @@ export async function handler(pending: string[], fail: string[]) { if (fs.existsSync(p) && !fail.includes(i)) { try { logger.info('Handler init: %s', i); - eval('require')(p); + require(p); } catch (e) { fail.push(i); logger.error('Handler Load Fail: %s', i); @@ -136,7 +112,7 @@ export async function template(pending: string[], fail: string[]) { try { const files = getFiles(p); for (const file of files) { - if (file.endsWith('.tsx')) global.Hydro.ui.template[file] = eval('require')(path.resolve(p, file)); + if (file.endsWith('.tsx')) global.Hydro.ui.template[file] = require(path.resolve(p, file)); global.Hydro.ui.template[file] = await fs.readFile(path.resolve(p, file), 'utf-8'); } logger.info('Template init: %s', i); @@ -157,7 +133,7 @@ export async function model(pending: string[], fail: string[]) { if (fs.existsSync(p) && !fail.includes(i)) { try { logger.info('Model init: %s', i); - eval('require')(p); + require(p); } catch (e) { fail.push(i); logger.error('Model Load Fail: %s', i); @@ -175,7 +151,7 @@ export async function lib(pending: string[], fail: string[]) { if (fs.existsSync(p) && !fail.includes(i)) { try { logger.info('Lib init: %s', i); - eval('require')(p); + require(p); } catch (e) { fail.push(i); logger.error('Lib Load Fail: %s', i); @@ -193,7 +169,7 @@ export async function service(pending: string[], fail: string[]) { if (fs.existsSync(p) && !fail.includes(i)) { try { logger.info('Service init: %s', i); - eval('require')(p); + require(p); } catch (e) { fail.push(i); logger.error('Service Load Fail: %s', i); @@ -215,7 +191,7 @@ export async function script(pending: string[], fail: string[], active: string[] if (await fs.pathExists(p) && !fail.includes(i)) { try { logger.info('Script init: %s', i); - eval('require')(p); + require(p); } catch (e) { fail.push(i); logger.error('Script Load Fail: %s', i); diff --git a/packages/hydrooj/src/entry/worker.ts b/packages/hydrooj/src/entry/worker.ts index 03da4846..76af692b 100644 --- a/packages/hydrooj/src/entry/worker.ts +++ b/packages/hydrooj/src/entry/worker.ts @@ -9,8 +9,7 @@ import options from '../options'; import * as bus from '../service/bus'; import db from '../service/db'; import { - builtinHandler, builtinLib, builtinModel, - builtinScript, handler, lib, locale, model, script, service, setting, template, + handler, lib, locale, model, script, service, setting, template, } from './common'; const logger = new Logger('worker'); @@ -45,15 +44,7 @@ export async function load() { const storage = require('../service/storage'); await storage.start(); if (detail) logger.info('finish: storage.connect'); - for (const i of builtinLib) { - let t; - try { - t = require.resolve(`../lib/${i}`); - } catch (e) { - t = require.resolve(`@hydrooj/utils/lib/${i}`); - } - require(t); - } + require('../lib/index'); if (detail) logger.info('finish: lib.builtin'); await lib(pending, fail); if (detail) logger.info('finish: lib.extra'); @@ -64,9 +55,9 @@ export async function load() { if (detail) logger.info('finish: server'); await service(pending, fail); if (detail) logger.info('finish: service.extra'); - for (const i of builtinModel) require(`../model/${i}`); + require('../model/index'); if (detail) logger.info('finish: model.builtin'); - for (const i of builtinHandler) require(`../handler/${i}`); + require('../handler/index'); if (detail) logger.info('finish: handler.builtin'); await model(pending, fail); if (detail) logger.info('finish: model.extra'); @@ -79,7 +70,7 @@ export async function load() { if (detail) logger.info('finish: handler.apply'); const notfound = require('../handler/notfound'); await notfound.apply(); - for (const i of builtinScript) require(`../script/${i}`); + require('../script/index'); if (detail) logger.info('finish: script.builtin'); await script(pending, fail, active); if (detail) logger.info('finish: script.extra'); diff --git a/packages/hydrooj/src/handler/index.ts b/packages/hydrooj/src/handler/index.ts new file mode 100644 index 00000000..c72beb7f --- /dev/null +++ b/packages/hydrooj/src/handler/index.ts @@ -0,0 +1,16 @@ +import './home'; +import './problem'; +import './record'; +import './judge'; +import './user'; +import './contest'; +import './training'; +import './discussion'; +import './manage'; +import './import'; +import './misc'; +import './homework'; +import './domain'; +import './status'; +import './api'; +import './blog'; diff --git a/packages/hydrooj/src/handler/problem.ts b/packages/hydrooj/src/handler/problem.ts index 07ea5819..1d4d6c9c 100644 --- a/packages/hydrooj/src/handler/problem.ts +++ b/packages/hydrooj/src/handler/problem.ts @@ -114,15 +114,15 @@ export class ProblemHandler extends Handler { async cleanup() { if (this.response.template === 'problem_main.html' && this.request.json) { const { - page, pcount, ppcount, pdocs, psdict, category, + page, pcount, pcountRelation, ppcount, pdocs, psdict, category, qs, } = this.response.body; this.response.body = { title: this.renderTitle(this.translate('problem_main')), fragments: (await Promise.all([ this.renderHTML('partials/problem_list.html', { - page, ppcount, pcount, pdocs, psdict, + page, ppcount, pcount, pdocs, psdict, qs, }), - this.renderHTML('partials/problem_stat.html', { pcount }), + this.renderHTML('partials/problem_stat.html', { pcount, pcountRelation }), this.renderHTML('partials/problem_lucky.html', { category }), ])).map((i) => ({ html: i })), raw: { @@ -145,21 +145,25 @@ export class ProblemMainHandler extends ProblemHandler { const search = global.Hydro.lib.problemSearch; let sort: string[]; let fail = false; + let pcountRelation = 'eq'; if (category.length) query.$and = category.map((tag) => ({ tag })); if (q) category.push(q); if (category.length) this.extraTitleContent = category.join(','); + let total = 0; if (q) { if (search) { - const result = await search(domainId, q); - if (!result.length) fail = true; + const result = await search(domainId, q, { skip: (page - 1) * system.get('pagination.problem') }); + total = result.total; + pcountRelation = result.countRelation; + if (!result.hits.length) fail = true; if (!query.$and) query.$and = []; query.$and.push({ - $or: result.map((i) => { + $or: result.hits.map((i) => { const [did, docId] = i.split('/'); return { domainId: did, docId: +docId }; }), }); - sort = result; + sort = result.hits; } else query.$text = { $search: q }; } await bus.serial('problem/list', query, this); @@ -168,11 +172,15 @@ export class ProblemMainHandler extends ProblemHandler { ? [[], 0, 0] : await problem.list( domainId, query, - page, system.get('pagination.problem'), + sort.length ? 1 : page, system.get('pagination.problem'), undefined, this.user._id, ); + if (total) { + pcount = total; + ppcount = Math.ceil(total / system.get('pagination.problem')); + } if (sort) pdocs = pdocs.sort((a, b) => sort.indexOf(`${a.domainId}/${a.docId}`) - sort.indexOf(`${b.domainId}/${b.docId}`)); - if (q) { + if (q && page === 1) { const pdoc = await problem.get(domainId, +q || q, problem.PROJECTION_LIST); if (pdoc && problem.canViewBy(pdoc, this.user)) { const count = pdocs.length; @@ -192,7 +200,14 @@ export class ProblemMainHandler extends ProblemHandler { ); } this.response.body = { - page, pcount, ppcount, pdocs, psdict, category: category.join(','), + page, + pcount, + ppcount, + pcountRelation, + pdocs, + psdict, + category: category.join(','), + qs: q ? `q=${encodeURIComponent(q)}` : '', }; } @@ -829,8 +844,8 @@ export class ProblemPrefixListHandler extends Handler { const search = global.Hydro.lib.problemSearch; if (pdocs.length < 20) { if (search) { - const result = await search(domainId, prefix, 20 - pdocs.length); - const docs = await problem.getMulti(domainId, { docId: { $in: result } }).toArray(); + const result = await search(domainId, prefix, { limit: 20 - pdocs.length }); + const docs = await problem.getMulti(domainId, { docId: { $in: result.hits.map((i) => +i.split('/')[1]) } }).toArray(); pdocs.push(...docs); } } diff --git a/packages/hydrooj/src/interface.ts b/packages/hydrooj/src/interface.ts index 63d602a2..0cb2c1b7 100644 --- a/packages/hydrooj/src/interface.ts +++ b/packages/hydrooj/src/interface.ts @@ -103,6 +103,13 @@ export interface GDoc { uids: number[]; } +export interface UserPreferenceDoc { + _id: ObjectID; + filename: string; + uid: number; + content: string; +} + export type ownerInfo = { owner: number, maintainer?: number[] }; export type User = import('./model/user').User; @@ -579,6 +586,7 @@ export interface Collections { 'document.status': any; 'problem': ProblemDoc; 'user': Udoc; + 'user.preference': UserPreferenceDoc; 'vuser': VUdoc; 'user.group': GDoc; 'check': any; @@ -631,11 +639,23 @@ export interface Service { storage: typeof import('./service/storage'), } -interface GeoIP { +export interface GeoIP { provider: string, lookup: (ip: string, locale?: string) => any, } +export interface ProblemSearchResponse { + hits: string[]; + total: number; + countRelation: 'eq' | 'gte'; +} +export interface ProblemSearchOptions { + limit?: number; + skip?: number; +} + +export type ProblemSearch = (domainId: string, q: string, options?: ProblemSearchOptions) => Promise; + export interface Lib extends Record { download: typeof import('./lib/download'), difficulty: typeof import('./lib/difficulty'), @@ -655,6 +675,7 @@ export interface Lib extends Record { validator: typeof import('./lib/validator'), template?: any, geoip?: GeoIP, + problemSearch: ProblemSearch; } export interface UI { diff --git a/packages/hydrooj/src/lib/index.ts b/packages/hydrooj/src/lib/index.ts new file mode 100644 index 00000000..84d51587 --- /dev/null +++ b/packages/hydrooj/src/lib/index.ts @@ -0,0 +1,16 @@ +import './jwt'; +import './download'; +import './i18n'; +import './mail'; +import './useragent'; +import './crypto'; +import './misc'; +import './paginate'; +import './hash.hydro'; +import './rank'; +import './validator'; +import './ui'; +import './testdataConfig'; +import './difficulty'; +import './content'; +import './avatar'; diff --git a/packages/hydrooj/src/logger.ts b/packages/hydrooj/src/logger.ts index e86894ce..891aa60b 100644 --- a/packages/hydrooj/src/logger.ts +++ b/packages/hydrooj/src/logger.ts @@ -39,6 +39,7 @@ export class Logger { } constructor(public name: string) { + // eslint-disable-next-line no-constructor-return if (name in instances) return instances[name]; let hash = 0; for (let i = 0; i < name.length; i++) { diff --git a/packages/hydrooj/src/model/contest.ts b/packages/hydrooj/src/model/contest.ts index a2aa1a6c..78ecf112 100644 --- a/packages/hydrooj/src/model/contest.ts +++ b/packages/hydrooj/src/model/contest.ts @@ -125,7 +125,9 @@ const acm = buildContestRule({ const data = await document.collStatus.aggregate([ { $match: { - domainId: tdoc.domainId, docType: document.TYPE_CONTEST, docId: tdoc.docId, + domainId: tdoc.domainId, + docType: document.TYPE_CONTEST, + docId: tdoc.docId, accept: { $gte: 1 }, detail: { $elemMatch: { status: STATUS.STATUS_ACCEPTED } }, }, @@ -230,9 +232,10 @@ const oi = buildContestRule({ const psdict = {}; const first = {}; for (const pid of tdoc.pids) { - // eslint-disable-next-line @typescript-eslint/no-use-before-define + // eslint-disable-next-line @typescript-eslint/no-use-before-define, no-await-in-loop const [data] = await getMultiStatus(tdoc.domainId, { - docType: document.TYPE_CONTEST, docId: tdoc.docId, + docType: document.TYPE_CONTEST, + docId: tdoc.docId, [`detail.${pid}.status`]: STATUS.STATUS_ACCEPTED, }).sort({ [`detail.${pid}.rid`]: 1 }).limit(1).toArray(); first[pid] = data ? data.detail[pid].rid.generationTime : new ObjectID().generationTime; diff --git a/packages/hydrooj/src/model/index.ts b/packages/hydrooj/src/model/index.ts new file mode 100644 index 00000000..48485fe3 --- /dev/null +++ b/packages/hydrooj/src/model/index.ts @@ -0,0 +1,19 @@ +import './builtin'; +import './document'; +import './domain'; +import './blacklist'; +import './opcount'; +import './setting'; +import './token'; +import './user'; +import './storage'; +import './problem'; +import './record'; +import './contest'; +import './message'; +import './solution'; +import './training'; +import './discussion'; +import './system'; +import './oplog'; +import './blog'; diff --git a/packages/hydrooj/src/model/task.ts b/packages/hydrooj/src/model/task.ts index 6208b0e6..f90fb183 100644 --- a/packages/hydrooj/src/model/task.ts +++ b/packages/hydrooj/src/model/task.ts @@ -72,7 +72,7 @@ class WorkerService implements BaseService { const start = Date.now(); await Promise.race([ this.handlers[doc.subType](doc), - new Promise((resolve) => setTimeout(resolve, 300000)), + sleep(300000), ]); const spent = Date.now() - start; if (spent > 500) logger.warn('Slow worker task (%d ms): %s', spent, doc); diff --git a/packages/hydrooj/src/script/index.ts b/packages/hydrooj/src/script/index.ts new file mode 100644 index 00000000..863b2dd5 --- /dev/null +++ b/packages/hydrooj/src/script/index.ts @@ -0,0 +1,6 @@ +import './rating'; +import './problemStat'; +import './blacklist'; +import './deleteUser'; +import './storageUsage'; +import './checkUpdate'; diff --git a/packages/hydrooj/src/service/__mocks__/db.ts b/packages/hydrooj/src/service/__mocks__/db.ts deleted file mode 100644 index 7959da11..00000000 --- a/packages/hydrooj/src/service/__mocks__/db.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { Collection, Db, MongoClient } from 'mongodb'; -import { BaseService, Collections } from '../../interface'; -import * as bus from '../bus'; - -interface MongoConfig { - protocol?: string, - username?: string, - password?: string, - host?: string, - port?: string, - name?: string, - url?: string, - prefix?: string, -} - -class MongoService implements BaseService { - public client: MongoClient; - public db: Db; - public started = false; - private opts: MongoConfig; - - async start(opts: MongoConfig) { - this.opts = opts; - this.client = await MongoClient.connect(global.__MONGO_URI__, { useNewUrlParser: true, useUnifiedTopology: true }); - this.db = this.client.db(global.__MONGO_DB_NAME_); - await bus.parallel('database/connect', this.db); - this.started = true; - } - - public collection(c: K): Collection { - if (this.opts.prefix) return this.db.collection(`${this.opts.prefix}.${c}`); - return this.db.collection(c); - } - - public async stop() { - await this.client.close(); - await this.client2.close(); - } -} - -const service = new MongoService(); -global.Hydro.service.db = service as any; -export = service; diff --git a/packages/login-with-osu/README.md b/packages/login-with-osu/README.md deleted file mode 100644 index 795f2ffa..00000000 --- a/packages/login-with-osu/README.md +++ /dev/null @@ -1 +0,0 @@ -# Login-With-Osu diff --git a/packages/login-with-osu/lib.ts b/packages/login-with-osu/lib.ts deleted file mode 100644 index 0fb0a5dc..00000000 --- a/packages/login-with-osu/lib.ts +++ /dev/null @@ -1,73 +0,0 @@ -import 'hydrooj'; - -import * as superagent from 'superagent'; - -declare module 'hydrooj' { - interface SystemKeys { - 'login-with-osu.id': string, - 'login-with-osu.secret': string, - } - interface Lib { - oauth_osu: typeof import('./lib'), - } -} - -const BASE_URL = 'https://osu.ppy.sh/'; - -async function get() { - const { system, token } = global.Hydro.model; - const [[appid, url], [state]] = await Promise.all([ - system.getMany([ - 'login-with-osu.id', - 'server.url', - ]), - token.add(token.TYPE_OAUTH, 600, { redirect: this.request.referer }), - ]); - this.response.redirect = `${BASE_URL}oauth/authorize?client_id=${appid}&state=${state}&redirect_uri=${url}oauth/osu/callback&response_type=code`; -} - -async function callback({ state, code }) { - const { system, token } = global.Hydro.model; - const { UserFacingError } = global.Hydro.error; - const [[appid, secret, url], s] = await Promise.all([ - system.getMany([ - 'login-with-osu.id', - 'login-with-osu.secret', - 'server.url', - ]), - token.get(state, token.TYPE_OAUTH), - ]); - const res = await superagent.post(`${BASE_URL}oauth/token`) - .send({ - client_id: appid, - client_secret: secret, - code, - grant_type: 'authorization_code', - redirect_uri: `${url}oauth/github/callback`, - }) - .set('accept', 'application/json'); - if (res.body.error) { - throw new UserFacingError( - res.body.error, res.body.error_description, res.body.error_uri, - ); - } - const t = res.body.access_token; - const userInfo = await superagent.get(`${BASE_URL}api/v2/me`) - .set('User-Agent', 'Hydro-OAuth') - .set('Authorization', `Bearer ${t}`); - const ret = { - _id: `${userInfo.body.id}@osu.local`, - email: `${userInfo.body.id}@osu.local`, - bio: '', - uname: [userInfo.body.username], - }; - this.response.redirect = s.redirect; - await token.del(s._id, token.TYPE_OAUTH); - return ret; -} - -global.Hydro.lib.oauth_osu = { - text: 'Login with Osu', - callback, - get, -}; diff --git a/packages/login-with-osu/locales/zh.yaml b/packages/login-with-osu/locales/zh.yaml deleted file mode 100644 index b75e84d9..00000000 --- a/packages/login-with-osu/locales/zh.yaml +++ /dev/null @@ -1 +0,0 @@ -Login with Osu: 使用 Osu 登录 \ No newline at end of file diff --git a/packages/login-with-osu/package.json b/packages/login-with-osu/package.json deleted file mode 100644 index 28e9ec70..00000000 --- a/packages/login-with-osu/package.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "name": "@hydrooj/login-with-osu", - "version": "0.1.0", - "main": "package.json", - "repository": "git@github.com:hydro-dev/Hydro.git", - "author": "undefined ", - "license": "AGPL-3.0-or-later", - "preferUnplugged": true, - "devDependencies": { - "@types/superagent": "^4.1.15" - }, - "peerDependencies": { - "hydrooj": "*" - }, - "dependencies": { - "superagent": "^7.1.1" - } -} diff --git a/packages/login-with-osu/setting.yaml b/packages/login-with-osu/setting.yaml deleted file mode 100644 index 025648ee..00000000 --- a/packages/login-with-osu/setting.yaml +++ /dev/null @@ -1,15 +0,0 @@ -id: - type: text - category: system - desc: Osu OAuth AppID - name: id -secret: - type: text - category: system - desc: Osu OAuth Secret - name: secret -proxy: - type: text - category: system - desc: Osu OAuth Proxy - name: proxy \ No newline at end of file diff --git a/packages/migrate-hustoj/script.ts b/packages/migrate-hustoj/script.ts index 4c5bd698..b1bd1cdc 100644 --- a/packages/migrate-hustoj/script.ts +++ b/packages/migrate-hustoj/script.ts @@ -56,7 +56,7 @@ const nameMap = { export async function run({ host = 'localhost', port = 3306, name = 'jol', username, password, domainId, contestType = 'oi', - dataDir, + dataDir, rerun = false, }, report: Function) { const src = mysql.createConnection({ host, @@ -65,7 +65,9 @@ export async function run({ password, database: name, }); - await new Promise((resolve, reject) => src.connect((err) => (err ? reject(err) : resolve(null)))); + await new Promise((resolve, reject) => { + src.connect((err) => (err ? reject(err) : resolve(null))); + }); const query = (q: string | mysql.Query) => new Promise<[values: any[], fields: mysql.FieldInfo[]]>((res, rej) => { src.query(q, (err, val, fields) => { if (err) rej(err); @@ -151,23 +153,29 @@ export async function run({ for (let pageId = 0; pageId < pageCount; pageId++) { const [pdocs] = await query(`SELECT * FROM \`problem\` LIMIT ${pageId * step}, ${step}`); for (const pdoc of pdocs) { - const pid = await problem.add( - domainId, `P${pdoc.problem_id}`, - pdoc.title, buildContent({ - description: pdoc.description, - input: pdoc.input, - output: pdoc.output, - samples: [[pdoc.sample_input.trim(), pdoc.sample_output.trim()]], - hint: pdoc.hint, - source: pdoc.source, - }, 'html'), - 1, pdoc.source.split(' ').map((i) => i.trim()).filter((i) => i), pdoc.defunct === 'Y', - ); - pidMap[pdoc.problem_id] = pid; + if (rerun) { + const opdoc = await problem.get(domainId, `P${pdoc.problem_id}`); + if (opdoc) pidMap[pdoc.problem_id] = opdoc.docId; + } + if (!pidMap[pdoc.problem_id]) { + const pid = await problem.add( + domainId, `P${pdoc.problem_id}`, + pdoc.title, buildContent({ + description: pdoc.description, + input: pdoc.input, + output: pdoc.output, + samples: [[pdoc.sample_input.trim(), pdoc.sample_output.trim()]], + hint: pdoc.hint, + source: pdoc.source, + }, 'html'), + 1, pdoc.source.split(' ').map((i) => i.trim()).filter((i) => i), pdoc.defunct === 'Y', + ); + pidMap[pdoc.problem_id] = pid; + } const [cdoc] = await query(`SELECT * FROM \`privilege\` WHERE \`rightstr\` = 'p${pdoc.problem_id}'`); const maintainer = []; for (let i = 1; i < cdoc.length; i++) maintainer.push(uidMap[cdoc[i].user_id]); - await problem.edit(domainId, pid, { + await problem.edit(domainId, pidMap[pdoc.problem_id], { nAccept: 0, nSubmit: pdoc.submit, config: `time: ${pdoc.time_limit}s\nmemory: ${pdoc.memory_limit}m`, @@ -264,6 +272,7 @@ export async function run({ src.end(); + if (!dataDir) return true; if (dataDir.endsWith('/')) dataDir = dataDir.slice(0, -1); const files = await fs.readdir(dataDir, { withFileTypes: true }); for (const file of files) { diff --git a/packages/sonic/model.ts b/packages/sonic/model.ts index b9a85438..81a0fd6f 100644 --- a/packages/sonic/model.ts +++ b/packages/sonic/model.ts @@ -46,8 +46,13 @@ bus.on('problem/del', async (domainId, docId) => { await Promise.all(tasks); }); -global.Hydro.lib.problemSearch = async (domainId: string, query: string, limit = system.get('pagination.problem')) => { +global.Hydro.lib.problemSearch = async (domainId, query, opts) => { + const limit = opts?.limit || system.get('pagination.problem'); const ids = await sonic.query('problem', `${domainId}@title`, query, { limit }); if (limit - ids.length > 0) ids.push(...await sonic.query('problem', `${domainId}@content`, query, { limit: limit - ids.length })); - return ids; + return { + countRelation: ids.length >= limit ? 'gte' : 'eq', + total: ids.length, + hits: ids, + }; }; diff --git a/packages/sonic/package.json b/packages/sonic/package.json index d65ff033..a6fb1c2a 100644 --- a/packages/sonic/package.json +++ b/packages/sonic/package.json @@ -1,6 +1,6 @@ { "name": "@hydrooj/sonic", - "version": "1.2.4", + "version": "1.2.5", "description": "Sonic search service", "main": "service.js", "typings": "service.d.ts", diff --git a/packages/ui-default/locales/zh.yaml b/packages/ui-default/locales/zh.yaml index b3059c06..5d333d8c 100644 --- a/packages/ui-default/locales/zh.yaml +++ b/packages/ui-default/locales/zh.yaml @@ -527,6 +527,7 @@ Or, with automatically filled invitation code: 或者，这是可以自动填写 Ordered List: 有序列表 Original Score: 原始分数 OS Info: 系统信息 +Over {0} problems: 超过 {0} 道题目 Owner: 所有者 page.problem_detail.sidebar.show_category: 点击显示 page.training_detail.invalid_when_not_enrolled: 未参加训练计划时您不能查看题目详情。 diff --git a/packages/ui-default/package.json b/packages/ui-default/package.json index 4f9712ae..ba6c7079 100644 --- a/packages/ui-default/package.json +++ b/packages/ui-default/package.json @@ -1,6 +1,6 @@ { "name": "@hydrooj/ui-default", - "version": "4.34.12", + "version": "4.34.13", "author": "undefined ", "license": "AGPL-3.0", "main": "hydro.js", diff --git a/packages/ui-default/templates/partials/problem_list.html b/packages/ui-default/templates/partials/problem_list.html index 9618369b..c6877ca2 100644 --- a/packages/ui-default/templates/partials/problem_list.html +++ b/packages/ui-default/templates/partials/problem_list.html @@ -6,7 +6,7 @@ {% if not pdocs.length %} {{ nothing.render('Sorry, there is no problem in the problem set') }} {% else %} - {{ paginator.render(page, ppcount, position='top') }} + {{ paginator.render(page, ppcount, position='top', add_qs=qs) }} @@ -73,6 +73,6 @@ {%- endfor -%}
- {{ paginator.render(page, ppcount) }} + {{ paginator.render(page, ppcount, add_qs=qs) }} {% endif %} diff --git a/packages/ui-default/templates/partials/problem_stat.html b/packages/ui-default/templates/partials/problem_stat.html index 6c541585..1b9fb623 100644 --- a/packages/ui-default/templates/partials/problem_stat.html +++ b/packages/ui-default/templates/partials/problem_stat.html @@ -1,3 +1,3 @@
-

{{ _('{0} problems').format(pcount) }}

+

{{ _('{0} problems' if pcountRelation == 'eq' else 'Over {0} problems').format(pcount) }}

\ No newline at end of file diff --git a/packages/vjudge/src/providers/uoj.ts b/packages/vjudge/src/providers/uoj.ts index 59ca61c5..686ac86a 100644 --- a/packages/vjudge/src/providers/uoj.ts +++ b/packages/vjudge/src/providers/uoj.ts @@ -252,7 +252,8 @@ export default class UOJProvider implements IBasicProvider { } } if (document.querySelector('tbody').innerHTML.includes('Judging')) continue; - const score = +summary.children[3]?.children[0].innerHTML || 0; + // eslint-disable-next-line no-unsafe-optional-chaining + const score = +summary.children[3]?.children[0]?.innerHTML || 0; const status = score === 100 ? STATUS.STATUS_ACCEPTED : STATUS.STATUS_WRONG_ANSWER; return await end({ status,