core&ui&elastic: support elastic search and allow search result pagination
undefined
3 years ago
parent
ea6529b523
commit
18fc66ecb7
@ -0,0 +1,11 @@
|
||||
{
|
||||
"name": "@hydrooj/elastic-search",
|
||||
"version": "1.0.0",
|
||||
"main": "package.json",
|
||||
"repository": "https://github.com/hydro-dev/Hydro",
|
||||
"author": "undefined <i@undefined.moe>",
|
||||
"license": "AGPL-3.0-or-later",
|
||||
"dependencies": {
|
||||
"@elastic/elasticsearch": "^8.1.0"
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,95 @@
|
||||
import { Client } from '@elastic/elasticsearch';
|
||||
import { omit } from 'lodash';
|
||||
import DomainModel from 'hydrooj/src/model/domain';
|
||||
import ProblemModel from 'hydrooj/src/model/problem';
|
||||
import * as system from 'hydrooj/src/model/system';
|
||||
import { iterateAllProblem, iterateAllProblemInDomain } from 'hydrooj/src/pipelineUtils';
|
||||
import * as bus from 'hydrooj/src/service/bus';
|
||||
|
||||
const client = new Client({ node: system.get('elasticsearch.url') || 'http://192.168.1.82:9200' });
|
||||
|
||||
const indexOmit = ['_id', 'docType', 'data', 'additional_file', 'config', 'stats', 'assign'];
|
||||
|
||||
bus.on('problem/add', async (doc, docId) => {
|
||||
await client.index({
|
||||
index: 'problem',
|
||||
id: `${doc.domainId}/${docId}`,
|
||||
document: omit(doc, indexOmit),
|
||||
});
|
||||
});
|
||||
bus.on('problem/edit', async (pdoc) => {
|
||||
await client.index({
|
||||
index: 'problem',
|
||||
id: `${pdoc.domainId}/${pdoc.docId}`,
|
||||
document: omit(pdoc, indexOmit),
|
||||
});
|
||||
});
|
||||
bus.on('problem/del', async (domainId, docId) => {
|
||||
await client.delete({
|
||||
index: 'problem',
|
||||
id: `${domainId}/${docId}`,
|
||||
});
|
||||
});
|
||||
|
||||
global.Hydro.lib.problemSearch = async (domainId, q, opts) => {
|
||||
const size = opts?.limit || system.get('pagination.problem');
|
||||
const from = opts?.skip || 0;
|
||||
const union = await DomainModel.getUnion(domainId);
|
||||
const domainIds = [domainId, ...(union?.union || [])];
|
||||
const res = await client.search({
|
||||
index: 'problem',
|
||||
size,
|
||||
from,
|
||||
query: {
|
||||
simple_query_string: {
|
||||
query: q,
|
||||
fields: ['tag^5', 'title^3', 'content'],
|
||||
},
|
||||
},
|
||||
post_filter: {
|
||||
bool: {
|
||||
minimum_should_match: 1,
|
||||
should: domainIds.map((i) => ({ match: { domainId: i } })),
|
||||
},
|
||||
},
|
||||
});
|
||||
return {
|
||||
countRelation: typeof res.hits.total === 'number' ? 'eq' : res.hits.total.relation,
|
||||
total: typeof res.hits.total === 'number' ? res.hits.total : res.hits.total.value,
|
||||
hits: res.hits.hits.map((i) => i._id),
|
||||
};
|
||||
};
|
||||
|
||||
export const description = 'Elastic problem search re-index';
|
||||
|
||||
export async function run({ domainId }, report) {
|
||||
try {
|
||||
if (domainId) await client.deleteByQuery({ index: 'problem', query: { match: { domainId } } });
|
||||
else await client.deleteByQuery({ index: 'problem', query: { match_all: {} } });
|
||||
} catch (e) {
|
||||
if (!e.message.includes('index_not_found_exception')) throw e;
|
||||
}
|
||||
let i = 0;
|
||||
const cb = async (pdoc) => {
|
||||
i++;
|
||||
if (!(i % 1000)) report({ message: `${i} problems indexed` });
|
||||
await client.index({
|
||||
index: 'problem',
|
||||
id: `${pdoc.domainId}/${pdoc.docId}`,
|
||||
document: omit(pdoc, indexOmit),
|
||||
});
|
||||
};
|
||||
if (domainId) await iterateAllProblemInDomain(domainId, ProblemModel.PROJECTION_PUBLIC, cb);
|
||||
else await iterateAllProblem(ProblemModel.PROJECTION_PUBLIC, cb);
|
||||
await client.indices.refresh({ index: 'problem' });
|
||||
return true;
|
||||
}
|
||||
|
||||
export const validate = {
|
||||
$or: [
|
||||
{ domainId: 'string' },
|
||||
{ domainId: 'undefined' },
|
||||
],
|
||||
};
|
||||
|
||||
global.Hydro.script.ensureElasticSearch = { run, description, validate };
|
||||
@ -1,2 +1,3 @@
|
||||
export { default as vj4 } from './vj4';
|
||||
// eslint-disable-next-line no-restricted-exports
|
||||
export { default as hydro, default } from './hydro';
|
||||
|
||||
@ -0,0 +1,129 @@
|
||||
import child from 'child_process';
|
||||
import os from 'os';
|
||||
import path from 'path';
|
||||
import fs from 'fs-extra';
|
||||
import arg from '@hydrooj/utils/lib/arg';
|
||||
|
||||
const argv = arg();
|
||||
|
||||
const exec = (...args) => {
|
||||
console.log('Executing: ', args[0], args[1].join(' '));
|
||||
const res = child.spawnSync(...args);
|
||||
if (res.error) throw res.error;
|
||||
if (res.status) throw new Error(`Error: Exited with code ${res.status}`);
|
||||
return res;
|
||||
};
|
||||
|
||||
function buildUrl(opts) {
|
||||
let mongourl = `${opts.protocol || 'mongodb'}://`;
|
||||
if (opts.username) mongourl += `${opts.username}:${opts.password}@`;
|
||||
mongourl += `${opts.host}:${opts.port}/${opts.name}`;
|
||||
if (opts.url) mongourl = opts.url;
|
||||
return mongourl;
|
||||
}
|
||||
|
||||
const hydroPath = path.resolve(os.homedir(), '.hydro');
|
||||
fs.ensureDirSync(hydroPath);
|
||||
const addonPath = path.resolve(hydroPath, 'addon.json');
|
||||
if (!fs.existsSync(addonPath)) fs.writeFileSync(addonPath, '[]');
|
||||
let addons = JSON.parse(fs.readFileSync(addonPath).toString());
|
||||
|
||||
if (argv._[0] === 'db') {
|
||||
const dbConfig = fs.readFileSync(path.resolve(hydroPath, 'config.json'), 'utf-8');
|
||||
const url = buildUrl(JSON.parse(dbConfig));
|
||||
child.spawn('mongo', [url], { stdio: 'inherit' });
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
if (argv._[0] === 'backup') {
|
||||
const dbConfig = fs.readFileSync(path.resolve(hydroPath, 'config.json'), 'utf-8');
|
||||
const url = buildUrl(JSON.parse(dbConfig));
|
||||
const dir = `${os.tmpdir()}/${Math.random().toString(36).substring(2)}`;
|
||||
exec('mongodump', [url, `--out=${dir}/dump`], { stdio: 'inherit' });
|
||||
const env = `${os.homedir()}/.hydro/env`;
|
||||
if (fs.existsSync(env)) fs.copySync(env, `${dir}/env`);
|
||||
const target = `${process.cwd()}/backup-${new Date().toISOString().replace(':', '-').split(':')[0]}.zip`;
|
||||
exec('zip', ['-r', target, 'dump'], { cwd: dir, stdio: 'inherit' });
|
||||
if (!argv.dbOnly) {
|
||||
exec('zip', ['-r', target, 'file'], { cwd: '/data', stdio: 'inherit' });
|
||||
}
|
||||
exec('rm', ['-rf', dir]);
|
||||
console.log(`Database backup saved at ${target}`);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
if (argv._[0] === 'restore') {
|
||||
const dbConfig = fs.readFileSync(path.resolve(hydroPath, 'config.json'), 'utf-8');
|
||||
const url = buildUrl(JSON.parse(dbConfig));
|
||||
const dir = `${os.tmpdir()}/${Math.random().toString(36).substring(2)}`;
|
||||
if (!fs.existsSync(argv._[1])) {
|
||||
console.error('Cannot find file');
|
||||
process.exit(1);
|
||||
}
|
||||
exec('unzip', [argv._[1], '-d', dir], { stdio: 'inherit' });
|
||||
exec('mongorestore', [`--uri=${url}`, `--dir=${dir}/dump/${JSON.parse(dbConfig).name}`, '--drop'], { stdio: 'inherit' });
|
||||
if (fs.existsSync(`${dir}/file`)) {
|
||||
exec('rm', ['-rf', '/data/file/*'], { stdio: 'inherit' });
|
||||
exec('bash', ['-c', `mv ${dir}/file/* /data/file`], { stdio: 'inherit' });
|
||||
}
|
||||
if (fs.existsSync(`${dir}/env`)) {
|
||||
fs.copySync(`${dir}/env`, `${os.homedir()}/.hydro/env`, { overwrite: true });
|
||||
}
|
||||
fs.removeSync(dir);
|
||||
console.log('Successfully restored.');
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
if (!addons.includes('@hydrooj/ui-default')) {
|
||||
try {
|
||||
const ui = argv.ui as string || '@hydrooj/ui-default';
|
||||
require.resolve(ui);
|
||||
addons.push(ui);
|
||||
} catch (e) {
|
||||
console.error('Please also install @hydrooj/ui-default');
|
||||
}
|
||||
}
|
||||
|
||||
if (argv._[0] && argv._[0] !== 'cli') {
|
||||
const operation = argv._[0];
|
||||
const arg1 = argv._[1];
|
||||
const arg2 = argv._[2];
|
||||
if (operation === 'addon') {
|
||||
if (arg1 === 'create') {
|
||||
fs.mkdirSync('/root/addon');
|
||||
child.execSync('yarn init -y', { cwd: '/root/addon' });
|
||||
fs.mkdirSync('/root/addon/templates');
|
||||
fs.mkdirSync('/root/addon/locales');
|
||||
fs.mkdirSync('/root/addon/public');
|
||||
addons.push('/root/addon');
|
||||
} else if (arg1 === 'add') {
|
||||
for (let i = 0; i < addons.length; i++) {
|
||||
if (addons[i] === arg2) {
|
||||
addons.splice(i, 1);
|
||||
break;
|
||||
}
|
||||
}
|
||||
addons.push(arg2);
|
||||
} else if (arg1 === 'remove') {
|
||||
for (let i = 0; i < addons.length; i++) {
|
||||
if (addons[i] === arg2) {
|
||||
addons.splice(i, 1);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
addons = Array.from(new Set(addons));
|
||||
console.log('Current Addons: ', addons);
|
||||
fs.writeFileSync(addonPath, JSON.stringify(addons, null, 2));
|
||||
process.exit(0);
|
||||
}
|
||||
console.error('Unknown command: ', argv._[0]);
|
||||
} else {
|
||||
const hydro = require('../src/loader');
|
||||
addons = Array.from(new Set(addons));
|
||||
for (const addon of addons) hydro.addon(addon);
|
||||
(argv._[0] === 'cli' ? hydro.loadCli : hydro.load)().catch((e) => {
|
||||
console.error(e);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
@ -0,0 +1,16 @@
|
||||
import './home';
|
||||
import './problem';
|
||||
import './record';
|
||||
import './judge';
|
||||
import './user';
|
||||
import './contest';
|
||||
import './training';
|
||||
import './discussion';
|
||||
import './manage';
|
||||
import './import';
|
||||
import './misc';
|
||||
import './homework';
|
||||
import './domain';
|
||||
import './status';
|
||||
import './api';
|
||||
import './blog';
|
||||
@ -0,0 +1,16 @@
|
||||
import './jwt';
|
||||
import './download';
|
||||
import './i18n';
|
||||
import './mail';
|
||||
import './useragent';
|
||||
import './crypto';
|
||||
import './misc';
|
||||
import './paginate';
|
||||
import './hash.hydro';
|
||||
import './rank';
|
||||
import './validator';
|
||||
import './ui';
|
||||
import './testdataConfig';
|
||||
import './difficulty';
|
||||
import './content';
|
||||
import './avatar';
|
||||
@ -0,0 +1,19 @@
|
||||
import './builtin';
|
||||
import './document';
|
||||
import './domain';
|
||||
import './blacklist';
|
||||
import './opcount';
|
||||
import './setting';
|
||||
import './token';
|
||||
import './user';
|
||||
import './storage';
|
||||
import './problem';
|
||||
import './record';
|
||||
import './contest';
|
||||
import './message';
|
||||
import './solution';
|
||||
import './training';
|
||||
import './discussion';
|
||||
import './system';
|
||||
import './oplog';
|
||||
import './blog';
|
||||
@ -0,0 +1,6 @@
|
||||
import './rating';
|
||||
import './problemStat';
|
||||
import './blacklist';
|
||||
import './deleteUser';
|
||||
import './storageUsage';
|
||||
import './checkUpdate';
|
||||
@ -1,43 +0,0 @@
|
||||
import { Collection, Db, MongoClient } from 'mongodb';
|
||||
import { BaseService, Collections } from '../../interface';
|
||||
import * as bus from '../bus';
|
||||
|
||||
interface MongoConfig {
|
||||
protocol?: string,
|
||||
username?: string,
|
||||
password?: string,
|
||||
host?: string,
|
||||
port?: string,
|
||||
name?: string,
|
||||
url?: string,
|
||||
prefix?: string,
|
||||
}
|
||||
|
||||
class MongoService implements BaseService {
|
||||
public client: MongoClient;
|
||||
public db: Db;
|
||||
public started = false;
|
||||
private opts: MongoConfig;
|
||||
|
||||
async start(opts: MongoConfig) {
|
||||
this.opts = opts;
|
||||
this.client = await MongoClient.connect(global.__MONGO_URI__, { useNewUrlParser: true, useUnifiedTopology: true });
|
||||
this.db = this.client.db(global.__MONGO_DB_NAME_);
|
||||
await bus.parallel('database/connect', this.db);
|
||||
this.started = true;
|
||||
}
|
||||
|
||||
public collection<K extends keyof Collections>(c: K): Collection<Collections[K]> {
|
||||
if (this.opts.prefix) return this.db.collection(`${this.opts.prefix}.${c}`);
|
||||
return this.db.collection(c);
|
||||
}
|
||||
|
||||
public async stop() {
|
||||
await this.client.close();
|
||||
await this.client2.close();
|
||||
}
|
||||
}
|
||||
|
||||
const service = new MongoService();
|
||||
global.Hydro.service.db = service as any;
|
||||
export = service;
|
||||
@ -1 +0,0 @@
|
||||
# Login-With-Osu
|
||||
@ -1,73 +0,0 @@
|
||||
import 'hydrooj';
|
||||
|
||||
import * as superagent from 'superagent';
|
||||
|
||||
declare module 'hydrooj' {
|
||||
interface SystemKeys {
|
||||
'login-with-osu.id': string,
|
||||
'login-with-osu.secret': string,
|
||||
}
|
||||
interface Lib {
|
||||
oauth_osu: typeof import('./lib'),
|
||||
}
|
||||
}
|
||||
|
||||
const BASE_URL = 'https://osu.ppy.sh/';
|
||||
|
||||
async function get() {
|
||||
const { system, token } = global.Hydro.model;
|
||||
const [[appid, url], [state]] = await Promise.all([
|
||||
system.getMany([
|
||||
'login-with-osu.id',
|
||||
'server.url',
|
||||
]),
|
||||
token.add(token.TYPE_OAUTH, 600, { redirect: this.request.referer }),
|
||||
]);
|
||||
this.response.redirect = `${BASE_URL}oauth/authorize?client_id=${appid}&state=${state}&redirect_uri=${url}oauth/osu/callback&response_type=code`;
|
||||
}
|
||||
|
||||
async function callback({ state, code }) {
|
||||
const { system, token } = global.Hydro.model;
|
||||
const { UserFacingError } = global.Hydro.error;
|
||||
const [[appid, secret, url], s] = await Promise.all([
|
||||
system.getMany([
|
||||
'login-with-osu.id',
|
||||
'login-with-osu.secret',
|
||||
'server.url',
|
||||
]),
|
||||
token.get(state, token.TYPE_OAUTH),
|
||||
]);
|
||||
const res = await superagent.post(`${BASE_URL}oauth/token`)
|
||||
.send({
|
||||
client_id: appid,
|
||||
client_secret: secret,
|
||||
code,
|
||||
grant_type: 'authorization_code',
|
||||
redirect_uri: `${url}oauth/github/callback`,
|
||||
})
|
||||
.set('accept', 'application/json');
|
||||
if (res.body.error) {
|
||||
throw new UserFacingError(
|
||||
res.body.error, res.body.error_description, res.body.error_uri,
|
||||
);
|
||||
}
|
||||
const t = res.body.access_token;
|
||||
const userInfo = await superagent.get(`${BASE_URL}api/v2/me`)
|
||||
.set('User-Agent', 'Hydro-OAuth')
|
||||
.set('Authorization', `Bearer ${t}`);
|
||||
const ret = {
|
||||
_id: `${userInfo.body.id}@osu.local`,
|
||||
email: `${userInfo.body.id}@osu.local`,
|
||||
bio: '',
|
||||
uname: [userInfo.body.username],
|
||||
};
|
||||
this.response.redirect = s.redirect;
|
||||
await token.del(s._id, token.TYPE_OAUTH);
|
||||
return ret;
|
||||
}
|
||||
|
||||
global.Hydro.lib.oauth_osu = {
|
||||
text: 'Login with Osu',
|
||||
callback,
|
||||
get,
|
||||
};
|
||||
@ -1 +0,0 @@
|
||||
Login with Osu: 使用 Osu 登录
|
||||
@ -1,18 +0,0 @@
|
||||
{
|
||||
"name": "@hydrooj/login-with-osu",
|
||||
"version": "0.1.0",
|
||||
"main": "package.json",
|
||||
"repository": "git@github.com:hydro-dev/Hydro.git",
|
||||
"author": "undefined <i@undefined.moe>",
|
||||
"license": "AGPL-3.0-or-later",
|
||||
"preferUnplugged": true,
|
||||
"devDependencies": {
|
||||
"@types/superagent": "^4.1.15"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"hydrooj": "*"
|
||||
},
|
||||
"dependencies": {
|
||||
"superagent": "^7.1.1"
|
||||
}
|
||||
}
|
||||
@ -1,15 +0,0 @@
|
||||
id:
|
||||
type: text
|
||||
category: system
|
||||
desc: Osu OAuth AppID
|
||||
name: id
|
||||
secret:
|
||||
type: text
|
||||
category: system
|
||||
desc: Osu OAuth Secret
|
||||
name: secret
|
||||
proxy:
|
||||
type: text
|
||||
category: system
|
||||
desc: Osu OAuth Proxy
|
||||
name: proxy
|
||||
@ -1,3 +1,3 @@
|
||||
<div data-fragment-id="problem_stat">
|
||||
<p>{{ _('{0} problems').format(pcount) }}</p>
|
||||
<p>{{ _('{0} problems' if pcountRelation == 'eq' else 'Over {0} problems').format(pcount) }}</p>
|
||||
</div>
|
||||
Loading…
Reference in New Issue