From e06c86e9dd182d998a09cc0cc7dd685f5153b72a Mon Sep 17 00:00:00 2001 From: undefined Date: Tue, 23 Aug 2022 00:40:21 +0800 Subject: [PATCH] core&ui: goodbye, minio (#417) --- .devcontainer/Dockerfile | 1 - .devcontainer/docker-compose.yml | 10 +- .gitpod.Dockerfile | 5 +- .gitpod.yml | 1 - install/docker/backend/entrypoint.sh | 1 - install/docker/docker-compose.yml | 15 +- install/install.js | 15 -- install/nix/default.nix | 1 - install/nix/env.nix | 1 - install/reset.sh | 3 +- install/ubuntu-2004.sh | 9 - packages/hydrooj/package.json | 3 +- packages/hydrooj/src/entry/cli.ts | 3 +- packages/hydrooj/src/entry/worker.ts | 3 +- packages/hydrooj/src/handler/manage.ts | 72 ++---- packages/hydrooj/src/handler/misc.ts | 22 ++ packages/hydrooj/src/model/setting.ts | 8 - packages/hydrooj/src/model/storage.ts | 23 +- packages/hydrooj/src/options.ts | 8 +- packages/hydrooj/src/service/server.ts | 20 +- packages/hydrooj/src/service/storage.ts | 160 +++++++++--- packages/hydrooj/src/settings.ts | 231 ++++++++---------- packages/hydrooj/src/upgrade.ts | 34 ++- packages/ui-default/build/config/webpack.ts | 5 +- .../components/discussion/comments.page.tsx | 1 - .../components/editor/cmeditor.page.ts | 1 - .../components/monaco/languages/yaml.ts | 2 +- packages/ui-default/handler.ts | 11 + packages/ui-default/package.json | 3 +- packages/ui-default/pages/setting.page.tsx | 29 +++ .../ui-default/templates/manage_base.html | 1 + .../ui-default/templates/manage_config.html | 24 ++ .../templates/partials/setting.html | 3 +- packages/vjudge/src/providers/codeforces.ts | 1 - 34 files changed, 409 insertions(+), 321 deletions(-) create mode 100644 packages/ui-default/pages/setting.page.tsx create mode 100644 packages/ui-default/templates/manage_config.html diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 05f0a7d1..4405516b 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -18,4 +18,3 @@ RUN npm i pm2 yarn -g && \ sudo apt-get install clang -y --no-install-recommends && \ mkdir -p ~/.hydro && \ echo '{"host":"127.0.0.1","port":"27017","name":"hydro","username":"","password":""}' >~/.hydro/config.json && \ - echo "MINIO_ACCESS_KEY=minioadmin\nMINIO_SECRET_KEY=minioadmin" >~/.hydro/env diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index 5ff5e1d0..646de05d 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -9,6 +9,7 @@ services: VARIANT: 16 volumes: - ..:/workspace:cached + - testdata:/data/file command: sleep infinity network_mode: service:db db: @@ -16,14 +17,7 @@ services: restart: unless-stopped volumes: - mongodb-data:/data/db - minio: - image: minio/minio:latest - restart: unless-stopped - volumes: - - minio-data:/data - network_mode: service:db - command: minio server /data --console-address ":9001" volumes: mongodb-data: - minio-data: \ No newline at end of file + testdata: \ No newline at end of file diff --git a/.gitpod.Dockerfile b/.gitpod.Dockerfile index 2388afa4..a472f5a4 100644 --- a/.gitpod.Dockerfile +++ b/.gitpod.Dockerfile @@ -1,10 +1,9 @@ FROM gitpod/workspace-mongodb RUN npm i pm2 -g && \ - sudo wget https://dl.min.io/server/minio/release/linux-amd64/minio -O /usr/bin/minio && \ - sudo chmod 755 /usr/bin/minio && \ sudo apt-get update && \ sudo apt-get install clang -y && \ cargo install sonic-server --version 1.3.0 && \ mkdir -p /home/gitpod/.hydro && \ echo '{"host":"127.0.0.1","port":"27017","name":"hydro","username":"","password":""}' >/home/gitpod/.hydro/config.json && \ - echo "MINIO_ACCESS_KEY=hydro\nMINIO_SECRET_KEY=hydrohydro" >/home/gitpod/.hydro/env + mkdir /data/file -p + chmod 777 /data/file \ No newline at end of file diff --git a/.gitpod.yml b/.gitpod.yml index ddb79476..a6dd1417 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -4,7 +4,6 @@ tasks: - init: | yarn pm2 start mongod - pm2 start "MINIO_ACCESS_KEY=hydro MINIO_SECRET_KEY=hydrohydro minio server /home/gitpod/file" --name minio yarn build:ui npx hydrooj cli system set server.port 2333 npx hydrooj cli user create root@hydro.local root rootroot 2 diff --git a/install/docker/backend/entrypoint.sh b/install/docker/backend/entrypoint.sh index e65aa608..573709cf 100644 --- a/install/docker/backend/entrypoint.sh +++ b/install/docker/backend/entrypoint.sh @@ -12,7 +12,6 @@ fi if [ ! -f "$ROOT/first" ]; then echo "for marking use only!" > "$ROOT/first" - hydrooj cli system set file.endPoint http://oj-minio:9000/ hydrooj cli user create systemjudge@systemjudge.local root rootroot hydrooj cli user setSuperAdmin 2 diff --git a/install/docker/docker-compose.yml b/install/docker/docker-compose.yml index e9052be1..28660944 100644 --- a/install/docker/docker-compose.yml +++ b/install/docker/docker-compose.yml @@ -1,16 +1,6 @@ version: '3.7' services: - oj-minio: - image: minio/minio - container_name: oj-minio - command: server /data - restart: always - volumes: - - ./data/minio:/data - environment: - - MINIO_ACCESS_KEY=CHANGE_THIS - - MINIO_SECRET_KEY=CHANGE_THIS # Warning: mongodb here is not password-protected. # DO NOT EXPOSE THIS SERVICE TO THE PUBLIC. @@ -26,13 +16,10 @@ services: container_name: oj-backend restart: always depends_on: - - oj-minio - oj-mongo volumes: + - ./data/file:/data/file - ./data/backend:/root/.hydro - environment: - - MINIO_ACCESS_KEY=CHANGE_THIS - - MINIO_SECRET_KEY=CHANGE_THIS ports: - "0.0.0.0:80:8888" # In docker mode, change THIS port instead of port in system settings! diff --git a/install/install.js b/install/install.js index 83fc4aaf..d7b35054 100644 --- a/install/install.js +++ b/install/install.js @@ -20,7 +20,6 @@ const locales = { 'error.nodeVersionPraseFail': '无法解析 Node 版本号，请尝试手动安装。', 'install.pm2': '正在安装 PM2...', 'install.createDatabaseUser': '正在创建数据库用户...', - 'install.minio': '正在安装 MinIO...', 'install.compiler': '正在安装编译器...', 'install.hydro': '正在安装 Hydro...', 'install.done': 'Hydro 安装成功！', @@ -45,7 +44,6 @@ const locales = { 'error.nodeVersionPraseFail': 'Unable to parse Node version, please try to install manually.', 'install.pm2': 'Installing PM2...', 'install.createDatabaseUser': 'Creating database user...', - 'install.minio': 'Installing MinIO...', 'install.compiler': 'Installing compiler...', 'install.hydro': 'Installing Hydro...', 'install.done': 'Hydro installation completed!', @@ -83,8 +81,6 @@ if (!cpuInfoFile.includes('avx2')) { let migration; let retry = 0; log.info('install.start'); -const MINIO_ACCESS_KEY = randomstring(32); -const MINIO_SECRET_KEY = randomstring(32); let DATABASE_PASSWORD = randomstring(32); // TODO read from args const CN = true; @@ -193,13 +189,6 @@ To disable this feature, checkout our sourcecode.`); skip: () => !exec('pm2 -v').code, operations: ['yarn global add pm2'], }, - { - init: 'install.minio', - skip: () => !exec('minio -v').code, - operations: [ - 'nix-env -iA nixpkgs.minio', - ], - }, { init: 'install.compiler', operations: [ @@ -249,8 +238,6 @@ To disable this feature, checkout our sourcecode.`); operations: [ ['pm2 stop all', { ignore: true }], () => fs.writefile(`${__env.HOME}/.hydro/mount.yaml`, mount), - `echo "MINIO_ACCESS_KEY=${MINIO_ACCESS_KEY}\nMINIO_SECRET_KEY=${MINIO_SECRET_KEY}" >/root/.hydro/env`, - `pm2 start "MINIO_ACCESS_KEY=${MINIO_ACCESS_KEY} MINIO_SECRET_KEY=${MINIO_SECRET_KEY} minio server /data/file" --name minio`, 'pm2 start mongod --name mongodb -- --auth --bind_ip 0.0.0.0', () => sleep(1000), `pm2 start bash --name hydro-sandbox -- -c "ulimit -s unlimited && hydro-sandbox -mount-conf ${__env.HOME}/.hydro/mount.yaml"`, @@ -291,8 +278,6 @@ To disable this feature, checkout our sourcecode.`); () => log.info('extra.restartTerm'), () => log.info('extra.dbUser'), () => log.info('extra.dbPassword', DATABASE_PASSWORD), - () => log.info('MINIO_ACCESS_KEY=%s', MINIO_ACCESS_KEY), - () => log.info('MINIO_SECRET_KEY=%s', MINIO_SECRET_KEY), ], }, ]; diff --git a/install/nix/default.nix b/install/nix/default.nix index 5b150017..12a7f443 100644 --- a/install/nix/default.nix +++ b/install/nix/default.nix @@ -15,7 +15,6 @@ in pkgs.dockerTools.buildImage { name = "hydro-web"; paths = [ hydro.mongodb4 - pkgs.minio pkgs.nodejs pkgs.yarn ]; diff --git a/install/nix/env.nix b/install/nix/env.nix index 0d701c31..dcab5d48 100644 --- a/install/nix/env.nix +++ b/install/nix/env.nix @@ -6,7 +6,6 @@ in pkgs.buildEnv { name = "hydro-env"; paths = [ mongo - pkgs.minio pkgs.nodejs pkgs.yarn pkgs.git diff --git a/install/reset.sh b/install/reset.sh index d19c3faa..dd89402c 100644 --- a/install/reset.sh +++ b/install/reset.sh @@ -15,7 +15,6 @@ mongo 127.0.0.1:27017/hydro /tmp/createUser.js echo "{\"host\":\"127.0.0.1\",\"port\":\"27017\",\"name\":\"hydro\",\"username\":\"hydro\",\"password\":\"$db_password\"}" >~/.hydro/config.json pm2 stop mongod pm2 del mongod -pm2 restart minio pm2 start mongodb -pm2 restart hydrooj +pm2 restart all pm2 save diff --git a/install/ubuntu-2004.sh b/install/ubuntu-2004.sh index 67014965..e9642de8 100644 --- a/install/ubuntu-2004.sh +++ b/install/ubuntu-2004.sh @@ -7,8 +7,6 @@ echo "详情请参阅文档 -> https://hydro.js.org" echo "使用 Ctrl-C 退出该脚本，或是等待十秒后继续。" echo "Will continue installation in 10 secs, press Ctrl-C to exit" sleep 10 -MINIO_ACCESS_KEY=$(cat /dev/urandom | head -n 10 | md5sum | head -c 20) -MINIO_SECRET_KEY=$(cat /dev/urandom | head -n 10 | md5sum | head -c 20) # Basic echo "apt-get update" @@ -54,11 +52,6 @@ pm2 del mongod >/dev/null echo 'Starting mongodb' pm2 start "mongod --auth --bind_ip 0.0.0.0" --name mongodb -# Install MinIO -wget http://dl.minio.org.cn/server/minio/release/linux-amd64/minio -chmod +x minio -pm2 start "MINIO_ACCESS_KEY=$MINIO_ACCESS_KEY MINIO_SECRET_KEY=$MINIO_SECRET_KEY ./minio server /data/file" --name minio - # Install Compiler echo 'Installing g++' apt-get install -y g++ >/dev/null @@ -81,5 +74,3 @@ pm2 save echo "Done" echo "Database username: hydro" echo "Database password: $db_password" -echo "MINIO_ACCESS_KEY=$MINIO_ACCESS_KEY -MINIO_SECRET_KEY=$MINIO_SECRET_KEY" >~/.hydro/env diff --git a/packages/hydrooj/package.json b/packages/hydrooj/package.json index 3bc9c586..d268a0c3 100644 --- a/packages/hydrooj/package.json +++ b/packages/hydrooj/package.json @@ -1,6 +1,6 @@ { "name": "hydrooj", - "version": "3.15.19", + "version": "3.16.0", "bin": "bin/hydrooj.js", "main": "src/loader", "module": "src/loader", @@ -15,7 +15,6 @@ "@graphql-tools/schema": "^8.5.1", "@hydrooj/utils": "workspace:*", "adm-zip": "0.5.5", - "ajv": "^8.11.0", "cac": "^6.7.12", "cookies": "^0.8.0", "detect-browser": "^5.3.0", diff --git a/packages/hydrooj/src/entry/cli.ts b/packages/hydrooj/src/entry/cli.ts index e520e293..3119bcc1 100644 --- a/packages/hydrooj/src/entry/cli.ts +++ b/packages/hydrooj/src/entry/cli.ts @@ -88,8 +88,9 @@ export async function load() { require('../options'); const opts = options(); await db.start(opts); + await require('../settings').loadConfig(); const storage = require('../service/storage'); - await storage.start(); + await storage.loadStorageService(); require('../lib/index'); await lib(pending, fail); const systemModel = require('../model/system'); diff --git a/packages/hydrooj/src/entry/worker.ts b/packages/hydrooj/src/entry/worker.ts index e94e3f45..f4577695 100644 --- a/packages/hydrooj/src/entry/worker.ts +++ b/packages/hydrooj/src/entry/worker.ts @@ -37,12 +37,13 @@ export async function load() { if (detail) logger.info('finish: locale/template/static'); const opts = options(); await db.start(opts); + await require('../settings').loadConfig(); if (detail) logger.info('finish: db.connect'); const modelSystem = require('../model/system'); await modelSystem.runConfig(); if (detail) logger.info('finish: config'); const storage = require('../service/storage'); - await storage.start(); + await storage.loadStorageService(); if (detail) logger.info('finish: storage.connect'); require('../lib/index'); if (detail) logger.info('finish: lib.builtin'); diff --git a/packages/hydrooj/src/handler/manage.ts b/packages/hydrooj/src/handler/manage.ts index f8485826..e6de73a4 100644 --- a/packages/hydrooj/src/handler/manage.ts +++ b/packages/hydrooj/src/handler/manage.ts @@ -1,7 +1,7 @@ import { exec } from 'child_process'; import { inspect } from 'util'; -import Ajv from 'ajv'; import * as yaml from 'js-yaml'; +import Schema from 'schemastery'; import * as check from '../check'; import { BadRequestError, ValidationError } from '../error'; import { @@ -19,12 +19,10 @@ import { Connection, ConnectionHandler, Handler, param, Route, Types, } from '../service/server'; -import { schema } from '../settings'; +import { configSource, saveConfig, SystemSettings } from '../settings'; import * as judge from './judge'; const logger = new Logger('manage'); -const ajv = new Ajv({ useDefaults: true }); -const validator = ajv.compile(schema); function set(key: string, value: any) { if (setting.SYSTEM_SETTINGS_BY_KEY[key]) { @@ -156,55 +154,12 @@ class SystemSettingHandler extends SystemHandler { for (const s of this.response.body.settings) { this.response.body.current[s.key] = system.get(s.key); } - const hide = []; - const raw = system.get('_') || ''; - let config = yaml.load(raw); - const valid = validator(config); - if (!(raw.trim() && valid)) { - const data = {}; - for (const key in schema.properties) { - data[key] = {}; - for (const subkey in (schema.definitions[key] as any).properties || {}) { - if (system.get(`${key}.${subkey}`)) { - data[key][subkey] = system.get(`${key}.${subkey}`); - if ((schema.definitions[key] as any).properties[subkey]?.writeOnly) { - if (data[key][subkey] instanceof Array) { - hide.push(...data[key][subkey]); - } else hide.push(data[key][subkey]); - } - } - } - } - validator(data); - config = yaml.dump(data); - } else config = yaml.dump(config); - this.response.body.hide = hide; - this.response.body.config = config; } async post(args: any) { const tasks = []; const booleanKeys = args.booleanKeys || {}; delete args.booleanKeys; - if (args._) { - if (typeof args._ !== 'string') throw new ValidationError('config'); - try { - const payload = yaml.load(args._); - const valid = validator(payload); - if (!valid) { - throw new ValidationError('config', null, validator.errors[0]); - } - for (const key in payload) { - for (const subkey in payload[key]) { - tasks.push(system.set(`${key}.${subkey}`, payload[key][subkey])); - } - } - } catch (e) { - throw new ValidationError('config', null, e.message); - } - await system.set('_', args._); - delete args._; - } for (const key in args) { if (typeof args[key] === 'object') { for (const subkey in args[key]) { @@ -228,10 +183,25 @@ class SystemSettingHandler extends SystemHandler { } } -class SystemSettingSchemaHandler extends SystemHandler { +class SystemConfigHandler extends SystemHandler { async get() { - this.response.body = JSON.stringify(schema); - this.response.type = 'application/json'; + this.response.template = 'manage_config.html'; + this.response.body = { + schema: Schema.intersect(SystemSettings).toJSON(), + value: configSource, + }; + } + + @param('value', Types.String) + async post(domainId: string, value: string) { + let config; + try { + config = yaml.load(value); + Schema.intersect(SystemSettings)(config); + } catch (e) { + throw new ValidationError('value'); + } + await saveConfig(config); } } @@ -294,7 +264,7 @@ async function apply() { Route('manage_dashboard', '/manage/dashboard', SystemDashboardHandler); Route('manage_script', '/manage/script', SystemScriptHandler); Route('manage_setting', '/manage/setting', SystemSettingHandler); - Route('manage_setting_schema', '/manage/setting/schema.json', SystemSettingSchemaHandler); + Route('manage_config', '/manage/config', SystemConfigHandler); Route('manage_user_import', '/manage/userimport', SystemUserImportHandler); Connection('manage_check', '/manage/check-conn', SystemCheckConnHandler); } diff --git a/packages/hydrooj/src/handler/misc.ts b/packages/hydrooj/src/handler/misc.ts index 3caed93a..725af302 100644 --- a/packages/hydrooj/src/handler/misc.ts +++ b/packages/hydrooj/src/handler/misc.ts @@ -1,9 +1,11 @@ /* eslint-disable camelcase */ import { statSync } from 'fs'; import { pick } from 'lodash'; +import { lookup } from 'mime-types'; import { BadRequestError, ForbiddenError, ValidationError, } from '../error'; +import { md5 } from '../lib/crypto'; import { PRIV } from '../model/builtin'; import * as oplog from '../model/oplog'; import storage from '../model/storage'; @@ -12,6 +14,8 @@ import user from '../model/user'; import { Handler, param, post, Route, Types, } from '../service/server'; +import { encodeRFC5987ValueChars } from '../service/storage'; +import { builtinConfig } from '../settings'; import { sortFiles } from '../utils'; class SwitchLanguageHandler extends Handler { @@ -101,6 +105,23 @@ export class FSDownloadHandler extends Handler { } } +export class StorageHandler extends Handler { + @param('target', Types.Name) + @param('filename', Types.Name, true) + @param('expire', Types.UnsignedInt) + @param('secret', Types.String) + async get(domainId: string, target: string, filename = '', expire: number, secret: string) { + const expected = md5(`${target}/${expire}/${builtinConfig.file.secret}`); + if (expire < Date.now()) throw new ForbiddenError('Link expired'); + if (secret !== expected) throw new ForbiddenError('Invalid secret'); + this.response.body = await storage.get(target); + this.response.type = (target.endsWith('.out') || target.endsWith('.ans')) + ? 'text/plain' + : lookup(target) || 'application/octet-stream'; + if (filename) this.response.disposition = `attachment; filename="${encodeRFC5987ValueChars(filename)}"`; + } +} + export class SwitchAccountHandler extends Handler { @param('uid', Types.Int) async get(domainId: string, uid: number) { @@ -113,6 +134,7 @@ export async function apply() { Route('switch_language', '/language/:lang', SwitchLanguageHandler); Route('home_files', '/file', FilesHandler); Route('fs_download', '/file/:uid/:filename', FSDownloadHandler); + Route('storage', '/storage', StorageHandler); Route('switch_account', '/account', SwitchAccountHandler, PRIV.PRIV_EDIT_SYSTEM); } diff --git a/packages/hydrooj/src/model/setting.ts b/packages/hydrooj/src/model/setting.ts index 0d3db2f0..6eb05067 100644 --- a/packages/hydrooj/src/model/setting.ts +++ b/packages/hydrooj/src/model/setting.ts @@ -183,14 +183,6 @@ const ignoreUA = [ ].join('\n'); SystemSetting( - Setting('setting_file', 'file.endPoint', 'http://127.0.0.1:9000', 'text', 'file.endPoint', 'Storage engine endPoint'), - Setting('setting_file', 'file.accessKey', null, 'text', 'file.accessKey', 'Storage engine accessKey'), - Setting('setting_file', 'file.secretKey', null, 'password', 'file.secretKey', 'Storage engine secret', FLAG_SECRET), - Setting('setting_file', 'file.bucket', 'hydro', 'text', 'file.bucket', 'Storage engine bucket'), - Setting('setting_file', 'file.region', 'us-east-1', 'text', 'file.region', 'Storage engine region'), - Setting('setting_file', 'file.pathStyle', true, 'boolean', 'file.pathStyle', 'pathStyle endpoint'), - Setting('setting_file', 'file.endPointForUser', '/fs/', 'text', 'file.endPointForUser', 'EndPoint for user'), - Setting('setting_file', 'file.endPointForJudge', '/fs/', 'text', 'file.endPointForJudge', 'EndPoint for judge'), Setting('setting_smtp', 'smtp.user', null, 'text', 'smtp.user', 'SMTP Username'), Setting('setting_smtp', 'smtp.pass', null, 'password', 'smtp.pass', 'SMTP Password', FLAG_SECRET), Setting('setting_smtp', 'smtp.host', null, 'text', 'smtp.host', 'SMTP Server Host'), diff --git a/packages/hydrooj/src/model/storage.ts b/packages/hydrooj/src/model/storage.ts index d35ffd91..13ceac7e 100644 --- a/packages/hydrooj/src/model/storage.ts +++ b/packages/hydrooj/src/model/storage.ts @@ -1,7 +1,6 @@ import { extname } from 'path'; import { escapeRegExp } from 'lodash'; import { lookup } from 'mime-types'; -import { ItemBucketMetadata } from 'minio'; import moment from 'moment'; import { nanoid } from 'nanoid'; import type { Readable } from 'stream'; @@ -14,11 +13,8 @@ import TaskModel from './task'; export class StorageModel { static coll = db.collection('storage'); - static async put(path: string, file: string | Buffer | Readable, meta?: ItemBucketMetadata, owner?: number); - static async put(path: string, file: string | Buffer | Readable, owner?: number); - static async put(path: string, file: string | Buffer | Readable, arg0?: ItemBucketMetadata | number, arg1?: number) { - const meta = typeof arg0 === 'object' ? arg0 : {}; - const owner = (typeof arg0 === 'number' ? arg0 : arg1) ?? 1; + static async put(path: string, file: string | Buffer | Readable, owner?: number) { + const meta = {}; await StorageModel.del([path]); meta['Content-Type'] = (path.endsWith('.ans') || path.endsWith('.out')) ? 'text/plain' @@ -27,7 +23,7 @@ export class StorageModel { // Make sure id is not used // eslint-disable-next-line no-await-in-loop while (await StorageModel.coll.findOne({ _id })) _id = `${nanoid(3)}/${nanoid()}${extname(path)}`; - await storage.put(_id, file, meta); + await storage.put(_id, file); const { metaData, size, etag } = await storage.getMeta(_id); await StorageModel.coll.insertOne({ _id, meta: metaData, path, size, etag, lastModified: new Date(), owner, @@ -63,15 +59,10 @@ export class StorageModel { static async list(target: string, recursive = true) { if (target.includes('..') || target.includes('//')) throw new Error('Invalid path'); if (target.length && !target.endsWith('/')) target += '/'; - const results = recursive - ? await StorageModel.coll.find({ - path: { $regex: new RegExp(`^${escapeRegExp(target)}`, 'i') }, - autoDelete: null, - }).toArray() - : await StorageModel.coll.find({ - path: { $regex: new RegExp(`^${escapeRegExp(target)}[^/]+$`) }, - autoDelete: null, - }).toArray(); + const results = await StorageModel.coll.find({ + path: { $regex: new RegExp(`^${escapeRegExp(target)}${recursive ? '' : '[^/]+$'}`, 'i') }, + autoDelete: null, + }).toArray(); return results.map((i) => ({ ...i, name: i.path.split(target)[1], prefix: target, })); diff --git a/packages/hydrooj/src/options.ts b/packages/hydrooj/src/options.ts index 34eab251..2833944d 100644 --- a/packages/hydrooj/src/options.ts +++ b/packages/hydrooj/src/options.ts @@ -1,6 +1,6 @@ -import fs from 'fs'; import os from 'os'; import path from 'path'; +import { existsSync, readFileSync } from 'fs-extra'; import { findFileSync } from '@hydrooj/utils/lib/utils'; import { Logger } from './logger'; @@ -8,8 +8,8 @@ const logger = new Logger('options'); export = function load() { const envFile = path.resolve(os.homedir(), '.hydro', 'env'); - if (fs.existsSync(envFile)) { - const content = fs.readFileSync(envFile).toString().replace(/\r/g, ''); + if (existsSync(envFile)) { + const content = readFileSync(envFile).toString().replace(/\r/g, ''); for (const line of content.split('\n')) { if (!line.includes('=')) continue; process.env[line.split('=')[0]] = line.split('=')[1].trim(); @@ -19,7 +19,7 @@ export = function load() { if (!f) return null; let result: any = {}; try { - result = JSON.parse(fs.readFileSync(f).toString()); + result = JSON.parse(readFileSync(f, 'utf-8')); } catch (e) { logger.error('Cannot read config file %o', e); result = {}; diff --git a/packages/hydrooj/src/service/server.ts b/packages/hydrooj/src/service/server.ts index df14af2c..fb901c66 100644 --- a/packages/hydrooj/src/service/server.ts +++ b/packages/hydrooj/src/service/server.ts @@ -20,6 +20,7 @@ import { PERM, PRIV } from '../model/builtin'; import * as opcount from '../model/opcount'; import * as system from '../model/system'; import { User } from '../model/user'; +import { builtinConfig } from '../settings'; import { errorMessage } from '../utils'; import * as bus from './bus'; import * as decorators from './decorators'; @@ -29,6 +30,7 @@ import rendererLayer from './layers/renderer'; import responseLayer from './layers/response'; import userLayer from './layers/user'; import { Router } from './router'; +import { encodeRFC5987ValueChars } from './storage'; export * from './decorators'; @@ -109,11 +111,17 @@ const serializer = (showDisplayName = false) => (k: string, v: any) => { export async function prepare() { app.keys = system.get('session.keys') as unknown as string[]; - app.use(proxy('/fs', { - target: system.get('file.endPoint'), + const proxyMiddleware = proxy('/fs', { + target: builtinConfig.file.endPoint, changeOrigin: true, rewrite: (p) => p.replace('/fs', ''), - })); + }); + app.use(async (ctx, next) => { + if (!ctx.path.startsWith('/fs/')) return await next(); + if (ctx.request.search.toLowerCase().includes('x-amz-credential')) return await proxyMiddleware(ctx, next); + ctx.request.path = ctx.path = ctx.path.split('/fs')[1]; + return await next(); + }); app.use(Compress()); for (const dir of global.publicDirs) { app.use(cache(dir, { @@ -122,9 +130,9 @@ export async function prepare() { } if (process.env.DEV) { app.use(async (ctx: Context, next: Function) => { - const startTime = new Date().getTime(); + const startTime = Date.now(); await next(); - const endTime = new Date().getTime(); + const endTime = Date.now(); if (ctx.nolog || ctx.response.headers.nolog) return; ctx._remoteAddress = ctx.request.ip; logger.debug(`${ctx.request.method} /${ctx.domainId || 'system'}${ctx.request.path} \ @@ -216,7 +224,7 @@ export class Handler extends HandlerCommon { this.response.body = data; this.response.template = null; this.response.type = 'application/octet-stream'; - if (name) this.response.disposition = `attachment; filename="${encodeURIComponent(name)}"`; + if (name) this.response.disposition = `attachment; filename="${encodeRFC5987ValueChars(name)}"`; } async init() { diff --git a/packages/hydrooj/src/service/storage.ts b/packages/hydrooj/src/service/storage.ts index 787b385c..9a1fdc2c 100644 --- a/packages/hydrooj/src/service/storage.ts +++ b/packages/hydrooj/src/service/storage.ts @@ -1,9 +1,16 @@ +import { dirname, resolve } from 'path'; import { Readable } from 'stream'; import { URL } from 'url'; -import { createReadStream } from 'fs-extra'; +import { + copyFile, createReadStream, ensureDir, + remove, stat, writeFile, +} from 'fs-extra'; +import { lookup } from 'mime-types'; import { BucketItem, Client, ItemBucketMetadata } from 'minio'; +import { md5 } from '../lib/crypto'; import { Logger } from '../logger'; -import * as system from '../model/system'; +import { builtinConfig } from '../settings'; +import { MaybeArray } from '../typeutils'; const logger = new Logger('storage'); @@ -18,38 +25,38 @@ interface StorageOptions { endPointForJudge?: string; } -interface MinioEndpointConfig { +interface EndpointConfig { endPoint: string; port: number; useSSL: boolean; } -function parseMainEndpointUrl(endpoint: string): MinioEndpointConfig { +function parseMainEndpointUrl(endpoint: string): EndpointConfig { if (!endpoint) throw new Error('Empty endpoint'); const url = new URL(endpoint); - const result: Partial = {}; - if (url.pathname !== '/') throw new Error('Main MinIO endpoint URL of a sub-directory is not supported.'); + const result: Partial = {}; + if (url.pathname !== '/') throw new Error('Main endpoint URL of a sub-directory is not supported.'); if (url.username || url.password || url.hash || url.search) { - throw new Error('Authorization, search parameters and hash are not supported for main MinIO endpoint URL.'); + throw new Error('Authorization, search parameters and hash are not supported for main endpoint URL.'); } if (url.protocol === 'http:') result.useSSL = false; else if (url.protocol === 'https:') result.useSSL = true; else { throw new Error( - `Invalid protocol "${url.protocol}" for main MinIO endpoint URL. Only HTTP and HTTPS are supported.`, + `Invalid protocol "${url.protocol}" for main endpoint URL. Only HTTP and HTTPS are supported.`, ); } result.endPoint = url.hostname; result.port = url.port ? Number(url.port) : result.useSSL ? 443 : 80; - return result as MinioEndpointConfig; + return result as EndpointConfig; } function parseAlternativeEndpointUrl(endpoint: string): (originalUrl: string) => string { if (!endpoint) return (originalUrl) => originalUrl; const pathonly = endpoint.startsWith('/'); if (pathonly) endpoint = `https://localhost${endpoint}`; const url = new URL(endpoint); - if (url.hash || url.search) throw new Error('Search parameters and hash are not supported for alternative MinIO endpoint URL.'); - if (!url.pathname.endsWith('/')) throw new Error("Alternative MinIO endpoint URL's pathname must ends with '/'."); + if (url.hash || url.search) throw new Error('Search parameters and hash are not supported for alternative endpoint URL.'); + if (!url.pathname.endsWith('/')) throw new Error("Alternative endpoint URL's pathname must ends with '/'."); return (originalUrl) => { const parsedOriginUrl = new URL(originalUrl); const replaced = new URL(parsedOriginUrl.pathname.slice(1) + parsedOriginUrl.search + parsedOriginUrl.hash, url).toString(); @@ -59,7 +66,7 @@ function parseAlternativeEndpointUrl(endpoint: string): (originalUrl: string) => }; } // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/encodeURIComponent -function encodeRFC5987ValueChars(str: string) { +export function encodeRFC5987ValueChars(str: string) { return ( encodeURIComponent(str) // Note that although RFC3986 reserves "!", RFC5987 does not, @@ -72,7 +79,7 @@ function encodeRFC5987ValueChars(str: string) { ); } -class StorageService { +class RemoteStorageService { public client: Client; public error = ''; public opts: StorageOptions; @@ -80,13 +87,17 @@ class StorageService { async start() { try { - const [ - endPoint, accessKey, secretKey, bucket, region, - pathStyle, endPointForUser, endPointForJudge, - ] = system.getMany([ - 'file.endPoint', 'file.accessKey', 'file.secretKey', 'file.bucket', 'file.region', - 'file.pathStyle', 'file.endPointForUser', 'file.endPointForJudge', - ]); + logger.info('Starting storage service with endpoint:', builtinConfig.file.endPoint); + const { + endPoint, + accessKey, + secretKey, + bucket, + region, + pathStyle, + endPointForUser, + endPointForJudge, + } = builtinConfig.file; this.opts = { endPoint, accessKey, @@ -97,11 +108,6 @@ class StorageService { endPointForUser, endPointForJudge, }; - if (process.env.MINIO_ACCESS_KEY) { - logger.info('Using MinIO key from environment variables'); - this.opts.accessKey = process.env.MINIO_ACCESS_KEY; - this.opts.secretKey = process.env.MINIO_SECRET_KEY; - } this.client = new Client({ ...parseMainEndpointUrl(this.opts.endPoint), pathStyle: this.opts.pathStyle, @@ -133,7 +139,7 @@ class StorageService { if (target.includes('..') || target.includes('//')) throw new Error('Invalid path'); if (typeof file === 'string') file = createReadStream(file); try { - return await this.client.putObject(this.opts.bucket, target, file, meta); + await this.client.putObject(this.opts.bucket, target, file, meta); } catch (e) { e.stack = new Error().stack; throw e; @@ -154,11 +160,7 @@ class StorageService { async del(target: string | string[]) { if (typeof target === 'string') { if (target.includes('..') || target.includes('//')) throw new Error('Invalid path'); - } else { - for (const t of target) { - if (t.includes('..') || t.includes('//')) throw new Error('Invalid path'); - } - } + } else if (target.find((t) => t.includes('..') || t.includes('//'))) throw new Error('Invalid path'); try { if (typeof target === 'string') return await this.client.removeObject(this.opts.bucket, target); return await this.client.removeObjects(this.opts.bucket, target); @@ -173,7 +175,7 @@ class StorageService { if (target.includes('..') || target.includes('//')) throw new Error('Invalid path'); try { const stream = this.client.listObjects(this.opts.bucket, target, recursive); - return await new Promise((resolve, reject) => { + return await new Promise((r, reject) => { const results: BucketItem[] = []; stream.on('data', (result) => { if (result.size) { @@ -184,7 +186,7 @@ class StorageService { }); } }); - stream.on('end', () => resolve(results)); + stream.on('end', () => r(results)); stream.on('error', reject); }); } catch (e) { @@ -237,6 +239,92 @@ class StorageService { } } -const service = new StorageService(); -global.Hydro.service.storage = service; -export = service; +class LocalStorageService { + client: null; + error: null; + dir: string; + opts: null; + private replaceWithAlternativeUrlFor: Record<'user' | 'judge', (originalUrl: string) => string>; + + async start() { + logger.debug('Loading local storage service with path:', builtinConfig.file.path); + await ensureDir(builtinConfig.file.path); + this.dir = builtinConfig.file.path; + this.replaceWithAlternativeUrlFor = { + user: parseAlternativeEndpointUrl(builtinConfig.file.endPointForUser), + judge: parseAlternativeEndpointUrl(builtinConfig.file.endPointForJudge), + }; + } + + async put(target: string, file: string | Buffer | Readable) { + if (target.includes('..') || target.includes('//')) throw new Error('Invalid path'); + target = resolve(this.dir, target); + await ensureDir(dirname(target)); + if (typeof file === 'string') await copyFile(file, target); + else await writeFile(target, file); + } + + async get(target: string, path?: string) { + if (target.includes('..') || target.includes('//')) throw new Error('Invalid path'); + target = resolve(this.dir, target); + if (path) await copyFile(target, path); + return createReadStream(target); + } + + async del(target: MaybeArray) { + const targets = typeof target === 'string' ? [target] : target; + if (targets.find((i) => i.includes('..') || i.includes('//'))) throw new Error('Invalid path'); + await Promise.all(targets.map((i) => remove(resolve(this.dir, i)))); + } + + async getMeta(target: string) { + if (target.includes('..') || target.includes('//')) throw new Error('Invalid path'); + target = resolve(this.dir, target); + const file = await stat(target); + return { + size: file.size, + etag: Buffer.from(target).toString('base64'), + lastModified: file.mtime, + metaData: { + 'Content-Type': (target.endsWith('.ans') || target.endsWith('.out')) + ? 'text/plain' + : lookup(target) || 'application/octet-stream', + 'Content-Length': file.size, + }, + }; + } + + async signDownloadLink(target: string, filename = '', noExpire = false, useAlternativeEndpointFor?: 'user' | 'judge'): Promise { + if (target.includes('..') || target.includes('//')) throw new Error('Invalid path'); + const url = new URL('https://localhost/storage'); + url.searchParams.set('target', target); + if (filename) url.searchParams.set('filename', filename); + const expire = (Date.now() + (noExpire ? 7 * 24 * 3600 : 600) * 1000).toString(); + url.searchParams.set('expire', expire); + url.searchParams.set('secret', md5(`${target}/${expire}/${builtinConfig.file.secret}`)); + if (useAlternativeEndpointFor) return this.replaceWithAlternativeUrlFor[useAlternativeEndpointFor](url.toString()); + return `/${url.toString().split('localhost/')[1]}`; + } + + async signUpload() { + throw new Error('Not implemented'); + } + + async list() { + throw new Error('deprecated'); + } +} + +let service; // eslint-disable-line import/no-mutable-exports + +export async function loadStorageService() { + service = builtinConfig.file.type === 's3' ? new RemoteStorageService() : new LocalStorageService(); + global.Hydro.service.storage = service; + await service.start(); +} + +export default new Proxy({}, { + get(self, key) { + return service[key]; + }, +}) as RemoteStorageService | LocalStorageService; diff --git a/packages/hydrooj/src/settings.ts b/packages/hydrooj/src/settings.ts index 602a4eb1..d7d1cb36 100644 --- a/packages/hydrooj/src/settings.ts +++ b/packages/hydrooj/src/settings.ts @@ -1,136 +1,111 @@ -import { JSONSchema7Definition } from 'json-schema'; +import yaml from 'js-yaml'; +import { nanoid } from 'nanoid'; +import Schema from 'schemastery'; +import * as bus from 'hydrooj/src/service/bus'; +import { Logger } from './logger'; +import { NestKeys } from './typeutils'; -type Def = Exclude; +const defaultPath = process.env.CI ? '/tmp/file' : '/data/file/hydro'; +const FileSetting = Schema.intersect([ + Schema.object({ + type: Schema.union([ + Schema.const('file').description('local file provider').required(), + Schema.const('s3').description('s3 provider').required(), + ] as const).description('provider type').default('file'), + endPointForUser: Schema.string().default('/fs/').required(), + endPointForJudge: Schema.string().default('/fs/').required(), + }).description('setting_file'), + Schema.union([ + Schema.object({ + type: Schema.const('file').required(), + path: Schema.string().default(defaultPath).description('Storage path').required(), + secret: Schema.string().description('Download file sign secret').default(nanoid()), + }), + Schema.object({ + type: Schema.const('s3').required(), + endPoint: Schema.string().required(), + accessKey: Schema.string().required().description('access key'), + secretKey: Schema.string().required().description('secret key').role('secret'), + bucket: Schema.string().default('hydro').required(), + region: Schema.string().default('us-east-1').required(), + pathStyle: Schema.boolean().default(true).required(), + }), + ] as const), +] as const).default({ + type: 'file', + path: defaultPath, + endPointForUser: '/fs/', + endPointForJudge: '/fs/', + secret: nanoid(), +}); -function port(examples: number[] = []) { - const res: Def = { - type: 'integer', minimum: 1, maximum: 65535, - }; - if (examples.length) { - res.default = examples[0]; - res.examples = examples; +const builtinSettings = Schema.object({ + file: FileSetting, +}); +export const SystemSettings: Schema[] = [builtinSettings]; +export let configSource = ''; // eslint-disable-line import/no-mutable-exports +export let systemConfig: any = {}; // eslint-disable-line import/no-mutable-exports +const logger = new Logger('settings'); +const update = []; + +export async function loadConfig() { + const config = await global.Hydro.service.db.collection('system').findOne({ _id: 'config' }); + try { + configSource = config?.value || '{}'; + systemConfig = yaml.load(configSource); + logger.info('Successfully loaded config'); + for (const u of update) u(); + } catch (e) { + logger.error('Failed to load config', e.message); } - return res; +} +export async function saveConfig(config: any) { + Schema.intersect(SystemSettings)(config); + const value = yaml.dump(config); + await global.Hydro.service.db.collection('system').updateOne({ _id: 'config' }, { $set: { value } }, { upsert: true }); + bus.broadcast('config/update'); +} +export async function setConfig(key: string, value: any) { + const path = key.split('.'); + const t = path.pop(); + let cursor = systemConfig; + for (const p of path) { + if (!cursor[p]) cursor[p] = {}; + cursor = cursor[p]; + } + cursor[t] = value; + await saveConfig(systemConfig); } -export const Schema = { - string(title: string, defaultValue: string, extra?: T) { - return { - type: 'string' as 'string', - default: defaultValue, - title, - ...extra, - }; - }, - boolean(title: string, defaultValue: boolean, extra?: T) { - return { - type: 'boolean' as 'boolean', - default: defaultValue, - title, - ...extra, - }; - }, - integer(title: string, defaultValue: number, extra?: T) { - return { - type: 'integer' as 'integer', - default: defaultValue, - title, - ...extra, - }; - }, -}; - -const definitions: Record = { - smtp: { - type: 'object', - properties: { - user: Schema.string('SMTP Username', 'noreply@hydro.ac'), - from: Schema.string('Mail From', 'Hydro '), - pass: Schema.string('SMTP Password', '', { writeOnly: true }), - host: Schema.string('SMTP Server Host', 'smtp.hydro.ac', { pattern: '^[a-zA-Z0-9\\-\\.]+$' }), - port: Schema.integer('SMTP Server Port', 25, { examples: [25, 465], minimum: 1, maximum: 65535 }), - secure: Schema.boolean('Use SSL', false), - verify: Schema.boolean('Verify register email', false), - }, - additionalProperties: false, - }, - file: { - type: 'object', - properties: { - endPoint: Schema.string('Storage engine endPoint', 'http://localhost:9000', { - pattern: '^https?://[a-zA-Z0-9\\-\\.]+/?$', - }), - accessKey: Schema.string('Storage engine accessKey', ''), - secretKey: Schema.string('Storage engine secretKey', '', { writeOnly: true }), - bucket: Schema.string('Storage engine bucket', 'hydro'), - region: Schema.string('Storage engine region', 'us-east-1'), - pathStyle: Schema.boolean('pathStyle endpoint', true), - endPointForUser: Schema.string('EndPoint for user', '/fs/'), - endPointForJudge: Schema.string('EndPoint for judge', '/fs/'), - }, - required: ['endPoint', 'accessKey', 'secretKey'], - additionalProperties: false, - }, - server: { - type: 'object', - properties: { - name: Schema.string('Server Name', 'Hydro'), - url: Schema.string('Self URL', 'https://hydro.ac/', { pattern: '/$' }), - cdn: Schema.string('CDN prefix', '/', { - pattern: '/$', examples: ['/', 'https://cdn.hydro.ac/'], - }), - port: port([8888, 80, 443]), - xff: Schema.string('IP Header', '', { examples: ['x-forwarded-for', 'x-real-ip'], pattern: '^[a-z-]+$' }), - xhost: Schema.string('Host Header', '', { examples: ['x-real-host'], pattern: '^[a-z-]+$' }), - language: { type: 'string', enum: Object.keys(global.Hydro.locales) }, - upload: Schema.string('Upload size limit', '256m', { pattern: '^[0-9]+[mkg]b?$' }), - login: Schema.boolean('Enable builtin login', true), - message: Schema.boolean('Enable message', true), - blog: Schema.boolean('Enable blog', true), - checkUpdate: Schema.boolean('Daily update check', true), - }, - required: ['url', 'port', 'language'], - }, - limit: { - type: 'object', - properties: { - problem_files_max: { type: 'integer', minimum: 0 }, - }, - }, - session: { - type: 'object', - properties: { - keys: { - type: 'array', items: { type: 'string' }, default: [String.random(32)], writeOnly: true, +export function requestConfig(s: Schema): { + config: ReturnType>, + setConfig: (key: NestKeys>>, value: any) => Promise, +} { + SystemSettings.push(s); + let curValue = s(systemConfig); + update.push(() => { + try { + curValue = s(systemConfig); + } catch (e) { + logger.warn('Cannot read config: ', e.message); + curValue = null; + } + }); + return { + config: new Proxy(curValue as any, { + get(self, key: string) { + return curValue?.[key]; + }, + set(self) { + throw new Error(`Not allowed to set setting ${self.p.join('.')}`); }, - secure: { type: 'boolean', default: false }, - saved_expire_seconds: { type: 'integer', minimum: 300, default: 3600 * 24 * 30 }, - unsaved_expire_seconds: { type: 'integer', minimum: 60, default: 3600 * 3 }, - }, - }, - user: { - type: 'object', - properties: { - quota: { type: 'integer', minimum: 0 }, - }, - }, -}; + }), + setConfig, + }; +} -export const schema: Def = { - type: 'object', - definitions, - properties: { - smtp: definitions.smtp, - file: definitions.file, - server: definitions.server, - limit: definitions.limit, - session: definitions.session, - user: definitions.user, - }, - additionalProperties: true, -}; +const builtin = requestConfig(builtinSettings); +export const builtinConfig = builtin.config; +export const setBuiltinConfig = builtin.setConfig; -export function addDef(key: string, def: Def) { - definitions[key] = def; - schema.properties[key] = definitions[key]; -} +bus.on('config/update', loadConfig); diff --git a/packages/hydrooj/src/upgrade.ts b/packages/hydrooj/src/upgrade.ts index 45b16ed9..4ad3ff86 100644 --- a/packages/hydrooj/src/upgrade.ts +++ b/packages/hydrooj/src/upgrade.ts @@ -29,6 +29,7 @@ import { } from './pipelineUtils'; import db from './service/db'; import storage from './service/storage'; +import { setBuiltinConfig } from './settings'; import { streamToBuffer } from './utils'; import welcome from './welcome'; @@ -94,7 +95,7 @@ const scripts: UpgradeScript[] = [ try { const [file, current] = await Promise.all([ streamToBuffer(gridfs.openDownloadStream(pdoc.data)), - storage.list(`problem/${pdoc.domainId}/${pdoc.docId}/testdata/`), + storage.list(`problem/${pdoc.domainId}/${pdoc.docId}/testdata/`) as any, ]); const zip = new AdmZip(file); const entries = zip.getEntries(); @@ -146,8 +147,8 @@ const scripts: UpgradeScript[] = [ await iterateAllProblem(['docId', 'domainId', 'config'], async (pdoc) => { logger.info('%s/%s', pdoc.domainId, pdoc.docId); const [data, additional_file] = await Promise.all([ - storage.list(`problem/${pdoc.domainId}/${pdoc.docId}/testdata/`), - storage.list(`problem/${pdoc.domainId}/${pdoc.docId}/additional_file/`), + storage.list(`problem/${pdoc.domainId}/${pdoc.docId}/testdata/`) as any, + storage.list(`problem/${pdoc.domainId}/${pdoc.docId}/additional_file/`) as any, ]); await problem.edit( pdoc.domainId, pdoc.docId, @@ -802,6 +803,33 @@ const scripts: UpgradeScript[] = [ }); return true; }, + async function _66_67() { + const [ + endPoint, accessKey, secretKey, bucket, region, + pathStyle, endPointForUser, endPointForJudge, + ] = system.getMany([ + 'file.endPoint', 'file.accessKey', 'file.secretKey', 'file.bucket', 'file.region', + 'file.pathStyle', 'file.endPointForUser', 'file.endPointForJudge', + ]); + if ((endPoint && accessKey) || process.env.MINIO_ACCESS_KEY) { + await setBuiltinConfig('file', { + type: 's3', + endPoint: process.env.MINIO_ACCESS_KEY ? 'http://127.0.0.1:9000/' : endPoint, + accessKey: process.env.MINIO_ACCESS_KEY || accessKey, + secretKey: process.env.MINIO_SECRET_KEY || secretKey, + bucket, + region, + pathStyle, + endPointForUser, + endPointForJudge, + }); + setTimeout(() => { + logger.info('Upgrade done. please restart the server.'); + process.exit(0); + }, 1000); + } + return true; + }, ]; export default scripts; diff --git a/packages/ui-default/build/config/webpack.ts b/packages/ui-default/build/config/webpack.ts index 8aa6edc3..98d83b58 100644 --- a/packages/ui-default/build/config/webpack.ts +++ b/packages/ui-default/build/config/webpack.ts @@ -96,7 +96,7 @@ export default function (env: { production?: boolean, measure?: boolean } = {}) chunkFilename: '[name].[chunkhash].chunk.js', }, resolve: { - extensions: ['.js', '.jsx', '.ts', '.tsx'], + extensions: ['.js', '.jsx', '.ts', '.tsx', '.cjs'], alias: { vj: root(), }, @@ -148,8 +148,9 @@ export default function (env: { production?: boolean, measure?: boolean } = {}) }, }, { - test: /\.[jt]sx?$/, + test: /\.[mc]?[jt]sx?$/, exclude: /@types\//, + type: 'javascript/auto', use: [esbuildLoader()], }, { diff --git a/packages/ui-default/components/discussion/comments.page.tsx b/packages/ui-default/components/discussion/comments.page.tsx index cb203150..41a72373 100644 --- a/packages/ui-default/components/discussion/comments.page.tsx +++ b/packages/ui-default/components/discussion/comments.page.tsx @@ -107,7 +107,6 @@ async function onCommentClickReplyComment(ev, options: any = {}) { } async function onCommentClickReplyReply(ev) { - console.log(ev); const $evTarget = $(ev.currentTarget); const $mediaBody = $evTarget.closest('.media__body'); const uid = $mediaBody diff --git a/packages/ui-default/components/editor/cmeditor.page.ts b/packages/ui-default/components/editor/cmeditor.page.ts index 74e50ce3..37114ffc 100644 --- a/packages/ui-default/components/editor/cmeditor.page.ts +++ b/packages/ui-default/components/editor/cmeditor.page.ts @@ -7,7 +7,6 @@ function runSubstitute($container: JQuery) { $container.find(`textarea[data-${language}]`).get().forEach((element) => { const config: any = { language }; if ($(element).data('model')) config.model = $(element).data('model'); - if ($(element).data('hide')) config.hide = $(element).data('hide'); CmEditor.getOrConstruct($(element), config); }); } diff --git a/packages/ui-default/components/monaco/languages/yaml.ts b/packages/ui-default/components/monaco/languages/yaml.ts index df6e9a88..50187a91 100644 --- a/packages/ui-default/components/monaco/languages/yaml.ts +++ b/packages/ui-default/components/monaco/languages/yaml.ts @@ -98,7 +98,7 @@ setDiagnosticsOptions({ schema: problemConfigSchema as any, }, { - uri: `${UiContext.cdn_prefix}manage/setting/schema.json`, + uri: '/manage/config/schema.json', fileMatch: ['hydro://system/setting.yaml'], }, ], diff --git a/packages/ui-default/handler.ts b/packages/ui-default/handler.ts index fa59d46e..9cd9ca53 100644 --- a/packages/ui-default/handler.ts +++ b/packages/ui-default/handler.ts @@ -11,9 +11,12 @@ import user from 'hydrooj/src/model/user'; import * as bus from 'hydrooj/src/service/bus'; import { UiContextBase } from 'hydrooj/src/service/layers/base'; import { Handler, Route } from 'hydrooj/src/service/server'; +import { SystemSettings } from 'hydrooj/src/settings'; import { ObjectID } from 'mongodb'; import { tmpdir } from 'os'; import { join } from 'path'; +import Schema from 'schemastery'; +import convert from 'schemastery-jsonschema'; import markdown from './backendlib/markdown'; declare module 'hydrooj/src/interface' { @@ -157,6 +160,13 @@ class LanguageHandler extends ResourceHandler { } } +class SystemConfigSchemaHandler extends Handler { + async get() { + const schema = convert(Schema.intersect(SystemSettings) as any, true); + this.response.body = schema; + } +} + class RichMediaHandler extends Handler { async renderUser(domainId, payload) { let d = payload.domainId || domainId; @@ -213,6 +223,7 @@ global.Hydro.handler.ui = async () => { Route('set_theme', '/set_theme/:theme', SetThemeHandler); Route('constant', '/constant/:version', UiConstantsHandler); Route('markdown', '/markdown', MarkdownHandler); + Route('config_schema', '/manage/config/schema.json', SystemConfigSchemaHandler, PRIV.PRIV_EDIT_SYSTEM); Route('lang', '/l/:lang', LanguageHandler); Route('media', '/media', RichMediaHandler); }; diff --git a/packages/ui-default/package.json b/packages/ui-default/package.json index ac0ecac8..851c8a21 100644 --- a/packages/ui-default/package.json +++ b/packages/ui-default/package.json @@ -1,6 +1,6 @@ { "name": "@hydrooj/ui-default", - "version": "4.39.8", + "version": "4.39.9", "author": "undefined ", "license": "AGPL-3.0", "main": "hydro.js", @@ -129,6 +129,7 @@ "mongodb": "^3.7.3", "nunjucks": "^3.2.3", "p-queue": "^7.3.0", + "schemastery-jsonschema": "^1.0.3", "streamsaver": "^2.0.6", "xss": "^1.0.14" } diff --git a/packages/ui-default/pages/setting.page.tsx b/packages/ui-default/pages/setting.page.tsx new file mode 100644 index 00000000..b53d1ac6 --- /dev/null +++ b/packages/ui-default/pages/setting.page.tsx @@ -0,0 +1,29 @@ +import yaml from 'js-yaml'; +import Schema from 'schemastery'; +import Notification from 'vj/components/notification'; +import { NamedPage } from 'vj/misc/Page'; +import request from 'vj/utils/request'; + +const page = new NamedPage('manage_config', async () => { + const schema = new Schema(UiContext.schema); + setInterval(() => { + try { + const v = yaml.load($('#config').val().toString()); + schema(v); + $('#info').text(''); + } catch (e) { + console.debug(e); + $('#info').text(e.message); + } + }, 1000); + $('#submit').on('click', () => { + const value = $('#config').val(); + request.post('', { value }).then(() => { + Notification.success('保存成功'); + }).catch((e) => { + Notification.error('保存失败:', e.message); + }); + }); +}); + +export default page; diff --git a/packages/ui-default/templates/manage_base.html b/packages/ui-default/templates/manage_base.html index 2751e3c6..a50fa48d 100644 --- a/packages/ui-default/templates/manage_base.html +++ b/packages/ui-default/templates/manage_base.html @@ -16,6 +16,7 @@ {{ sidemenu.render_item(null, 'manage_script') }} {{ sidemenu.render_item(null, 'manage_user_import') }} {{ sidemenu.render_item(null, 'manage_setting') }} + {{ sidemenu.render_item(null, 'manage_config') }} diff --git a/packages/ui-default/templates/manage_config.html b/packages/ui-default/templates/manage_config.html new file mode 100644 index 00000000..9041670d --- /dev/null +++ b/packages/ui-default/templates/manage_config.html @@ -0,0 +1,24 @@ +{% extends "manage_base.html" %} +{% block manage_content %} +{{ set(UiContext, 'schema', schema) }} +
+
+
+
+ +

+
+
+
+
+ +
+
+
+
+{% endblock %} diff --git a/packages/ui-default/templates/partials/setting.html b/packages/ui-default/templates/partials/setting.html index a403d62c..6c76bc50 100644 --- a/packages/ui-default/templates/partials/setting.html +++ b/packages/ui-default/templates/partials/setting.html @@ -8,7 +8,6 @@ {% if not setting.flag|bitand(model.setting.FLAG_HIDDEN) %} {% set secret = setting.flag|bitand(model.setting.FLAG_SECRET) != 0 %} {% if setting.type == 'text' or setting.type == 'password' or setting.type == 'number' or setting.type == 'float' %} - {% set isFileConfig = (setting.name === 'file.accessKey' or setting.name === 'file.secretKey') and process.env.MINIO_ACCESS_KEY %} {{ form.form_text({ type:setting.type, label:setting.name, @@ -16,7 +15,7 @@ name:setting.key, value:'' if (secret or isFileConfig) else (current[setting.key]|default(setting.value)), disabled:setting.flag|bitand(2), - placeholder:'Please edit in ~/.hydro/env instead' if isFileConfig else (_('(Not changed)') if secret else '') + placeholder:_('(Not changed)') if secret else '' }) }} {% elif setting.type == 'select' %} {{ form.form_select({ diff --git a/packages/vjudge/src/providers/codeforces.ts b/packages/vjudge/src/providers/codeforces.ts index 7b15d2e5..064fcd84 100644 --- a/packages/vjudge/src/providers/codeforces.ts +++ b/packages/vjudge/src/providers/codeforces.ts @@ -214,7 +214,6 @@ export default class CodeforcesProvider implements IBasicProvider { await page.waitForRequest((req) => { if (req.method() !== 'POST') return false; if (!req.url().endsWith('/enter')) return false; - console.log(req); return true; }, { timeout: 24 * 3600 * 1000 }); await page.waitForTimeout(10 * 1000);