/* eslint-disable no-await-in-loop */ import mariadb from 'mariadb'; import xml2js from 'xml2js'; import { AdmZip, ContestModel, DomainModel, fs, MessageModel, moment, noop, NotFoundError, ObjectId, postJudge, ProblemConfigFile, ProblemModel, RecordDoc, RecordModel, STATUS, SubtaskType, SystemModel, Time, UserModel, ValidationError, yaml, } from 'hydrooj'; const statusMap = { Accepted: STATUS.STATUS_ACCEPTED, 'Compile Error': STATUS.STATUS_COMPILE_ERROR, 'Wrong Answer': STATUS.STATUS_WRONG_ANSWER, 'Runtime Error': STATUS.STATUS_RUNTIME_ERROR, 'Memory Limit Exceeded': STATUS.STATUS_MEMORY_LIMIT_EXCEEDED, 'Time Limit Exceeded': STATUS.STATUS_TIME_LIMIT_EXCEEDED, 'Output Limit Exceeded': STATUS.STATUS_OUTPUT_LIMIT_EXCEEDED, 'Dangerous Syscalls': STATUS.STATUS_RUNTIME_ERROR, 'Judgement Failed': STATUS.STATUS_SYSTEM_ERROR, 'No Comment': STATUS.STATUS_SYSTEM_ERROR, Skippped: STATUS.STATUS_CANCELED, Judging: STATUS.STATUS_JUDGING, }; const sexMap = { U: 3, M: 1, F: 2, }; const langMap = { C: 'c', 'C++': 'cc.cc98', 'C++11': 'cc.cc11', Java8: 'java', Java11: 'java', Pascal: 'pas', Python2: 'py.py2', Python3: 'py.py3', }; function handleMailLower(mail: string) { let data = mail.trim().toLowerCase(); if (data.endsWith('@googlemail.com')) data = data.replace('@googlemail.com', '@gmail.com'); if (data.endsWith('@gmail.com')) { const [prev] = data.split('@'); data = `${prev.replace(/[.+]/g, '')}@gmail.com`; } return data; } export async function run({ host = '172.17.0.2', port = 3306, name = 'app_uoj233', username, password, domainId, dataDir, rerun = true, randomMail = false, }, report: Function) { const src = await mariadb.createConnection({ host, port, user: username, password, database: name, }); const query = (q: string) => new Promise((res, rej) => { src.query(q).then((r) => res(r)).catch((e) => rej(e)); }); const target = await DomainModel.get(domainId); if (!target) throw new NotFoundError(domainId); report({ message: 'Connected to database' }); /* CREATE TABLE `user_info` ( `usergroup` char(1) NOT NULL DEFAULT 'U', `username` varchar(20) NOT NULL, `email` varchar(50) NOT NULL, `password` char(32) NOT NULL, `svn_password` char(10) NOT NULL, `rating` int(11) NOT NULL DEFAULT '1500', `qq` bigint(20) NOT NULL, `sex` char(1) NOT NULL DEFAULT 'U', `ac_num` int(11) NOT NULL, `register_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, `remote_addr` varchar(50) NOT NULL, `http_x_forwarded_for` varchar(50) NOT NULL, `remember_token` char(60) NOT NULL, `motto` varchar(200) NOT NULL, `cellphone` varchar(15) NOT NULL, PRIMARY KEY (`username`), KEY `rating` (`rating`,`username`), KEY `ac_num` (`ac_num`,`username`) ) ENGINE=MyISAM DEFAULT CHARSET=utf8mb4; */ const configFile = await fs.readFile(`${dataDir}/opt/uoj/web/app/.config.php`, 'utf8'); const salt = configFile.match(/'client_salt' => '(.+?)'/)[1]; const uidMap: Record = {}; const udocs = await query('SELECT * FROM `user_info`'); const priv = await SystemModel.get('default.priv'); report({ message: udocs.map((u) => u.username.toLowerCase()) }); const precheck = await UserModel.getMulti({ unameLower: { $in: udocs.map((u) => u.username.toLowerCase()) } }).toArray(); if (precheck.length) throw new Error(`Conflict username: ${precheck.map((u) => u.unameLower).join(', ')}`); for (const udoc of udocs) { if (randomMail) delete udoc.email; let current = await UserModel.getByEmail(domainId, udoc.email || `${udoc.username}@universaloj.local`); current ||= await UserModel.getByUname(domainId, udoc.username); if (current) { report({ message: `duplicate user with email ${udoc.email}: ${current.uname},${udoc.username}` }); uidMap[udoc.username] = current._id; } else { const [u] = await UserModel.coll.find({}).sort({ _id: -1 }).limit(1).toArray(); const uid = Math.max((u?._id || 0) + 1, 2); await UserModel.coll.insertOne({ _id: uid, uname: udoc.username, unameLower: udoc.username.toLowerCase(), mail: udoc.email || `${udoc.username}@universaloj.local`, mailLower: handleMailLower(udoc.email || `${udoc.username}@universaloj.local`), regat: new Date(udoc.register_time), hash: udoc.password, salt, hashType: 'uoj', ip: [udoc.http_x_forwarded_for || udoc.remote_addr || '127.0.0.1'], loginat: new Date(), loginip: '127.0.0.1', priv, avatar: `gravatar:${udoc.email || `${udoc.username}@universaloj.local`}`, bio: udoc.motto || '', gender: sexMap[udoc.sex] || 3, qq: udoc.qq.toString() || null, phone: udoc.cellphone || null, }); if (udoc.usergroup === 'S') await UserModel.setSuperAdmin(uid); uidMap[udoc.username] = uid; await DomainModel.setUserInDomain(domainId, uid, { displayName: udoc.nickname || '', nAccept: udoc.ac_num, }); } } report({ message: 'user finished' }); /* CREATE TABLE `user_msg` ( `id` int(10) unsigned NOT NULL AUTO_INCREMENT, `sender` varchar(20) NOT NULL, `receiver` varchar(20) NOT NULL, `message` varchar(5000) NOT NULL, `send_time` datetime NOT NULL, `read_time` datetime DEFAULT NULL, PRIMARY KEY (`id`) ) ENGINE=MyISAM AUTO_INCREMENT=8299 DEFAULT CHARSET=utf8mb4; CREATE TABLE `user_system_msg` ( `id` int(11) NOT NULL AUTO_INCREMENT, `title` varchar(100) COLLATE utf8mb4_unicode_ci NOT NULL, `content` varchar(300) COLLATE utf8mb4_unicode_ci NOT NULL, `receiver` varchar(20) COLLATE utf8mb4_unicode_ci NOT NULL, `send_time` datetime NOT NULL, `read_time` datetime DEFAULT NULL, PRIMARY KEY (`id`) ) ENGINE=InnoDB AUTO_INCREMENT=16814 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; */ const messages = await query('SELECT * FROM `user_msg`'); await Promise.all(messages.map((msg) => MessageModel.coll.insertOne({ _id: Time.getObjectID(new Date(msg.send_time), false), from: uidMap[msg.sender], to: uidMap[msg.receiver], content: msg.message, flag: 0, }))); const systemMessages = await query('SELECT * FROM `user_system_msg`'); await Promise.all(systemMessages.map((msg) => MessageModel.coll.insertOne({ _id: Time.getObjectID(new Date(msg.send_time), false), from: 1, to: uidMap[msg.receiver], content: msg.content, flag: 0, }))); report({ message: 'message finished' }); /* CREATE TABLE `problems` ( `id` int(10) unsigned NOT NULL AUTO_INCREMENT, `title` text NOT NULL, `is_hidden` tinyint(1) NOT NULL DEFAULT '0', `submission_requirement` text, `hackable` tinyint(1) NOT NULL DEFAULT '0', `extra_config` varchar(500) NOT NULL DEFAULT '{"view_content_type":"ALL_AFTER_AC","view_details_type":"SELF","view_all_details_type":"SELF"}', `zan` int(11) NOT NULL, `ac_num` int(11) NOT NULL DEFAULT '0', `submit_num` int(11) NOT NULL DEFAULT '0', `difficulty` int(11) NOT NULL DEFAULT '9999', PRIMARY KEY (`id`) ) ENGINE=MyISAM AUTO_INCREMENT=1545 DEFAULT CHARSET=utf8mb4; CREATE TABLE `problems_contents` ( `id` int(11) NOT NULL, `statement` mediumtext NOT NULL, `statement_md` mediumtext NOT NULL, PRIMARY KEY (`id`) ) ENGINE=MyISAM DEFAULT CHARSET=utf8mb4; CREATE TABLE `problems_permissions` ( `username` varchar(20) NOT NULL, `problem_id` int(11) NOT NULL, PRIMARY KEY (`username`,`problem_id`) ) ENGINE=MyISAM DEFAULT CHARSET=utf8mb4; CREATE TABLE `problems_tags` ( `id` int(11) NOT NULL AUTO_INCREMENT, `problem_id` int(11) NOT NULL, `tag` varchar(30) NOT NULL, PRIMARY KEY (`id`), KEY `problem_id` (`problem_id`), KEY `tag` (`tag`) ) ENGINE=MyISAM AUTO_INCREMENT=4 DEFAULT CHARSET=utf8mb4; */ const pidMap: Record = {}; const [{ 'count(*)': pcount }] = await query('SELECT count(*) FROM `problems`'); const step = 50; const pageCount = Math.ceil(Number(pcount) / step); for (let pageId = 0; pageId < pageCount; pageId++) { const pdocs = await query(`SELECT * FROM \`problems\` LIMIT ${pageId * step}, ${step}`); for (const pdoc of pdocs) { if (rerun) { const opdoc = await ProblemModel.get(domainId, `P${pdoc.id}`); if (opdoc) pidMap[pdoc.id] = opdoc.docId; } if (!pidMap[pdoc.id]) { const content = await query(`SELECT * FROM \`problems_contents\` WHERE \`id\` = ${pdoc.id}`); const pid = await ProblemModel.add(domainId, `P${pdoc.id}`, pdoc.title, content[0].statement_md || '', 1); pidMap[pdoc.id] = pid; } const [permissions, tags] = await Promise.all([ query(`SELECT * FROM \`problems_permissions\` WHERE \`problem_id\` = ${pdoc.id}`), query(`SELECT * FROM \`problems_tags\` WHERE \`problem_id\` = ${pdoc.id}`), ]); const maintainer = permissions.map((p) => uidMap[p.username]).slice(1); await ProblemModel.edit(domainId, pidMap[pdoc.id], { nAccept: pdoc.ac_num || 0, nSubmit: pdoc.submit_num || 0, hidden: !!pdoc.is_hidden, tag: tags.map((t) => t.tag), owner: uidMap[permissions[0]?.username] || 1, maintainer, }); } console.log({ message: `Synced ${pageId * step} / ${pcount} problems` }); } report({ message: 'problem finished' }); /* CREATE TABLE `contests` ( `id` int(11) NOT NULL AUTO_INCREMENT, `name` varchar(50) NOT NULL, `start_time` datetime NOT NULL, `last_min` int(11) NOT NULL, `player_num` int(11) NOT NULL, `status` varchar(50) NOT NULL, `extra_config` varchar(200) NOT NULL, `zan` int(11) NOT NULL, PRIMARY KEY (`id`) ) ENGINE=MyISAM DEFAULT CHARSET=utf8mb4; CREATE TABLE `contests_asks` ( `id` int(11) NOT NULL AUTO_INCREMENT, `contest_id` int(11) NOT NULL, `username` varchar(20) NOT NULL, `question` text NOT NULL, `answer` text NOT NULL, `post_time` datetime NOT NULL, `reply_time` datetime NOT NULL, `is_hidden` tinyint(1) DEFAULT '0', PRIMARY KEY (`id`) ) ENGINE=MyISAM DEFAULT CHARSET=utf8mb4; CREATE TABLE `contests_notice` ( `contest_id` int(11) NOT NULL, `title` varchar(30) NOT NULL, `content` varchar(500) NOT NULL, `time` datetime NOT NULL, KEY `contest_id` (`contest_id`) ) ENGINE=MyISAM DEFAULT CHARSET=utf8mb4; CREATE TABLE `contests_permissions` ( `username` varchar(20) NOT NULL, `contest_id` int(11) NOT NULL, PRIMARY KEY (`username`,`contest_id`) ) ENGINE=MyISAM DEFAULT CHARSET=utf8mb4; CREATE TABLE `contests_problems` ( `problem_id` int(11) NOT NULL, `contest_id` int(11) NOT NULL, PRIMARY KEY (`problem_id`,`contest_id`) ) ENGINE=MyISAM DEFAULT CHARSET=utf8mb4; CREATE TABLE `contests_registrants` ( `username` varchar(20) NOT NULL, `user_rating` int(11) NOT NULL, `contest_id` int(11) NOT NULL, `has_participated` tinyint(1) NOT NULL, `rank` int(11) NOT NULL, PRIMARY KEY (`contest_id`,`username`) ) ENGINE=MyISAM DEFAULT CHARSET=utf8mb4; CREATE TABLE `contests_submissions` ( `contest_id` int(11) NOT NULL, `submitter` varchar(20) NOT NULL, `problem_id` int(11) NOT NULL, `submission_id` int(11) NOT NULL, `score` int(11) NOT NULL, `penalty` int(11) NOT NULL, PRIMARY KEY (`contest_id`,`submitter`,`problem_id`) ) ENGINE=MyISAM DEFAULT CHARSET=utf8mb4; */ const tidMap: Record = {}; const tdocs = await query('SELECT * FROM `contests`'); for (const tdoc of tdocs) { const [permissions, problems, notices] = await Promise.all([ query(`SELECT * FROM \`contests_permissions\` WHERE \`contest_id\` = ${tdoc.id}`), query(`SELECT * FROM \`contests_problems\` WHERE \`contest_id\` = ${tdoc.id} ORDER BY \`problem_rank\` ASC`), // query(`SELECT * FROM \`contests_asks\` WHERE \`contest_id\` = ${tdoc.id}`), query(`SELECT * FROM \`contests_notice\` WHERE \`contest_id\` = ${tdoc.id}`), ]); let content = ''; for (const notice of notices) { content += `## Notice: ${notice.title}\n${notice.content}\n${moment(notice.start_time).format('YYYY-MM-DD HH:mm:ss')}\n`; } const pids = problems.map((p) => pidMap[p.problem_id]); const maintainer = permissions.map((p) => uidMap[p.username]).slice(1); const info = JSON.parse(tdoc.extra_config) || {}; const startAt = moment(tdoc.start_time); const endAt = startAt.clone().add(tdoc.last_min, 'minutes'); const tid = await ContestModel.add( domainId, tdoc.name, content, uidMap[permissions[0]?.username] || 1, info.contest_type?.toLowerCase() || 'oi', startAt.toDate(), endAt.toDate(), pids, !Object.keys(info).includes('unrated'), { maintainer }, ); tidMap[tdoc.id] = tid.toHexString(); } report({ message: 'contest finished' }); /* `id` int(10) unsigned NOT NULL AUTO_INCREMENT, `problem_id` int(10) unsigned NOT NULL, `contest_id` int(10) unsigned DEFAULT NULL, `submit_time` datetime NOT NULL, `submitter` varchar(20) NOT NULL, `content` text NOT NULL, `language` varchar(15) NOT NULL, `tot_size` int(11) NOT NULL, `judge_time` datetime DEFAULT NULL, `result` blob NOT NULL, `status` varchar(20) NOT NULL, `result_error` varchar(20) DEFAULT NULL, `score` int(11) DEFAULT NULL, `used_time` int(11) NOT NULL DEFAULT '0', `used_memory` int(11) NOT NULL DEFAULT '0', `is_hidden` tinyint(1) NOT NULL, `status_details` varchar(100) NOT NULL, `contest_penalty` int(11) DEFAULT NULL, */ if (dataDir.endsWith('/')) dataDir = dataDir.slice(0, -1); const [{ 'count(*)': rcount }] = await query('SELECT count(*) FROM `submissions`'); const rpageCount = Math.ceil(Number(rcount) / step); for (let pageId = 0; pageId < rpageCount; pageId++) { const rdocs = await query(`SELECT * FROM \`submissions\` LIMIT ${pageId * step}, ${step}`); for (const rdoc of rdocs) { const data: RecordDoc = { status: statusMap[rdoc.result_error] || STATUS.STATUS_WAITING, _id: Time.getObjectID(new Date(rdoc.submit_time), false), uid: uidMap[rdoc.submitter] || 1, code: '', lang: langMap[rdoc.language] || '', pid: pidMap[rdoc.problem_id] || 0, domainId, score: rdoc.score || 0, time: rdoc.used_time || 0, memory: rdoc.used_memory || 0, judgeTexts: [], compilerTexts: [], testCases: [], judgeAt: new Date(rdoc.judge_time), rejudged: false, judger: 1, }; const content = JSON.parse(rdoc.content); try { let zip: AdmZip; try { zip = new AdmZip(await fs.readFile(`${dataDir}/opt/uoj/web/app/storage${content.file_name}`)); } catch (e) { throw new ValidationError('zip', null, e.message); } data.code = zip.getEntries().find((i) => i.name.endsWith('answer.code')).getData().toString(); } catch { /* ignore no code */ } const result = JSON.parse(Buffer.from(rdoc.result, 'base64').toString('utf8')); if (result.error) { if (data.status === STATUS.STATUS_COMPILE_ERROR) data.compilerTexts.push(result.error); else data.judgeTexts.push(result.error); } else { // TODO final_result if (!result.details) continue; try { const details = await xml2js.parseStringPromise(result.details); if (details.tests.subtask) { details.tests.subtask.forEach((subtask) => { if (!subtask.test) { data.testCases.push({ subtaskId: subtask.$.num, id: 1, score: 0, time: 0, memory: 0, message: 'Skipped', status: STATUS.STATUS_CANCELED, }); return; } data.testCases.push(...subtask.test.map((curCase, caseIndex) => ({ subtaskId: subtask.$.num, id: caseIndex + 1, score: curCase.$.score, time: curCase.$.time === '-1' ? 0 : curCase.time, memory: curCase.$.memory === '-1' ? 0 : curCase.memory, message: curCase.res[0] || '', status: statusMap[curCase.$.info] || STATUS.STATUS_WAITING, }))); }); } else if (details.tests.test) { data.testCases.push(...details.tests.test.map((curCase) => ({ subtaskId: 1, id: curCase.$.num, score: curCase.$.score, time: curCase.$.time === '-1' ? 0 : curCase.time, memory: curCase.$.memory === '-1' ? 0 : curCase.memory, message: curCase.res[0] || '', status: statusMap[curCase.$.info] || STATUS.STATUS_WAITING, }))); } data.status = Math.max(...data.testCases.map((x) => x.status)); } catch (e) { console.log(rdoc.id, result); } } if (rdoc.contest_id) { data.contest = new ObjectId(tidMap[rdoc.contest_id]); await ContestModel.attend(domainId, data.contest, uidMap[rdoc.submitter]).catch(noop); } await RecordModel.coll.insertOne(data); await postJudge(data).catch((err) => report({ message: err.message })); } console.log({ message: `Synced ${pageId * step} / ${rcount} records` }); } report({ message: 'record finished' }); // TODO: blog src.end(); const files = await fs.readdir(`${dataDir}/var/uoj_data/`, { withFileTypes: true }); for (const file of files) { if (!file.isDirectory()) continue; const datas = await fs.readdir(`${dataDir}/var/uoj_data/${file.name}`, { withFileTypes: true }); const pdoc = await ProblemModel.get(domainId, `P${file.name}`, undefined, true); if (!pdoc) continue; report({ message: `Syncing testdata for ${file.name}` }); const filenames = datas.map((i) => i.name); for (const data of datas) { if (data.isDirectory()) continue; await ProblemModel.addTestdata(domainId, pdoc.docId, data.name, `${dataDir}/var/uoj_data/${file.name}/${data.name}`); if (data.name === 'problem.conf') { const confInfo: any = { subtask_end: {}, subtask_score: {}, point: {}, }; const conf = await fs.readFile(`${dataDir}/var/uoj_data/${file.name}/${data.name}`, 'utf8'); const config: ProblemConfigFile = { subtasks: [], }; const lines = conf.replace(/\r/g, '').split('\n').map((i) => i.trim()).filter((i) => i); for (const line of lines) { const [key, value] = line.split(' '); if (key === 'use_builtin_checker') { config.checker = value; config.checker_type = 'testlib'; } else if (key === 'time_limit') { config.time = `${value}s`; } else if (key === 'memory_limit') { config.memory = `${value}mb`; } else if (key.startsWith('subtask_end_')) { confInfo.subtask_end[+key.slice(12)] = +value; } else if (key.startsWith('subtask_score_')) { confInfo.subtask_score[+key.slice(14)] = +value; } else if (key.startsWith('point_score_')) { confInfo.point[+key.slice(12)] = +value; } else confInfo[key] = value; } if (!config.checker && filenames.includes('chk.cpp')) { config.checker_type = 'testlib'; config.checker = 'chk.cpp'; } if (filenames.includes('val.cpp')) config.validator = 'val.cpp'; if (confInfo.n_tests && Object.keys(confInfo.point).length === +confInfo.n_tests) { config.subtasks.push(...Object.keys(confInfo.point).map((i) => ({ id: +i, score: confInfo.point[i], cases: [{ input: `${confInfo.input_pre}${i}.${confInfo.input_suf}`, output: `${confInfo.output_pre}${i}.${confInfo.output_suf}`, }], }))); } if (confInfo.n_subtasks && Object.keys(confInfo.subtask_end).length === +confInfo.n_subtasks && Object.keys(confInfo.subtask_score).length === +confInfo.n_subtasks) { config.subtasks.push(...[...new Array(+confInfo.n_subtasks)].map((v, i) => i + 1).map((i) => ({ id: +i, score: confInfo.subtask_score[i], cases: [...new Array(confInfo.subtask_end[i] - (confInfo.subtask_end[i - 1] || 0))].map((v, j) => ({ input: `${confInfo.input_pre}${j + (confInfo.subtask_end[i - 1] || 0) + 1}.${confInfo.input_suf}`, output: `${confInfo.output_pre}${j + (confInfo.subtask_end[i - 1] || 0) + 1}.${confInfo.output_suf}`, })), }))); } if (+confInfo.n_ex_tests) { if (!config.subtasks.length) { config.subtasks.push({ id: 1, score: 97, type: 'sum' as SubtaskType, cases: [...new Array(+confInfo.n_tests)].map((v, i) => i + 1).map((i) => ({ input: `${confInfo.input_pre}${i}.${confInfo.input_suf}`, output: `${confInfo.output_pre}${i}.${confInfo.output_suf}`, })), }); } config.subtasks.push({ id: Math.max(...config.subtasks.map((i) => i.id)) + 1, score: 3, cases: [...new Array(+confInfo.n_ex_tests)].map((v, i) => i + 1).map((i) => ({ input: `ex_${confInfo.input_pre}${i}.${confInfo.input_suf}`, output: `ex_${confInfo.output_pre}${i}.${confInfo.output_suf}`, })), }); } await ProblemModel.addTestdata(domainId, pdoc.docId, 'config.yaml', Buffer.from(yaml.dump(config))); } } } return true; }