From d4e5b91229152758a16262cb7be6091e7596b440 Mon Sep 17 00:00:00 2001 From: Jordan Eldredge Date: Mon, 14 Oct 2019 19:00:39 -0400 Subject: [PATCH] Move all s3 APIs into the database interface --- .eslintignore | 1 + experiments/skin-database/cli.js | 14 ++- experiments/skin-database/data/skins.js | 98 ++++++++++++++- .../discord-bot/commands/approve.js | 2 +- .../discord-bot/commands/reject.js | 2 +- .../discord-bot/commands/review.js | 13 +- .../discord-bot/commands/stats.js | 2 +- experiments/skin-database/discord-bot/s3.js | 112 ------------------ .../skin-database/discord-bot/utils.js | 4 +- experiments/skin-database/s3.js | 20 +++- 10 files changed, 135 insertions(+), 133 deletions(-) delete mode 100644 experiments/skin-database/discord-bot/s3.js diff --git a/.eslintignore b/.eslintignore index cdffcefe..9f1f891a 100644 --- a/.eslintignore +++ b/.eslintignore @@ -3,3 +3,4 @@ built/ coverage/ **/node_modules/ examples/webpack/bundle.js +experiments/tweetBot/env/ \ No newline at end of file diff --git a/experiments/skin-database/cli.js b/experiments/skin-database/cli.js index 24305109..437a6ee1 100755 --- a/experiments/skin-database/cli.js +++ b/experiments/skin-database/cli.js @@ -7,7 +7,6 @@ const logger = require("./logger"); const DiscordWinstonTransport = require("./DiscordWinstonTransport"); const Skins = require("./data/skins"); const db = require("./db"); -const S3 = require("./s3"); const Discord = require("discord.js"); const config = require("./config"); @@ -49,8 +48,8 @@ async function main() { ); switch (argv._[0]) { case "tweet": - const tweetableSkins = await S3.getTweetableSkins(); - if (tweetableSkins.length === 0) { + const tweetableSkin = await Skins.getSkinToTweet(); + if (tweetableSkin == null) { webhook.send( "Oops! I ran out of skins to tweet. Could someone please `!review` some more?" ); @@ -58,7 +57,6 @@ async function main() { break; } - const tweetableSkin = tweetableSkins[0]; const { md5, filename } = tweetableSkin; const output = await spawnPromise( path.resolve(__dirname, "../tweetBot/tweet.py"), @@ -70,8 +68,8 @@ async function main() { ] ); webhook.send(output.trim()); - await S3.markAsTweeted(md5); - const remainingSkinCount = tweetableSkins.length - 1; + await Skins.markAsTweeted(md5); + const remainingSkinCount = await Skins.getTweetableSkinCount(); if (remainingSkinCount < 10) { webhook.send( `Only ${remainingSkinCount} approved skins left. Could someone please \`!review\` some more?` @@ -96,6 +94,10 @@ async function main() { console.log(await Skins.getInternetArchiveUrl(hash)); break; } + case "reconcile": { + await Skins.reconcile(); + break; + } case "skin": { const hash = argv._[1]; logger.info({ hash }); diff --git a/experiments/skin-database/data/skins.js b/experiments/skin-database/data/skins.js index 5ed662e5..f0ede6f0 100644 --- a/experiments/skin-database/data/skins.js +++ b/experiments/skin-database/data/skins.js @@ -5,6 +5,17 @@ const iaItems = db.get("internetArchiveItems"); const S3 = require("../s3"); const logger = require("../logger"); +const TWEETABLE_QUERY = { + tweeted: { $ne: true }, + approved: true, + rejected: { $ne: true }, +}; + +const REVIEWABLE_QUERY = { + tweeted: { $ne: true }, + approved: { $ne: true }, + rejected: { $ne: true }, +}; function getSkinRecord(skin) { const { md5, @@ -74,7 +85,7 @@ async function getSkinByMd5(md5) { internetArchiveItemName = internetArchiveItem.identifier; internetArchiveUrl = getInternetArchiveUrl(internetArchiveItemName); } - const tweetStatus = await getTweetStatus(md5); + const tweetStatus = await getStatus(md5); return { ...getSkinRecord(skin), tweetStatus, @@ -108,8 +119,79 @@ function getInternetArchiveUrl(itemName) { return itemName == null ? null : `https://archive.org/details/${itemName}`; } -async function getTweetStatus(md5) { - return S3.getStatus(md5); +function getTweetableSkinCount() { + return skins.count(TWEETABLE_QUERY); +} + +async function markAsTweeted(md5) { + await skins.findOneAndUpdate({ md5 }, { $set: { tweeted: true } }); + return S3.markAsTweeted(md5); +} + +async function getStatus(md5) { + const skin = await skins.findOne({ md5 }); + if (skin.tweeted) { + return "TWEETED"; + } + if (skin.rejected) { + return "REJECTED"; + } + if (skin.approved) { + return "APPROVED"; + } + return "UNREVIEWED"; +} + +async function approve(md5) { + await skins.findOneAndUpdate({ md5 }, { $set: { approved: true } }); + return S3.approve(md5); +} + +async function reject(md5) { + await skins.findOneAndUpdate({ md5 }, { $set: { rejected: true } }); + return S3.reject(md5); +} + +async function getSkinToReview() { + const skin = await skins.findOne(REVIEWABLE_QUERY); + const { canonicalFilename, md5 } = getSkinRecord(skin); + return { filename: canonicalFilename, md5 }; +} + +async function getSkinToTweet() { + const skin = await skins.findOne(TWEETABLE_QUERY); + if (skin == null) { + return null; + } + const { canonicalFilename, md5 } = getSkinRecord(skin); + return { filename: canonicalFilename, md5 }; +} + +async function getStats() { + const approved = await skins.count({ approved: true }); + const rejected = await skins.count({ rejected: true }); + const tweeted = await skins.count({ tweeted: true }); + const tweetable = await getTweetableSkinCount(); + return { approved, rejected, tweeted, tweetable }; +} + +async function reconcile() { + const [approved, rejected, tweeted] = await Promise.all([ + S3.getAllApproved(), + S3.getAllRejected(), + S3.getAllTweeted(), + ]); + await Promise.all([ + ...approved.map(md5 => + skins.findOneAndUpdate({ md5 }, { $set: { approved: true } }) + ), + ...rejected.map(md5 => + skins.findOneAndUpdate({ md5 }, { $set: { rejected: true } }) + ), + ...tweeted.map(md5 => + skins.findOneAndUpdate({ md5 }, { $set: { tweeted: true } }) + ), + ]); } module.exports = { @@ -118,6 +200,14 @@ module.exports = { getScreenshotUrl, getSkinUrl, getInternetArchiveUrl, - getTweetStatus, getSkinByMd5, + markAsTweeted, + getStatus, + approve, + reject, + getSkinToReview, + getStats, + getTweetableSkinCount, + reconcile, + getSkinToTweet, }; diff --git a/experiments/skin-database/discord-bot/commands/approve.js b/experiments/skin-database/discord-bot/commands/approve.js index 10226322..27372773 100644 --- a/experiments/skin-database/discord-bot/commands/approve.js +++ b/experiments/skin-database/discord-bot/commands/approve.js @@ -1,4 +1,4 @@ -const { approve, getStatus } = require("../../s3"); +const { approve, getStatus } = require("../../data/skins"); const Utils = require("../utils"); const TWEET_BOT_CHANNEL_ID = "445577489834704906"; diff --git a/experiments/skin-database/discord-bot/commands/reject.js b/experiments/skin-database/discord-bot/commands/reject.js index 808594d9..4938be71 100644 --- a/experiments/skin-database/discord-bot/commands/reject.js +++ b/experiments/skin-database/discord-bot/commands/reject.js @@ -1,4 +1,4 @@ -const { reject, getStatus } = require("../../s3"); +const { reject, getStatus } = require("../../data/skins"); const Utils = require("../utils"); const TWEET_BOT_CHANNEL_ID = "445577489834704906"; diff --git a/experiments/skin-database/discord-bot/commands/review.js b/experiments/skin-database/discord-bot/commands/review.js index 81af41fb..369f6a78 100644 --- a/experiments/skin-database/discord-bot/commands/review.js +++ b/experiments/skin-database/discord-bot/commands/review.js @@ -1,12 +1,12 @@ -const { getSkinToReview } = require("../../s3"); +const Skins = require("../../data/skins"); const Utils = require("../utils"); async function reviewSkin(message) { - const skin = await getSkinToReview(); - if(skin == null) { + const skin = await Skins.getSkinToReview(); + if (skin == null) { throw new Error("No skins to review"); } - const {md5, filename} = skin; + const { md5 } = skin; await Utils.postSkin({ md5, title: filename => `Review: ${filename}`, @@ -27,7 +27,10 @@ async function handler(message, args) { await reviewSkin(message); } if (count > 1) { - message.channel.send(`Done reviewing ${count} skins. Thanks!`); + const tweetableCount = await Skins.getTweetableSkinCount(); + message.channel.send( + `Done reviewing ${count} skins. There are now ${tweetableCount} Tweetable skins. Thanks!` + ); } } diff --git a/experiments/skin-database/discord-bot/commands/stats.js b/experiments/skin-database/discord-bot/commands/stats.js index b4efa6dc..b97096b2 100644 --- a/experiments/skin-database/discord-bot/commands/stats.js +++ b/experiments/skin-database/discord-bot/commands/stats.js @@ -1,5 +1,5 @@ const { getCache } = require("../info"); -const { getStats } = require("../../s3"); +const { getStats } = require("../../data/skins"); async function handler(message) { const info = getCache(); diff --git a/experiments/skin-database/discord-bot/s3.js b/experiments/skin-database/discord-bot/s3.js deleted file mode 100644 index 58f89f46..00000000 --- a/experiments/skin-database/discord-bot/s3.js +++ /dev/null @@ -1,112 +0,0 @@ -const AWS = require("aws-sdk"); -AWS.config.update({ region: "us-west-2" }); - -const s3 = new AWS.S3(); - -function getFile(key) { - return new Promise((resolve, rejectPromise) => { - const bucketName = "winamp2-js-skins"; - s3.getObject({ Bucket: bucketName, Key: key }, (err, data) => { - if (err) { - rejectPromise(err); - return; - } - const body = Buffer.from(data.Body).toString("utf8"); - resolve(body); - }); - }); -} - -function putFile(key, body) { - return new Promise((resolve, rejectPromise) => { - const bucketName = "winamp2-js-skins"; - s3.putObject({ Bucket: bucketName, Key: key, Body: body }, err => { - if (err) { - rejectPromise(err); - return; - } - resolve(); - }); - }); -} - -function getLines(body) { - return body.split("\n").map(line => line.trim()); -} - -async function getStatus(md5) { - const [approved, rejected, tweeted] = await Promise.all([ - getFile("approved.txt"), - getFile("rejected.txt"), - getFile("tweeted.txt") - ]); - const approvedSet = new Set(getLines(approved)); - const rejectedSet = new Set(getLines(rejected)); - const tweetedSet = new Set(getLines(tweeted)); - if (tweetedSet.has(md5)) { - return "TWEETED"; - } - if (rejectedSet.has(md5)) { - return "REJECTED"; - } - if (approvedSet.has(md5)) { - return "APPROVED"; - } - return "UNREVIEWED"; -} - -async function getStats() { - const [approved, rejected, tweeted] = await Promise.all([ - getFile("approved.txt"), - getFile("rejected.txt"), - getFile("tweeted.txt") - ]); - return { - approved: new Set(approved).size - new Set(tweeted).size, - rejected: new Set(rejected).size, - tweeted: new Set(tweeted).size - }; -} - -async function getSkinToReview() { - const [filenames, approved, rejected, tweeted] = await Promise.all([ - getFile("filenames.txt"), - getFile("approved.txt"), - getFile("rejected.txt"), - getFile("tweeted.txt") - ]); - - const approvedSet = new Set(getLines(approved)); - const rejectedSet = new Set(getLines(rejected)); - const tweetedSet = new Set(getLines(tweeted)); - - const filenameLines = getLines(filenames); - const skins = filenameLines.map(line => { - const [md5, ...filename] = line.split(" "); - return { md5, filename: filename.join(" ") }; - }); - const toReview = skins.filter(({ md5 }) => { - return !( - approvedSet.has(md5) || - rejectedSet.has(md5) || - tweetedSet.has(md5) - ); - }); - return toReview[0]; -} - -async function appendLine(key, line) { - const currentContent = await getFile(key); - const newContent = `${currentContent}${line}\n`; - return putFile(key, newContent); -} - -async function approve(md5) { - return appendLine("approved.txt", md5); -} - -async function reject(md5) { - return appendLine("rejected.txt", md5); -} - -module.exports = { getSkinToReview, approve, reject, getStatus, getStats }; diff --git a/experiments/skin-database/discord-bot/utils.js b/experiments/skin-database/discord-bot/utils.js index 5f15a712..44e7397a 100644 --- a/experiments/skin-database/discord-bot/utils.js +++ b/experiments/skin-database/discord-bot/utils.js @@ -87,7 +87,7 @@ async function postSkin({ md5, title, dest }) { const user = vote.users.first(); switch (vote.emoji.name) { case "👍": - await approve(md5); + await Skins.approve(md5); logger.info(`${user.username} approved ${md5}`); await msg.channel.send( `${canonicalFilename} was approved by ${user.username}` @@ -95,7 +95,7 @@ async function postSkin({ md5, title, dest }) { msg.react("✅"); break; case "👎": - await reject(md5); + await Skins.reject(md5); logger.info(`${user.username} rejected ${md5}`); await msg.channel.send( `${canonicalFilename} was rejected by ${user.username}` diff --git a/experiments/skin-database/s3.js b/experiments/skin-database/s3.js index 1e6f85aa..fef2e5e0 100644 --- a/experiments/skin-database/s3.js +++ b/experiments/skin-database/s3.js @@ -31,7 +31,22 @@ function putFile(key, body) { } function getLines(body) { - return body.split("\n").map(line => line.trim()); + return body + .trim() + .split("\n") + .map(line => line.trim()); +} + +async function getAllApproved() { + return getLines(await getFile("approved.txt")); +} + +async function getAllRejected() { + return getLines(await getFile("rejected.txt")); +} + +async function getAllTweeted() { + return getLines(await getFile("tweeted.txt")); } async function getStatus(md5) { @@ -147,4 +162,7 @@ module.exports = { getStats, markAsTweeted, getTweetableSkins, + getAllApproved, + getAllRejected, + getAllTweeted, };