mirrors/webamp
1
0
Fork 0
mirror of https://github.com/captbaritone/webamp synced 2024-11-23 08:39:15 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Move all s3 APIs into the database interface

Browse Source
This commit is contained in:
Jordan Eldredge 2019-10-14 19:00:39 -04:00
parent f2c6835ee1
commit d4e5b91229
10 changed files with 135 additions and 133 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.eslintignore
View File

@ -3,3 +3,4 @@ built/
coverage/
**/node_modules/
examples/webpack/bundle.js
experiments/tweetBot/env/

14
experiments/skin-database/cli.js
View File

@ -7,7 +7,6 @@ const logger = require("./logger");
const DiscordWinstonTransport = require("./DiscordWinstonTransport");
const Skins = require("./data/skins");
const db = require("./db");
const S3 = require("./s3");
const Discord = require("discord.js");
const config = require("./config");
@ -49,8 +48,8 @@ async function main() {
);
switch (argv._[0]) {
case "tweet":
const tweetableSkins = await S3.getTweetableSkins();
if (tweetableSkins.length === 0) {
const tweetableSkin = await Skins.getSkinToTweet();
if (tweetableSkin == null) {
webhook.send(
"Oops! I ran out of skins to tweet. Could someone please `!review` some more?"
);
@ -58,7 +57,6 @@ async function main() {
break;
}
const tweetableSkin = tweetableSkins[0];
const { md5, filename } = tweetableSkin;
const output = await spawnPromise(
path.resolve(__dirname, "../tweetBot/tweet.py"),
@ -70,8 +68,8 @@ async function main() {
]
);
webhook.send(output.trim());
await S3.markAsTweeted(md5);
const remainingSkinCount = tweetableSkins.length - 1;
await Skins.markAsTweeted(md5);
const remainingSkinCount = await Skins.getTweetableSkinCount();
if (remainingSkinCount < 10) {
webhook.send(
`Only ${remainingSkinCount} approved skins left. Could someone please \`!review\` some more?`
@ -96,6 +94,10 @@ async function main() {
console.log(await Skins.getInternetArchiveUrl(hash));
break;
}
case "reconcile": {
await Skins.reconcile();
break;
}
case "skin": {
const hash = argv._[1];
logger.info({ hash });

98
experiments/skin-database/data/skins.js
View File

@ -5,6 +5,17 @@ const iaItems = db.get("internetArchiveItems");
const S3 = require("../s3");
const logger = require("../logger");
const TWEETABLE_QUERY = {
tweeted: { $ne: true },
approved: true,
rejected: { $ne: true },
};
const REVIEWABLE_QUERY = {
tweeted: { $ne: true },
approved: { $ne: true },
rejected: { $ne: true },
};
function getSkinRecord(skin) {
const {
md5,
@ -74,7 +85,7 @@ async function getSkinByMd5(md5) {
internetArchiveItemName = internetArchiveItem.identifier;
internetArchiveUrl = getInternetArchiveUrl(internetArchiveItemName);
}
const tweetStatus = await getTweetStatus(md5);
const tweetStatus = await getStatus(md5);
return {
...getSkinRecord(skin),
tweetStatus,
@ -108,8 +119,79 @@ function getInternetArchiveUrl(itemName) {
return itemName == null ? null : `https://archive.org/details/${itemName}`;
}
async function getTweetStatus(md5) {
return S3.getStatus(md5);
function getTweetableSkinCount() {
return skins.count(TWEETABLE_QUERY);
}
async function markAsTweeted(md5) {
await skins.findOneAndUpdate({ md5 }, { $set: { tweeted: true } });
return S3.markAsTweeted(md5);
}
async function getStatus(md5) {
const skin = await skins.findOne({ md5 });
if (skin.tweeted) {
return "TWEETED";
}
if (skin.rejected) {
return "REJECTED";
}
if (skin.approved) {
return "APPROVED";
}
return "UNREVIEWED";
}
async function approve(md5) {
await skins.findOneAndUpdate({ md5 }, { $set: { approved: true } });
return S3.approve(md5);
}
async function reject(md5) {
await skins.findOneAndUpdate({ md5 }, { $set: { rejected: true } });
return S3.reject(md5);
}
async function getSkinToReview() {
const skin = await skins.findOne(REVIEWABLE_QUERY);
const { canonicalFilename, md5 } = getSkinRecord(skin);
return { filename: canonicalFilename, md5 };
}
async function getSkinToTweet() {
const skin = await skins.findOne(TWEETABLE_QUERY);
if (skin == null) {
return null;
}
const { canonicalFilename, md5 } = getSkinRecord(skin);
return { filename: canonicalFilename, md5 };
}
async function getStats() {
const approved = await skins.count({ approved: true });
const rejected = await skins.count({ rejected: true });
const tweeted = await skins.count({ tweeted: true });
const tweetable = await getTweetableSkinCount();
return { approved, rejected, tweeted, tweetable };
}
async function reconcile() {
const [approved, rejected, tweeted] = await Promise.all([
S3.getAllApproved(),
S3.getAllRejected(),
S3.getAllTweeted(),
]);
await Promise.all([
...approved.map(md5 =>
skins.findOneAndUpdate({ md5 }, { $set: { approved: true } })
),
...rejected.map(md5 =>
skins.findOneAndUpdate({ md5 }, { $set: { rejected: true } })
),
...tweeted.map(md5 =>
skins.findOneAndUpdate({ md5 }, { $set: { tweeted: true } })
),
]);
}
module.exports = {
@ -118,6 +200,14 @@ module.exports = {
getScreenshotUrl,
getSkinUrl,
getInternetArchiveUrl,
getTweetStatus,
getSkinByMd5,
markAsTweeted,
getStatus,
approve,
reject,
getSkinToReview,
getStats,
getTweetableSkinCount,
reconcile,
getSkinToTweet,
};

2
experiments/skin-database/discord-bot/commands/approve.js
View File

@ -1,4 +1,4 @@
const { approve, getStatus } = require("../../s3");
const { approve, getStatus } = require("../../data/skins");
const Utils = require("../utils");
const TWEET_BOT_CHANNEL_ID = "445577489834704906";

2
experiments/skin-database/discord-bot/commands/reject.js
View File

@ -1,4 +1,4 @@
const { reject, getStatus } = require("../../s3");
const { reject, getStatus } = require("../../data/skins");
const Utils = require("../utils");
const TWEET_BOT_CHANNEL_ID = "445577489834704906";

11
experiments/skin-database/discord-bot/commands/review.js
View File

@ -1,12 +1,12 @@
const { getSkinToReview } = require("../../s3");
const Skins = require("../../data/skins");
const Utils = require("../utils");
async function reviewSkin(message) {
const skin = await getSkinToReview();
const skin = await Skins.getSkinToReview();
if (skin == null) {
throw new Error("No skins to review");
}
const {md5, filename} = skin;
const { md5 } = skin;
await Utils.postSkin({
md5,
title: filename => `Review: ${filename}`,
@ -27,7 +27,10 @@ async function handler(message, args) {
await reviewSkin(message);
}
if (count > 1) {
message.channel.send(`Done reviewing ${count} skins. Thanks!`);
const tweetableCount = await Skins.getTweetableSkinCount();
message.channel.send(
`Done reviewing ${count} skins. There are now ${tweetableCount} Tweetable skins. Thanks!`
);
}
}

2
experiments/skin-database/discord-bot/commands/stats.js
View File

@ -1,5 +1,5 @@
const { getCache } = require("../info");
const { getStats } = require("../../s3");
const { getStats } = require("../../data/skins");
async function handler(message) {
const info = getCache();

112
experiments/skin-database/discord-bot/s3.js
View File

@ -1,112 +0,0 @@
const AWS = require("aws-sdk");
AWS.config.update({ region: "us-west-2" });
const s3 = new AWS.S3();
function getFile(key) {
return new Promise((resolve, rejectPromise) => {
const bucketName = "winamp2-js-skins";
s3.getObject({ Bucket: bucketName, Key: key }, (err, data) => {
if (err) {
rejectPromise(err);
return;
}
const body = Buffer.from(data.Body).toString("utf8");
resolve(body);
});
});
}
function putFile(key, body) {
return new Promise((resolve, rejectPromise) => {
const bucketName = "winamp2-js-skins";
s3.putObject({ Bucket: bucketName, Key: key, Body: body }, err => {
if (err) {
rejectPromise(err);
return;
}
resolve();
});
});
}
function getLines(body) {
return body.split("\n").map(line => line.trim());
}
async function getStatus(md5) {
const [approved, rejected, tweeted] = await Promise.all([
getFile("approved.txt"),
getFile("rejected.txt"),
getFile("tweeted.txt")
]);
const approvedSet = new Set(getLines(approved));
const rejectedSet = new Set(getLines(rejected));
const tweetedSet = new Set(getLines(tweeted));
if (tweetedSet.has(md5)) {
return "TWEETED";
}
if (rejectedSet.has(md5)) {
return "REJECTED";
}
if (approvedSet.has(md5)) {
return "APPROVED";
}
return "UNREVIEWED";
}
async function getStats() {
const [approved, rejected, tweeted] = await Promise.all([
getFile("approved.txt"),
getFile("rejected.txt"),
getFile("tweeted.txt")
]);
return {
approved: new Set(approved).size - new Set(tweeted).size,
rejected: new Set(rejected).size,
tweeted: new Set(tweeted).size
};
}
async function getSkinToReview() {
const [filenames, approved, rejected, tweeted] = await Promise.all([
getFile("filenames.txt"),
getFile("approved.txt"),
getFile("rejected.txt"),
getFile("tweeted.txt")
]);
const approvedSet = new Set(getLines(approved));
const rejectedSet = new Set(getLines(rejected));
const tweetedSet = new Set(getLines(tweeted));
const filenameLines = getLines(filenames);
const skins = filenameLines.map(line => {
const [md5, ...filename] = line.split(" ");
return { md5, filename: filename.join(" ") };
});
const toReview = skins.filter(({ md5 }) => {
return !(
approvedSet.has(md5) ||
rejectedSet.has(md5) ||
tweetedSet.has(md5)
);
});
return toReview[0];
}
async function appendLine(key, line) {
const currentContent = await getFile(key);
const newContent = `${currentContent}${line}\n`;
return putFile(key, newContent);
}
async function approve(md5) {
return appendLine("approved.txt", md5);
}
async function reject(md5) {
return appendLine("rejected.txt", md5);
}
module.exports = { getSkinToReview, approve, reject, getStatus, getStats };

4
experiments/skin-database/discord-bot/utils.js
View File

@ -87,7 +87,7 @@ async function postSkin({ md5, title, dest }) {
const user = vote.users.first();
switch (vote.emoji.name) {
case "👍":
await approve(md5);
await Skins.approve(md5);
logger.info(`${user.username} approved ${md5}`);
await msg.channel.send(
`${canonicalFilename} was approved by ${user.username}`
@ -95,7 +95,7 @@ async function postSkin({ md5, title, dest }) {
msg.react("✅");
break;
case "👎":
await reject(md5);
await Skins.reject(md5);
logger.info(`${user.username} rejected ${md5}`);
await msg.channel.send(
`${canonicalFilename} was rejected by ${user.username}`

20
experiments/skin-database/s3.js
View File

@ -31,7 +31,22 @@ function putFile(key, body) {
}
function getLines(body) {
return body.split("\n").map(line => line.trim());
return body
.trim()
.split("\n")
.map(line => line.trim());
}
async function getAllApproved() {
return getLines(await getFile("approved.txt"));
}
async function getAllRejected() {
return getLines(await getFile("rejected.txt"));
}
async function getAllTweeted() {
return getLines(await getFile("tweeted.txt"));
}
async function getStatus(md5) {
@ -147,4 +162,7 @@ module.exports = {
getStats,
markAsTweeted,
getTweetableSkins,
getAllApproved,
getAllRejected,
getAllTweeted,
};