Move all s3 APIs into the database interface

.eslintignore

@@ -3,3 +3,4 @@ built/
 coverage/
 **/node_modules/
 examples/webpack/bundle.js
+experiments/tweetBot/env/

experiments/skin-database/cli.js

@@ -7,7 +7,6 @@ const logger = require("./logger");
 const DiscordWinstonTransport = require("./DiscordWinstonTransport");
 const Skins = require("./data/skins");
 const db = require("./db");
-const S3 = require("./s3");
 const Discord = require("discord.js");
 const config = require("./config");
 
@@ -49,8 +48,8 @@ async function main() {
   );
   switch (argv._[0]) {
     case "tweet":
-      const tweetableSkins = await S3.getTweetableSkins();
-      if (tweetableSkins.length === 0) {
+      const tweetableSkin = await Skins.getSkinToTweet();
+      if (tweetableSkin == null) {
         webhook.send(
           "Oops! I ran out of skins to tweet. Could someone please `!review` some more?"
         );
@@ -58,7 +57,6 @@ async function main() {
         break;
       }
 
-      const tweetableSkin = tweetableSkins[0];
       const { md5, filename } = tweetableSkin;
       const output = await spawnPromise(
         path.resolve(__dirname, "../tweetBot/tweet.py"),
@@ -70,8 +68,8 @@ async function main() {
         ]
       );
       webhook.send(output.trim());
-      await S3.markAsTweeted(md5);
-      const remainingSkinCount = tweetableSkins.length - 1;
+      await Skins.markAsTweeted(md5);
+      const remainingSkinCount = await Skins.getTweetableSkinCount();
       if (remainingSkinCount < 10) {
         webhook.send(
           `Only ${remainingSkinCount} approved skins left. Could someone please \`!review\` some more?`
@@ -96,6 +94,10 @@ async function main() {
       console.log(await Skins.getInternetArchiveUrl(hash));
       break;
     }
+    case "reconcile": {
+      await Skins.reconcile();
+      break;
+    }
     case "skin": {
       const hash = argv._[1];
       logger.info({ hash });

experiments/skin-database/data/skins.js

@@ -5,6 +5,17 @@ const iaItems = db.get("internetArchiveItems");
 const S3 = require("../s3");
 const logger = require("../logger");
 
+const TWEETABLE_QUERY = {
+  tweeted: { $ne: true },
+  approved: true,
+  rejected: { $ne: true },
+};
+
+const REVIEWABLE_QUERY = {
+  tweeted: { $ne: true },
+  approved: { $ne: true },
+  rejected: { $ne: true },
+};
 function getSkinRecord(skin) {
   const {
     md5,
@@ -74,7 +85,7 @@ async function getSkinByMd5(md5) {
     internetArchiveItemName = internetArchiveItem.identifier;
     internetArchiveUrl = getInternetArchiveUrl(internetArchiveItemName);
   }
-  const tweetStatus = await getTweetStatus(md5);
+  const tweetStatus = await getStatus(md5);
   return {
     ...getSkinRecord(skin),
     tweetStatus,
@@ -108,8 +119,79 @@ function getInternetArchiveUrl(itemName) {
   return itemName == null ? null : `https://archive.org/details/${itemName}`;
 }
 
-async function getTweetStatus(md5) {
-  return S3.getStatus(md5);
+function getTweetableSkinCount() {
+  return skins.count(TWEETABLE_QUERY);
+}
+
+async function markAsTweeted(md5) {
+  await skins.findOneAndUpdate({ md5 }, { $set: { tweeted: true } });
+  return S3.markAsTweeted(md5);
+}
+
+async function getStatus(md5) {
+  const skin = await skins.findOne({ md5 });
+  if (skin.tweeted) {
+    return "TWEETED";
+  }
+  if (skin.rejected) {
+    return "REJECTED";
+  }
+  if (skin.approved) {
+    return "APPROVED";
+  }
+  return "UNREVIEWED";
+}
+
+async function approve(md5) {
+  await skins.findOneAndUpdate({ md5 }, { $set: { approved: true } });
+  return S3.approve(md5);
+}
+
+async function reject(md5) {
+  await skins.findOneAndUpdate({ md5 }, { $set: { rejected: true } });
+  return S3.reject(md5);
+}
+
+async function getSkinToReview() {
+  const skin = await skins.findOne(REVIEWABLE_QUERY);
+  const { canonicalFilename, md5 } = getSkinRecord(skin);
+  return { filename: canonicalFilename, md5 };
+}
+
+async function getSkinToTweet() {
+  const skin = await skins.findOne(TWEETABLE_QUERY);
+  if (skin == null) {
+    return null;
+  }
+  const { canonicalFilename, md5 } = getSkinRecord(skin);
+  return { filename: canonicalFilename, md5 };
+}
+
+async function getStats() {
+  const approved = await skins.count({ approved: true });
+  const rejected = await skins.count({ rejected: true });
+  const tweeted = await skins.count({ tweeted: true });
+  const tweetable = await getTweetableSkinCount();
+  return { approved, rejected, tweeted, tweetable };
+}
+
+async function reconcile() {
+  const [approved, rejected, tweeted] = await Promise.all([
+    S3.getAllApproved(),
+    S3.getAllRejected(),
+    S3.getAllTweeted(),
+  ]);
+  await Promise.all([
+    ...approved.map(md5 =>
+      skins.findOneAndUpdate({ md5 }, { $set: { approved: true } })
+    ),
+    ...rejected.map(md5 =>
+      skins.findOneAndUpdate({ md5 }, { $set: { rejected: true } })
+    ),
+    ...tweeted.map(md5 =>
+      skins.findOneAndUpdate({ md5 }, { $set: { tweeted: true } })
+    ),
+  ]);
 }
 
 module.exports = {
@@ -118,6 +200,14 @@ module.exports = {
   getScreenshotUrl,
   getSkinUrl,
   getInternetArchiveUrl,
-  getTweetStatus,
   getSkinByMd5,
+  markAsTweeted,
+  getStatus,
+  approve,
+  reject,
+  getSkinToReview,
+  getStats,
+  getTweetableSkinCount,
+  reconcile,
+  getSkinToTweet,
 };

experiments/skin-database/discord-bot/commands/approve.js

@@ -1,4 +1,4 @@
-const { approve, getStatus } = require("../../s3");
+const { approve, getStatus } = require("../../data/skins");
 const Utils = require("../utils");
 
 const TWEET_BOT_CHANNEL_ID = "445577489834704906";

experiments/skin-database/discord-bot/commands/reject.js

@@ -1,4 +1,4 @@
-const { reject, getStatus } = require("../../s3");
+const { reject, getStatus } = require("../../data/skins");
 const Utils = require("../utils");
 
 const TWEET_BOT_CHANNEL_ID = "445577489834704906";

experiments/skin-database/discord-bot/commands/review.js

@@ -1,12 +1,12 @@
-const { getSkinToReview } = require("../../s3");
+const Skins = require("../../data/skins");
 const Utils = require("../utils");
 
 async function reviewSkin(message) {
-  const skin = await getSkinToReview();
-  if(skin == null) {
+  const skin = await Skins.getSkinToReview();
+  if (skin == null) {
     throw new Error("No skins to review");
   }
-  const {md5, filename} = skin;
+  const { md5 } = skin;
   await Utils.postSkin({
     md5,
     title: filename => `Review: ${filename}`,
@@ -27,7 +27,10 @@ async function handler(message, args) {
     await reviewSkin(message);
   }
   if (count > 1) {
-    message.channel.send(`Done reviewing ${count} skins. Thanks!`);
+    const tweetableCount = await Skins.getTweetableSkinCount();
+    message.channel.send(
+      `Done reviewing ${count} skins. There are now ${tweetableCount} Tweetable skins. Thanks!`
+    );
   }
 }
 

experiments/skin-database/discord-bot/commands/stats.js

@@ -1,5 +1,5 @@
 const { getCache } = require("../info");
-const { getStats } = require("../../s3");
+const { getStats } = require("../../data/skins");
 
 async function handler(message) {
   const info = getCache();

experiments/skin-database/discord-bot/s3.js

@@ -1,112 +0,0 @@
-const AWS = require("aws-sdk");
-AWS.config.update({ region: "us-west-2" });
-
-const s3 = new AWS.S3();
-
-function getFile(key) {
-  return new Promise((resolve, rejectPromise) => {
-    const bucketName = "winamp2-js-skins";
-    s3.getObject({ Bucket: bucketName, Key: key }, (err, data) => {
-      if (err) {
-        rejectPromise(err);
-        return;
-      }
-      const body = Buffer.from(data.Body).toString("utf8");
-      resolve(body);
-    });
-  });
-}
-
-function putFile(key, body) {
-  return new Promise((resolve, rejectPromise) => {
-    const bucketName = "winamp2-js-skins";
-    s3.putObject({ Bucket: bucketName, Key: key, Body: body }, err => {
-      if (err) {
-        rejectPromise(err);
-        return;
-      }
-      resolve();
-    });
-  });
-}
-
-function getLines(body) {
-  return body.split("\n").map(line => line.trim());
-}
-
-async function getStatus(md5) {
-  const [approved, rejected, tweeted] = await Promise.all([
-    getFile("approved.txt"),
-    getFile("rejected.txt"),
-    getFile("tweeted.txt")
-  ]);
-  const approvedSet = new Set(getLines(approved));
-  const rejectedSet = new Set(getLines(rejected));
-  const tweetedSet = new Set(getLines(tweeted));
-  if (tweetedSet.has(md5)) {
-    return "TWEETED";
-  }
-  if (rejectedSet.has(md5)) {
-    return "REJECTED";
-  }
-  if (approvedSet.has(md5)) {
-    return "APPROVED";
-  }
-  return "UNREVIEWED";
-}
-
-async function getStats() {
-  const [approved, rejected, tweeted] = await Promise.all([
-    getFile("approved.txt"),
-    getFile("rejected.txt"),
-    getFile("tweeted.txt")
-  ]);
-  return {
-    approved: new Set(approved).size - new Set(tweeted).size,
-    rejected: new Set(rejected).size,
-    tweeted: new Set(tweeted).size
-  };
-}
-
-async function getSkinToReview() {
-  const [filenames, approved, rejected, tweeted] = await Promise.all([
-    getFile("filenames.txt"),
-    getFile("approved.txt"),
-    getFile("rejected.txt"),
-    getFile("tweeted.txt")
-  ]);
-
-  const approvedSet = new Set(getLines(approved));
-  const rejectedSet = new Set(getLines(rejected));
-  const tweetedSet = new Set(getLines(tweeted));
-
-  const filenameLines = getLines(filenames);
-  const skins = filenameLines.map(line => {
-    const [md5, ...filename] = line.split(" ");
-    return { md5, filename: filename.join(" ") };
-  });
-  const toReview = skins.filter(({ md5 }) => {
-    return !(
-      approvedSet.has(md5) ||
-      rejectedSet.has(md5) ||
-      tweetedSet.has(md5)
-    );
-  });
-  return toReview[0];
-}
-
-async function appendLine(key, line) {
-  const currentContent = await getFile(key);
-  const newContent = `${currentContent}${line}\n`;
-  return putFile(key, newContent);
-}
-
-async function approve(md5) {
-  return appendLine("approved.txt", md5);
-}
-
-async function reject(md5) {
-  return appendLine("rejected.txt", md5);
-}
-
-module.exports = { getSkinToReview, approve, reject, getStatus, getStats };

experiments/skin-database/discord-bot/utils.js

@@ -87,7 +87,7 @@ async function postSkin({ md5, title, dest }) {
     const user = vote.users.first();
     switch (vote.emoji.name) {
       case "👍":
-        await approve(md5);
+        await Skins.approve(md5);
         logger.info(`${user.username} approved ${md5}`);
         await msg.channel.send(
           `${canonicalFilename} was approved by ${user.username}`
@@ -95,7 +95,7 @@ async function postSkin({ md5, title, dest }) {
         msg.react("✅");
         break;
       case "👎":
-        await reject(md5);
+        await Skins.reject(md5);
         logger.info(`${user.username} rejected ${md5}`);
         await msg.channel.send(
           `${canonicalFilename} was rejected by ${user.username}`

experiments/skin-database/s3.js

@@ -31,7 +31,22 @@ function putFile(key, body) {
 }
 
 function getLines(body) {
-  return body.split("\n").map(line => line.trim());
+  return body
+    .trim()
+    .split("\n")
+    .map(line => line.trim());
+}
+
+async function getAllApproved() {
+  return getLines(await getFile("approved.txt"));
+}
+
+async function getAllRejected() {
+  return getLines(await getFile("rejected.txt"));
+}
+
+async function getAllTweeted() {
+  return getLines(await getFile("tweeted.txt"));
 }
 
 async function getStatus(md5) {
@@ -147,4 +162,7 @@ module.exports = {
   getStats,
   markAsTweeted,
   getTweetableSkins,
+  getAllApproved,
+  getAllRejected,
+  getAllTweeted,
 };