remove nunjucks templating in experimental plugin themes (#4933)

* remove nunjucks usage from core themes

* adds package downloader script

* adds runtime validation for plugins still using nunjucks

* generateThemeCSS no longer needs to be async

* removes Nunjucks as valid type

.gitignore

@@ -8,6 +8,7 @@ pids
 *.pid
 *.seed
 *.pid.lock
+/scripts/npm-plugins
 lib-cov
 coverage
 .lock-wscript

package.json

@@ -36,6 +36,7 @@
     "hard-reset": "npm run clean && npm run bootstrap && npm run app-start",
     "type-check": "lerna run type-check",
     "changelog-image": "esr ./scripts/changelog-image/changelog-image.ts",
+    "download-all-npm-plugins": "esr ./scripts/download-all-npm-plugins.ts",
     "dev": "npm start --prefix packages/insomnia"
   },
   "devDependencies": {

packages/insomnia/src/plugins/misc.test.ts

@@ -1,6 +1,84 @@
 import { describe, expect, it } from '@jest/globals';
 
-import { validateThemeName } from './misc';
+import { containsNunjucks, PluginTheme, validateTheme, validateThemeName } from './misc';
+
+describe('containsNunjucks', () => {
+  it('will return true if the value contains nunjucks without', () => {
+    expect(containsNunjucks('{{asdf}}')).toBeTruthy();
+  });
+
+  it('will return true if the value contains nunjucks with spaces', () => {
+    expect(containsNunjucks('{{ asdf }}')).toBeTruthy();
+  });
+
+  it('will return false if the value contains nunjucks', () => {
+    expect(containsNunjucks('#rgb(1,2,3)')).toBeFalsy();
+  });
+});
+
+describe('validateTheme', () => {
+  const nunjucksValue = '{{ nunjucks.4.lyfe }}';
+  const name = 'mock-plugin';
+  const displayName = 'Mock Plugin';
+  const mockMessage = (path: string[]) => `[plugin] Nunjucks values in plugin themes are no longer valid. The plugin ${displayName} (${name}) has an invalid value, "${nunjucksValue}" at the path $.theme.${path.join('.')}`;
+
+  jest.spyOn(console, 'error').mockImplementation(() => {});
+
+  it('will validate rawCSS in the plugin theme', () => {
+    const pluginTheme: PluginTheme = {
+      name,
+      displayName,
+      theme: {
+        rawCss: nunjucksValue,
+      },
+    };
+
+    validateTheme(pluginTheme);
+
+    const message = mockMessage(['rawCss']);
+    expect(console.error).toHaveBeenLastCalledWith(message);
+  });
+
+  it('will validate top-level theme blocks in the plugin theme', () => {
+    const pluginTheme: PluginTheme = {
+      name,
+      displayName,
+      theme: {
+        background: {
+          default: nunjucksValue,
+          info: '#abcdef',
+        },
+      },
+    };
+
+    validateTheme(pluginTheme);
+
+    const message = mockMessage(['background', 'default']);
+    expect(console.error).toHaveBeenLastCalledWith(message);
+  });
+
+  it('will validate styles sub-theme blocks in the plugin theme', () => {
+    const pluginTheme: PluginTheme = {
+      name,
+      displayName,
+      theme: {
+        styles: {
+          appHeader: {
+            foreground: {
+              default: nunjucksValue,
+              info: '#abcdef',
+            },
+          },
+        },
+      },
+    };
+
+    validateTheme(pluginTheme);
+
+    const message = mockMessage(['styles', 'appHeader', 'foreground', 'default']);
+    expect(console.error).toHaveBeenLastCalledWith(message);
+  });
+});
 
 describe('validateThemeName', () => {
   it('will return valid names as-is', () => {

packages/insomnia/src/plugins/misc.ts

@@ -1,46 +1,57 @@
 import Color from 'color';
+import { forEach, keys, path } from 'ramda';
 import { unreachableCase } from 'ts-assert-unreachable';
 
 import { getAppDefaultTheme } from '../common/constants';
-import { render, THROW_ON_ERROR } from '../common/render';
 import { ThemeSettings } from '../models/settings';
 import type { Theme } from './index';
 import { ColorScheme, getThemes } from './index';
 
-interface ThemeBlock {
+export type HexColor = `#${string}`;
+export type RGBColor = `rgb(${string})`;
+export type RGBAColor = `rgba(${string})`;
+
+export type ThemeColor = HexColor | RGBColor | RGBAColor;
+
+// notice that for each sub-block (`background`, `foreground`, `highlight`) the `default` key is required if the sub-block is present
+export interface ThemeBlock {
   background?: {
-    default: string;
-    success?: string;
-    notice?: string;
-    warning?: string;
-    danger?: string;
-    surprise?: string;
-    info?: string;
+    default: ThemeColor;
+    success?: ThemeColor;
+    notice?: ThemeColor;
+    warning?: ThemeColor;
+    danger?: ThemeColor;
+    surprise?: ThemeColor;
+    info?: ThemeColor;
   };
   foreground?: {
-    default: string;
-    success?: string;
-    notice?: string;
-    warning?: string;
-    danger?: string;
-    surprise?: string;
-    info?: string;
+    default: ThemeColor;
+    success?: ThemeColor;
+    notice?: ThemeColor;
+    warning?: ThemeColor;
+    danger?: ThemeColor;
+    surprise?: ThemeColor;
+    info?: ThemeColor;
   };
   highlight?: {
-    default: string;
-    xxs?: string;
-    xs?: string;
-    sm?: string;
-    md?: string;
-    lg?: string;
-    xl?: string;
+    default: ThemeColor;
+    xxs?: ThemeColor;
+    xs?: ThemeColor;
+    sm?: ThemeColor;
+    md?: ThemeColor;
+    lg?: ThemeColor;
+    xl?: ThemeColor;
   };
 }
 
-type ThemeInner = ThemeBlock & {
-  rawCss?: string;
-  styles?:
-    | {
+export interface CompleteStyleBlock {
+  background: Required<Required<ThemeBlock>['background']>;
+  foreground: Required<Required<ThemeBlock>['foreground']>;
+  highlight: Required<Required<ThemeBlock>['highlight']>;
+}
+
+export interface StylesThemeBlocks {
+  appHeader?: ThemeBlock;
   dialog?: ThemeBlock;
   dialogFooter?: ThemeBlock;
   dialogHeader?: ThemeBlock;
@@ -53,10 +64,16 @@ type ThemeInner = ThemeBlock & {
   sidebar?: ThemeBlock;
   sidebarHeader?: ThemeBlock;
   sidebarList?: ThemeBlock;
+
+  /** does not respect parent wrapping theme */
   tooltip?: ThemeBlock;
+
   transparentOverlay?: ThemeBlock;
 }
-    | null;
+
+export type ThemeInner = ThemeBlock & {
+  rawCss?: string;
+  styles?: StylesThemeBlocks | null;
 };
 
 export interface PluginTheme {
@@ -77,70 +94,105 @@ export const validateThemeName = (name: string) => {
   return validName;
 };
 
-export async function generateThemeCSS(theme: PluginTheme) {
-  const renderedTheme: ThemeInner = await render(
-    theme.theme,
-    theme.theme,
-    null,
-    THROW_ON_ERROR,
-    theme.name,
+export const containsNunjucks = (data: string) => (
+  data.includes('{{') && data.includes('}}')
 );
-  const n = theme.name;
-  validateThemeName(theme.name);
+
+/** In July 2022, the ability to use Nunjucks in themes was removed. This validator is a means of alerting any users of a theme depending on Nunjucks.  The failure mode for this case (in practice) is that the CSS variable will just not be used, thus it's not something we'd want to go as far as throwing an error about. */
+export const validateTheme = (pluginTheme: PluginTheme) => {
+  const checkIfContainsNunjucks = (pluginTheme: PluginTheme) => (keyPath: string[]) => {
+    const data = path(keyPath, pluginTheme.theme);
+
+    if (!data) {
+      return;
+    }
+
+    if (typeof data === 'string' && containsNunjucks(data)) {
+      console.error(`[plugin] Nunjucks values in plugin themes are no longer valid. The plugin ${pluginTheme.displayName} (${pluginTheme.name}) has an invalid value, "${data}" at the path $.theme.${keyPath.join('.')}`);
+    }
+
+    if (typeof data === 'object') {
+      forEach(ownKey => {
+        checkIfContainsNunjucks(pluginTheme)([...keyPath, ownKey]);
+      }, keys(data));
+    }
+  };
+
+  const check = checkIfContainsNunjucks(pluginTheme);
+
+  check(['rawCss']);
+
+  forEach<keyof ThemeBlock>(rootPath => {
+    check([rootPath]);
+
+    forEach(style => {
+      check(['styles', style, rootPath]);
+    }, keys<StylesThemeBlocks>(pluginTheme.theme.styles ?? {}));
+  }, [
+    'background',
+    'foreground',
+    'highlight',
+  ]);
+
+};
+
+export const generateThemeCSS = (pluginTheme: PluginTheme) => {
+  const { theme, name } = pluginTheme;
+  validateTheme(pluginTheme);
+  validateThemeName(name);
 
   let css = '';
-  // For the top-level variables, merge with the base theme to ensure that
-  // we have everything we need.
+  // For the top-level variables, merge with the base theme to ensure that we have everything we need.
   css += wrapStyles(
-    n,
+    name,
     '',
     getThemeBlockCSS({
-      ...renderedTheme,
-      background: { ..._baseTheme.background, ...renderedTheme.background },
-      foreground: { ..._baseTheme.foreground, ...renderedTheme.foreground },
-      highlight: { ..._baseTheme.highlight, ...renderedTheme.highlight },
+      ...theme,
+      background: { ...baseTheme.background, ...theme.background },
+      foreground: { ...baseTheme.foreground, ...theme.foreground },
+      highlight: { ...baseTheme.highlight, ...theme.highlight },
     }),
   );
 
-  if (renderedTheme.styles) {
-    const styles = renderedTheme.styles;
+  if (theme.styles) {
+    const styles = theme.styles;
     // Dropdown Menus
     css += wrapStyles(
-      n,
+      name,
       '.theme--dropdown__menu',
       getThemeBlockCSS(styles.dropdown || styles.dialog),
     );
     // Tooltips
-    css += wrapStyles(n, '.theme--tooltip', getThemeBlockCSS(styles.tooltip || styles.dialog));
+    css += wrapStyles(name, '.theme--tooltip', getThemeBlockCSS(styles.tooltip || styles.dialog));
     // Overlay
     css += wrapStyles(
-      n,
+      name,
       '.theme--transparent-overlay',
       getThemeBlockCSS(styles.transparentOverlay),
     );
     // Dialogs
-    css += wrapStyles(n, '.theme--dialog', getThemeBlockCSS(styles.dialog));
-    css += wrapStyles(n, '.theme--dialog__header', getThemeBlockCSS(styles.dialogHeader));
-    css += wrapStyles(n, '.theme--dialog__footer', getThemeBlockCSS(styles.dialogFooter));
+    css += wrapStyles(name, '.theme--dialog', getThemeBlockCSS(styles.dialog));
+    css += wrapStyles(name, '.theme--dialog__header', getThemeBlockCSS(styles.dialogHeader));
+    css += wrapStyles(name, '.theme--dialog__footer', getThemeBlockCSS(styles.dialogFooter));
     // Panes
-    css += wrapStyles(n, '.theme--pane', getThemeBlockCSS(styles.pane));
-    css += wrapStyles(n, '.theme--pane__header', getThemeBlockCSS(styles.paneHeader));
-    // @ts-expect-error -- TSCONVERSION
-    css += wrapStyles(n, '.theme--app-header', getThemeBlockCSS(styles.appHeader));
+    css += wrapStyles(name, '.theme--pane', getThemeBlockCSS(styles.pane));
+    css += wrapStyles(name, '.theme--pane__header', getThemeBlockCSS(styles.paneHeader));
+    css += wrapStyles(name, '.theme--app-header', getThemeBlockCSS(styles.appHeader));
     // Sidebar Styles
-    css += wrapStyles(n, '.theme--sidebar', getThemeBlockCSS(styles.sidebar));
-    css += wrapStyles(n, '.theme--sidebar__list', getThemeBlockCSS(styles.sidebarList));
-    css += wrapStyles(n, '.theme--sidebar__header', getThemeBlockCSS(styles.sidebarHeader));
+    css += wrapStyles(name, '.theme--sidebar', getThemeBlockCSS(styles.sidebar));
+    css += wrapStyles(name, '.theme--sidebar__list', getThemeBlockCSS(styles.sidebarList));
+    css += wrapStyles(name, '.theme--sidebar__header', getThemeBlockCSS(styles.sidebarHeader));
     // Link
-    css += wrapStyles(n, '.theme--link', getThemeBlockCSS(styles.link));
+    css += wrapStyles(name, '.theme--link', getThemeBlockCSS(styles.link));
     // Code Editors
-    css += wrapStyles(n, '.theme--editor', getThemeBlockCSS(styles.editor));
+    css += wrapStyles(name, '.theme--editor', getThemeBlockCSS(styles.editor));
     // HACK: Dialog styles for CodeMirror dialogs too
-    css += wrapStyles(n, '.CodeMirror-info', getThemeBlockCSS(styles.dialog));
+    css += wrapStyles(name, '.CodeMirror-info', getThemeBlockCSS(styles.dialog));
   }
 
+  css += '\n';
   return css;
-}
+};
 
 function getThemeBlockCSS(block?: ThemeBlock) {
   if (!block) {
@@ -298,7 +350,7 @@ export async function setTheme(themeName: string) {
   body.setAttribute('theme', themeName);
 
   for (const theme of themes) {
-    let themeCSS = (await generateThemeCSS(theme.theme)) + '\n';
+    let themeCSS = generateThemeCSS(theme.theme);
     const { name } = theme.theme;
     const { rawCss } = theme.theme.theme;
     let s = document.querySelector(`style[data-theme-name="${name}"]`);
@@ -317,7 +369,7 @@ export async function setTheme(themeName: string) {
   }
 }
 
-const _baseTheme = {
+export const baseTheme: CompleteStyleBlock = {
   background: {
     default: '#fff',
     success: '#75ba24',

plugins/insomnia-plugin-core-themes/themes/default.js

@@ -1,3 +1,13 @@
+const sidebarBackground = {
+  default: '#2C2C2C',
+  success: '#7ecf2b',
+  notice: '#f0e137',
+  warning: '#ff9a1f',
+  danger: '#ff5631',
+  surprise: '#a896ff',
+  info: '#46c1e6',
+};
+
 module.exports = {
   name: 'default',
   displayName: 'Core Default',
@@ -37,15 +47,7 @@ module.exports = {
         },
       },
       sidebar: {
-        background: {
-          default: '#2C2C2C',
-          success: '#7ecf2b',
-          notice: '#f0e137',
-          warning: '#ff9a1f',
-          danger: '#ff5631',
-          surprise: '#a896ff',
-          info: '#46c1e6',
-        },
+        background: sidebarBackground,
         foreground: {
           default: '#e0e0e0',
         },
@@ -77,13 +79,8 @@ module.exports = {
       },
       pane: {
         background: {
+          ...sidebarBackground,
           default: '#292929',
-          success: '{{ styles.sidebar.background.success }}',
-          notice: '{{ styles.sidebar.background.notice }}',
-          warning: '{{ styles.sidebar.background.warning }}',
-          danger: '{{ styles.sidebar.background.danger }}',
-          surprise: '{{ styles.sidebar.background.surprise }}',
-          info: '{{ styles.sidebar.background.info }}',
         },
         foreground: {
           default: '#e0e0e0',

plugins/insomnia-plugin-core-themes/themes/one-light.js

@@ -1,3 +1,13 @@
+const sidebarBackground = {
+  default: '#eaeaeb',
+  success: '#50a14f',
+  notice: '#c18401',
+  warning: '#c18401',
+  danger: '#e45649',
+  surprise: '#a626a4',
+  info: '#0184bc',
+};
+
 module.exports = {
   name: 'one-light',
   displayName: 'One Light',
@@ -30,29 +40,14 @@ module.exports = {
         },
       },
       sidebar: {
-        background: {
-          default: '#eaeaeb',
-          success: '#50a14f',
-          notice: '#c18401',
-          warning: '#c18401',
-          danger: '#e45649',
-          surprise: '#a626a4',
-          info: '#0184bc',
-        },
+        background: sidebarBackground,
         foreground: {
           default: '#444',
         },
         highlight: {},
       },
       paneHeader: {
-        background: {
-          success: '{{ styles.sidebar.background.success }}',
-          notice: '{{ styles.sidebar.background.notice }}',
-          warning: '{{ styles.sidebar.background.warning }}',
-          danger: '{{ styles.sidebar.background.danger }}',
-          surprise: '{{ styles.sidebar.background.surprise }}',
-          info: '{{ styles.sidebar.background.info }}',
-        },
+        background: sidebarBackground,
       },
       transparentOverlay: {
         background: {

plugins/insomnia-plugin-core-themes/themes/studio-light.js

@@ -1,3 +1,12 @@
+const background = {
+  success: '#3d9c62',
+  notice: '#bb9700',
+  warning: '#d6803e',
+  danger: '#da5b56',
+  info: '#003052',
+  surprise: '#6030BF',
+};
+
 module.exports = {
   name: 'studio-light',
   displayName: 'Designer Light',
@@ -5,14 +14,7 @@ module.exports = {
     foreground: {
       default: '#555',
     },
-    background: {
-      success: '#3d9c62',
-      notice: '#bb9700',
-      warning: '#d6803e',
-      danger: '#da5b56',
-      info: '#003052',
-      surprise: '#6030BF',
-    },
+    background,
     styles: {
       appHeader: {
         background: {
@@ -26,8 +28,8 @@ module.exports = {
       },
       editor: {
         background: {
-          surprise: '{{ background.info }}',
-          info: '{{ background.surprise }}',
+          surprise: background.info,
+          info: background.surprise,
         },
       },
       dialog: {

scripts/download-all-npm-plugins.ts

@@ -0,0 +1,59 @@
+import https from 'https';
+import { statSync, mkdirSync, writeFileSync } from 'fs';
+import { execSync } from 'child_process';
+
+/*
+
+This simple (and zero dependency) script's purpose in life is to help the Insomnia team answer questions like "I wonder how many plugins use <XYZ>".
+In short, it uses the NPM search API and grabs all plugins and puts them into a temporary npm package, the thought being that you can then open up your editor on that temporary package's `node_modules` directory and be able to search the code of all plugins.
+
+Using NPM in this way is a little less error-prone than, say, if we were to clone each repo from GitHub, because what's on GitHub may not match what's on NPM -> and NPM is what the app uses.
+
+IMPORTANT:
+For this script to really work well for finding all plugins, it needs to handle multiple page sizes on the npm search API since there are more plugins on npm than the max page size allows.
+However, since the initial use-case for this script is for theme plugins (which there are only 36 of, as of 20220706), multiple pages are not handled.
+
+*/
+
+const npmSearchText = 'insomnia-plugin-theme';
+
+/** the default is 20 and the max is 250 */
+const pageSize = 250;
+
+/** https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#get-v1search */
+const npmSearchUrl = `https://registry.npmjs.org/-/v1/search?text=${npmSearchText}&size=${pageSize}`;
+
+const request = https.get(npmSearchUrl, response => {
+  let data: Uint8Array[] = [];
+  response.on('data', (chunk: Uint8Array) => {
+    data.push(chunk);
+  });
+  response.on('error', console.error);
+  response.on('end', () => {
+    const unparsed = Buffer.concat(data).toString();
+    const { objects } = JSON.parse(unparsed);
+    const names: string[] = objects.map(result => result.package.name);
+    console.log(names);
+
+    const directory = './scripts/npm-plugins';
+    try {
+      statSync(directory);
+    } catch (error) {
+      mkdirSync(directory);
+    }
+
+    const packageJson = {
+      name: 'npm-plugins',
+      dependencies: names.reduce((accumulator, npm) => ({
+        ...accumulator,
+        [npm]: '*',
+      }), {}),
+    };
+    writeFileSync(`${directory}/package.json`, JSON.stringify(packageJson, null, 2));
+    execSync(`npm install --prefix ${directory}`);
+  });
+});
+
+request.on('error', console.error);
+
+request.end();