feat: make single file upload to attachment available (#46)

* feat: make single file upload to attachment available

* fix: change file name case

* feat: refactor structure and make local server work

* test: skip bug case temporarily

* fix: use middleware to load storage static server

* fix: change meta from values to request.body back and refactor local server middleware

* adjust details

* http:

Co-authored-by: chenos <chenlinxh@gmail.com>
This commit is contained in:
Junyi 2020-12-23 12:46:13 +08:00 committed by GitHub
parent e6dfcf8418
commit eb7d5c594e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
26 changed files with 486 additions and 96 deletions

View File

@ -17,3 +17,5 @@ DB_POSTGRES_PORT=5432
HTTP_PORT=23000
VERDACCIO_PORT=4873
LOCAL_STORAGE_BASE_URL=http://localhost:23000/uploads

1
.gitignore vendored
View File

@ -10,3 +10,4 @@ packages/database/package-lock.json
packages/resourcer/package-lock.json
verdaccio
uploads/

View File

@ -14,3 +14,5 @@ DB_POSTGRES_PORT=5432
HTTP_PORT=23000
VERDACCIO_PORT=4873
LOCAL_STORAGE_BASE_URL=http://localhost:23000/uploads

View File

@ -115,6 +115,7 @@ api.resourcer.registerActionHandlers({...actions.common, ...actions.associate});
api.registerPlugin('plugin-collections', [path.resolve(__dirname, '../../../plugin-collections'), {}]);
api.registerPlugin('plugin-pages', [path.resolve(__dirname, '../../../plugin-pages'), {}]);
api.registerPlugin('plugin-users', [path.resolve(__dirname, '../../../plugin-users'), {}]);
api.registerPlugin('plugin-file-manager', [path.resolve(__dirname, '../../../plugin-file-manager'), {}]);
(async () => {
await api.loadPlugins();

View File

@ -160,6 +160,7 @@ const data = [
api.registerPlugin('plugin-collections', [path.resolve(__dirname, '../../../plugin-collections'), {}]);
api.registerPlugin('plugin-pages', [path.resolve(__dirname, '../../../plugin-pages'), {}]);
api.registerPlugin('plugin-users', [path.resolve(__dirname, '../../../plugin-users'), {}]);
api.registerPlugin('plugin-file-manager', [path.resolve(__dirname, '../../../plugin-file-manager'), {}]);
(async () => {
await api.loadPlugins();
@ -186,6 +187,14 @@ api.registerPlugin('plugin-users', [path.resolve(__dirname, '../../../plugin-use
token: "38979f07e1fca68fb3d2",
});
}
const Storage = database.getModel('storages');
await Storage.create({
title: '本地存储',
name: `local`,
type: 'local',
baseUrl: process.env.LOCAL_STORAGE_BASE_URL,
default: true
});
await database.getModel('collections').import(require('./collections/example').default);
await database.close();
})();

View File

@ -7,6 +7,11 @@
"@koa/multer": "^3.0.0",
"@nocobase/database": "^0.3.0-alpha.0",
"@nocobase/resourcer": "^0.3.0-alpha.0",
"@types/multer": "^1.4.5",
"koa-mount": "^4.0.0",
"koa-static": "^5.0.0",
"mime-match": "^1.0.2",
"mkdirp": "^1.0.4",
"multer": "^1.4.2"
},
"devDependencies": {

View File

@ -1,29 +1,147 @@
import { promises as fs } from 'fs';
import path from 'path';
import { FILE_FIELD_NAME } from '../constants';
import { getApp, getAgent } from '.';
import qs from 'qs';
import { FILE_FIELD_NAME, STORAGE_TYPE_LOCAL } from '../constants';
import { getApp, getAgent, getAPI } from '.';
describe('user fields', () => {
const DEFAULT_LOCAL_BASE_URL = process.env.LOCAL_STORAGE_BASE_URL || `http://localhost:${process.env.HTTP_PORT}/uploads`;
jest.setTimeout(30000);
describe('action', () => {
let app;
let agent;
let api;
let db;
beforeEach(async () => {
app = await getApp();
agent = getAgent(app);
api = getAPI(app);
db = app.database;
await db.sync({ force: true });
const Storage = app.database.getModel('storages');
await Storage.create({
name: `local_${Date.now().toString(36)}`,
type: STORAGE_TYPE_LOCAL,
baseUrl: DEFAULT_LOCAL_BASE_URL,
default: true
});
});
afterEach(async () => {
await db.close();
});
afterEach(() => db.close());
describe('', () => {
it('', async () => {
describe('direct attachment', () => {
it('upload file should be ok', async () => {
const response = await agent
.post('/api/attachments:upload')
.attach(FILE_FIELD_NAME, path.resolve(__dirname, './files/text.txt'));
console.log(response.body);
const matcher = {
title: 'text',
extname: '.txt',
path: '',
size: 13,
mimetype: 'text/plain',
meta: {},
storage_id: 1,
};
// 文件上传和解析是否正常
expect(response.body).toMatchObject(matcher);
// 文件的 url 是否正常生成
expect(response.body.url).toBe(`${DEFAULT_LOCAL_BASE_URL}${response.body.path}/${response.body.filename}`);
const Attachment = db.getModel('attachments');
const attachment = await Attachment.findOne({
where: { id: response.body.id },
include: ['storage']
});
const storage = attachment.get('storage');
// 文件的数据是否正常保存
expect(attachment).toMatchObject(matcher);
// 关联的存储引擎是否正确
expect(storage).toMatchObject({
type: 'local',
options: {},
rules: {},
path: '',
baseUrl: DEFAULT_LOCAL_BASE_URL,
default: true,
});
const { documentRoot = 'uploads' } = storage.options || {};
const destPath = path.resolve(path.isAbsolute(documentRoot) ? documentRoot : path.join(process.env.PWD, documentRoot), storage.path);
const file = await fs.readFile(`${destPath}/${attachment.filename}`);
// 文件是否保存到指定路径
expect(file.toString()).toBe('Hello world!\n');
const content = await agent.get(`/uploads${attachment.path}/${attachment.filename}`);
// 通过 url 是否能正确访问
expect(content.text).toBe('Hello world!\n');
});
});
describe('belongsTo attachment', () => {
it('upload with associatedKey, fail as 400 because file mimetype does not match', async () => {
const User = db.getModel('users');
const user = await User.create();
const response = await api.resource('users.avatar').upload({
associatedKey: user.id,
filePath: './files/text.txt'
});
expect(response.status).toBe(400);
});
it('upload with associatedKey', async () => {
const User = db.getModel('users');
const user = await User.create();
const response = await api.resource('users.avatar').upload({
associatedKey: user.id,
filePath: './files/image.png',
values: { width: 100, height: 100 }
});
const matcher = {
title: 'image',
extname: '.png',
path: '',
size: 255,
mimetype: 'image/png',
// TODO(optimize): 可以考虑使用 qs 的 decoder 来进行类型解析
// see: https://github.com/ljharb/qs/issues/91
// 或考虑使用 query-string 库的 parseNumbers 等配置项
meta: { width: '100', height: '100' },
storage_id: 1,
};
// 上传正常返回
expect(response.body).toMatchObject(matcher);
// 由于初始没有外键,无法获取
// await user.getAvatar()
const updatedUser = await User.findByPk(user.id, {
include: ['avatar']
});
// 外键更新正常
expect(updatedUser.get('avatar').id).toBe(response.body.id);
});
// TODO(bug): 没有 associatedKey 时路径解析资源名称不对,无法进入 action
it.skip('upload without associatedKey', async () => {
const response = await api.resource('users.avatar').upload({
filePath: './files/image.png',
values: { width: 100, height: 100 }
});
const matcher = {
title: 'image',
extname: '.png',
path: '',
size: 255,
mimetype: 'image/png',
// TODO(optimize): 可以考虑使用 qs 的 decoder 来进行类型解析
// see: https://github.com/ljharb/qs/issues/91
// 或考虑使用 query-string 库的 parseNumbers 等配置项
meta: { width: '100', height: '100' },
storage_id: 1,
};
// 上传返回正常
expect(response.body).toMatchObject(matcher);
});
});
});

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.0 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 255 B

View File

@ -5,8 +5,10 @@ import bodyParser from 'koa-bodyparser';
import { Dialect } from 'sequelize';
import Database from '@nocobase/database';
import { actions, middlewares } from '@nocobase/actions';
import { Application, middleware } from '@nocobase/server';
import { Application } from '@nocobase/server';
import middleware from '@nocobase/server/src/middleware'
import plugin from '../server';
import { FILE_FIELD_NAME } from '../constants';
function getTestKey() {
const { id } = require.main;
@ -65,9 +67,14 @@ export async function getApp() {
'file-manager': [plugin]
});
await app.loadPlugins();
await app.database.sync({
force: true,
app.database.import({
directory: path.resolve(__dirname, './tables')
});
try {
await app.database.sync();
} catch (error) {
console.error(error);
}
app.use(async (ctx, next) => {
ctx.db = app.database;
await next();
@ -114,9 +121,9 @@ export function getAPI(app: Application) {
return {
resource(name: string): any {
return new Proxy({}, {
get(target, method, receiver) {
get(target, method: string, receiver) {
return (params: ActionParams = {}) => {
const { associatedKey, resourceKey, values = {}, ...restParams } = params;
const { associatedKey, resourceKey, values = {}, filePath, ...restParams } = params;
let url = `/api/${name}`;
if (associatedKey) {
url = `/api/${name.split('.').join(`/${associatedKey}/`)}`;
@ -125,10 +132,19 @@ export function getAPI(app: Application) {
if (resourceKey) {
url += `/${resourceKey}`;
}
if (['list', 'get'].indexOf(method as string) !== -1) {
return agent.get(`${url}?${qs.stringify(restParams)}`);
} else {
return agent.post(`${url}?${qs.stringify(restParams)}`).send(values);
switch (method) {
case 'upload':
return agent.post(`${url}?${qs.stringify(restParams)}`)
.attach(FILE_FIELD_NAME, path.resolve(__dirname, filePath))
.field(values);
case 'list':
case 'get':
return agent.get(`${url}?${qs.stringify(restParams)}`);
default:
return agent.post(`${url}?${qs.stringify(restParams)}`).send(values);
}
}
}

View File

@ -0,0 +1,34 @@
import { TableOptions } from "@nocobase/database";
export default {
name: 'users',
fields: [
{
type: 'string',
name: 'name',
},
{
type: 'belongsTo',
name: 'avatar',
target: 'attachments',
attachment: {
// storage 为配置的默认引擎
rules: {
size: 1024 * 10,
mimetype: ['image/png']
}
}
},
{
type: 'belongsToMany',
name: 'photos',
target: 'attachments',
attachment: {
rules: {
size: 1024 * 100,
mimetype: ['image/*']
}
}
},
],
} as TableOptions;

View File

@ -1,23 +1,120 @@
import path from 'path';
import multer from '@koa/multer';
import actions from '@nocobase/actions';
import storageMakers from '../storages';
import * as Rules from '../rules';
import { FILE_FIELD_NAME, LIMIT_FILES, LIMIT_MAX_FILE_SIZE } from '../constants';
import { FILE_FIELD_NAME } from '../constants';
function getRules(ctx: actions.Context) {
const { resourceField } = ctx.action.params;
if (!resourceField) {
return ctx.storage.rules;
}
const { rules = {} } = resourceField.getOptions().attachment || {};
return Object.assign({}, ctx.storage.rules, rules);
}
// TODO(optimize): 需要优化错误处理,计算失败后需要抛出对应错误,以便程序处理
function getFileFilter(ctx: actions.Context) {
return (req, file, cb) => {
// size 交给 limits 处理
const { size, ...rules } = getRules(ctx);
const ruleKeys = Object.keys(rules);
const result = !ruleKeys.length || !ruleKeys
.some(key => typeof Rules[key] !== 'function'
|| !Rules[key](file, rules[key], ctx));
cb(null, result);
}
}
export async function middleware(ctx: actions.Context, next: actions.Next) {
const { actionName } = ctx.action.params;
const { resourceName, actionName, resourceField } = ctx.action.params;
if (actionName !== 'upload') {
return next();
}
const options = {};
const upload = multer(options);
// NOTE:
// 1. 存储引擎选择依赖于字段定义
// 2. 字段定义中需包含引擎的外键值
// 3. 无字段时按 storages 表的默认项
// 4. 插件初始化后应提示用户添加至少一个存储引擎并设为默认
const StorageModel = ctx.db.getModel('storages');
let storage;
if (resourceName === 'attachments') {
// 如果没有包含关联,则直接按默认文件上传至默认存储引擎
storage = await StorageModel.findOne({ where: { default: true } });
} else {
const fieldOptions = resourceField.getOptions();
storage = await StorageModel.findOne({
where: fieldOptions.defaultValue
? { [StorageModel.primaryKeyAttribute]: fieldOptions.defaultValue }
: { default: true }
});
}
if (!storage) {
console.error('[file-manager] no default or linked storage provided');
return ctx.throw(500);
}
// 传递已取得的存储引擎,避免重查
ctx.storage = storage;
const makeStorage = storageMakers.get(storage.type);
if (!makeStorage) {
console.error(`[file-manager] storage type "${storage.type}" is not defined`);
return ctx.throw(500);
}
const multerOptions = {
fileFilter: getFileFilter(ctx),
limits: {
fileSize: Math.min(getRules(ctx).size || LIMIT_MAX_FILE_SIZE, LIMIT_MAX_FILE_SIZE),
// 每次只允许提交一个文件
files: LIMIT_FILES
},
storage: makeStorage(storage),
};
const upload = multer(multerOptions);
return upload.single(FILE_FIELD_NAME)(ctx, next);
};
export async function action(ctx: actions.Context, next: actions.Next) {
const { [FILE_FIELD_NAME]: file } = ctx;
console.log(file);
ctx.body = file;
const { [FILE_FIELD_NAME]: file, storage } = ctx;
if (!file) {
return ctx.throw(400, 'file validation failed');
}
const { associatedName, associatedKey, resourceField } = ctx.action.params;
const extname = path.extname(file.filename);
const data = {
title: file.originalname.replace(extname, ''),
filename: file.filename,
extname,
// TODO(feature): 暂时两者相同,后面 storage.path 模版化以后,这里只是 file 实际的 path
path: storage.path,
size: file.size,
mimetype: file.mimetype,
// @ts-ignore
meta: ctx.request.body
}
const attachment = await ctx.db.sequelize.transaction(async transaction => {
// TODO(optimize): 应使用关联 accessors 获取
const result = await storage.createAttachment(data, { transaction });
if (associatedKey && resourceField) {
const Attachment = ctx.db.getModel('attachments');
const SourceModel = ctx.db.getModel(associatedName);
const source = await SourceModel.findByPk(associatedKey, { transaction });
await source[resourceField.getAccessors().set](result[Attachment.primaryKeyAttribute], { transaction });
}
return result;
});
// 将存储引擎的信息附在已创建的记录里,节省一次查询
attachment.setDataValue('storage', storage);
ctx.body = attachment;
await next();
};

View File

@ -6,15 +6,14 @@ export default {
internal: true,
fields: [
{
comment: '唯一 ID系统文件名',
type: 'uuid',
name: 'id',
primaryKey: true
comment: '用户文件名(不含扩展名)',
type: 'string',
name: 'title',
},
{
comment: '用户文件名',
comment: '系统文件名(含扩展名)',
type: 'string',
name: 'name',
name: 'filename'
},
{
comment: '扩展名(含“.”)',
@ -26,11 +25,12 @@ export default {
type: 'integer',
name: 'size',
},
{
comment: '文件类型mimetype 前半段,通常用于预览)',
type: 'string',
name: 'type',
},
// TODO: 使用暂不明确,以后再考虑
// {
// comment: '文件类型mimetype 前半段,通常用于预览)',
// type: 'string',
// name: 'type',
// },
{
type: 'string',
name: 'mimetype',
@ -41,7 +41,7 @@ export default {
name: 'storage',
},
{
comment: '相对路径',
comment: '相对路径(含“/”前缀)',
type: 'string',
name: 'path',
},
@ -49,11 +49,13 @@ export default {
comment: '其他文件信息(如图片的宽高)',
type: 'jsonb',
name: 'meta',
defaultValue: {}
},
{
comment: '网络访问地址',
type: 'url',
name: 'url'
type: 'formula',
name: 'url',
formula: '{{ storage.baseUrl }}{{ path }}/{{ filename }}'
}
],
actions: [

View File

@ -6,15 +6,22 @@ export default {
internal: true,
fields: [
{
comment: '标识名称,用于用户记忆',
title: '存储引擎名称',
comment: '存储引擎名称',
type: 'string',
name: 'title',
},
{
title: '英文标识',
// comment: '英文标识,用于代码层面配置',
type: 'string',
name: 'name',
unique: true,
},
{
comment: '类型标识,如 local/ali-oss 等',
type: 'string',
name: 'type',
defaultValue: 'local'
},
{
comment: '配置项',
@ -22,6 +29,12 @@ export default {
name: 'options',
defaultValue: {}
},
{
comment: '文件规则',
type: 'jsonb',
name: 'rules',
defaultValue: {}
},
{
comment: '存储相对路径模板',
type: 'string',
@ -34,5 +47,16 @@ export default {
name: 'baseUrl',
defaultValue: ''
},
// TODO(feature): 需要使用一个实现了可设置默认值的字段
{
comment: '默认引擎',
type: 'boolean',
name: 'default',
defaultValue: false
},
{
type: 'hasMany',
name: 'attachments'
}
]
} as TableOptions;

View File

@ -1 +1,6 @@
export const FILE_FIELD_NAME = 'file';
export const LIMIT_FILES = 1;
export const LIMIT_MAX_FILE_SIZE = 1024 * 1024 * 1024;
export const STORAGE_TYPE_LOCAL = 'local';
export const STORAGE_TYPE_ALI_OSS = 'ali-oss';

View File

@ -0,0 +1,19 @@
import { BELONGSTO, BelongsToOptions, FieldContext } from '@nocobase/database';
export interface URLOptions extends Omit<BelongsToOptions, 'type'> {
type: 'file';
}
export default class File extends BELONGSTO {
constructor({ type, ...options }, context: FieldContext) {
// const { multiple = false } = options;
super({
...options,
type: 'belongsTo',
} as BelongsToOptions, context);
}
public getDataType(): Function {
return BELONGSTO;
}
}

View File

@ -1,20 +0,0 @@
import { DataTypes } from 'sequelize';
import { VIRTUAL, VirtualOptions, FieldContext } from '@nocobase/database';
export interface URLOptions extends Omit<VirtualOptions, 'type'> {
type: 'url';
}
export default class URL extends VIRTUAL {
constructor({ type, ...options }, context: FieldContext) {
super({
...options,
type: 'virtual',
get() {
const storage = this.getDataValue('storage') || {};
return `${storage.baseUrl}${this.getDataValue('path')}/${this.getDataValue('id')}${this.getDataValue('extname')}`;
}
} as VirtualOptions, context);
}
}

View File

@ -1 +1 @@
export { default as URL } from './URL';
export { default as File } from './File';

View File

@ -0,0 +1,3 @@
export { default as mimetype } from './mimetype';
// TODO(feature): 提供注册新规则的方法,规则可动态添加

View File

@ -0,0 +1,5 @@
import match from 'mime-match';
export default function(file, options: string | string[] = '*', ctx): boolean {
return options.toString().split(',').some(match(file.mimetype));
}

View File

@ -1,24 +1,19 @@
import path from 'path';
import Database, { registerFields } from '@nocobase/database';
import Database from '@nocobase/database';
import Resourcer from '@nocobase/resourcer';
import * as fields from './fields';
import { IStorage } from './storages';
import {
action as uploadAction,
middleware as uploadMiddleware,
} from './actions/upload';
import {
middleware as localMiddleware,
} from './storages/local';
export interface FileManagerOptions {
storages: IStorage[]
}
export default async function (options: FileManagerOptions) {
export default async function () {
const database: Database = this.database;
const resourcer: Resourcer = this.resourcer;
registerFields(fields);
database.import({
directory: path.resolve(__dirname, 'collections'),
});
@ -26,4 +21,5 @@ export default async function (options: FileManagerOptions) {
// 暂时中间件只能通过 use 加进来
resourcer.use(uploadMiddleware);
resourcer.registerActionHandler('upload', uploadAction);
localMiddleware(this);
}

View File

@ -1,5 +0,0 @@
export default interface IStorage {
options: any;
put: (file, data) => Promise<any>
}

View File

@ -1,13 +0,0 @@
import IStorage from './IStorage';
export default class Local implements IStorage {
options: any;
constructor(options: any) {
this.options = options;
}
async put(file, data) {
}
}

View File

@ -1,2 +1,9 @@
export { default as IStorage } from './IStorage';
export { default as Local } from './Local';
import local from './local';
import { STORAGE_TYPE_LOCAL, STORAGE_TYPE_ALI_OSS } from '../constants';
const map = new Map<string, Function>();
map.set(STORAGE_TYPE_LOCAL, local);
export default map;

View File

@ -0,0 +1,82 @@
import crypto from 'crypto';
import path from 'path';
import { URL } from 'url';
import mkdirp from 'mkdirp';
import multer from 'multer';
import serve from 'koa-static';
import mount from 'koa-mount';
import { STORAGE_TYPE_LOCAL } from '../constants';
export function getDocumentRoot(storage): string {
const { documentRoot = 'uploads' } = storage.options || {};
// TODO(feature): 后面考虑以字符串模板的方式使用,可注入 req/action 相关变量,以便于区分文件夹
return path.resolve(path.isAbsolute(documentRoot)
? documentRoot
: path.join(process.env.PWD, documentRoot), storage.path);
}
// TODO(optimize): 初始化的时机不应该放在中间件里
export function middleware(app) {
const storages = new Map<string, any>();
const StorageModel = app.database.getModel('storages');
return app.use(async function(ctx, next) {
const items = await StorageModel.findAll({
where: {
type: STORAGE_TYPE_LOCAL,
}
});
const primaryKey = StorageModel.primaryKeyAttribute;
for (const storage of items) {
// TODO未解决 storage 更新问题
if (storages.has(storage[primaryKey])) {
continue;
}
const baseUrl = storage.get('baseUrl');
let url;
try {
url = new URL(baseUrl);
} catch (e) {
url = {
protocol: 'http:',
hostname: 'localhost',
port: process.env.HTTP_PORT,
pathname: baseUrl
};
}
// 以下情况才认为当前进程所应该提供静态服务
// 否则都忽略,交给其他 server 来提供(如 nginx/cdn 等)
// TODO(bug): https、端口 80 默认值和其他本地 ip/hostname 的情况未考虑
// TODO 实际应该用 NOCOBASE_ENV 来判断,或者抛给 env 处理
if (url.protocol === 'http:'
&& url.hostname === 'localhost'
&& url.port === process.env.HTTP_PORT
) {
const basePath = url.pathname.startsWith('/') ? url.pathname : `/${url.pathname}`;
app.use(mount(basePath, serve(getDocumentRoot(storage))));
}
storages.set(storage.primaryKey, storage);
}
await next();
});
}
export default (storage) => multer.diskStorage({
destination: function (req, file, cb) {
const destPath = getDocumentRoot(storage);
mkdirp(destPath).then(() => {
cb(null, destPath);
}).catch(cb);
},
filename: function (req, file, cb) {
crypto.randomBytes(16, (err, raw) => {
cb(err, err ? undefined : `${raw.toString('hex')}${path.extname(file.originalname)}`)
});
}
});

View File

@ -59,7 +59,7 @@ export async function getApp() {
app.resourcer.use(middlewares.associated);
app.resourcer.registerActionHandlers({...actions.associate, ...actions.common});
app.registerPlugin('collections', [path.resolve(__dirname, '../../../plugin-collections')]);
app.registerPlugin('users', [plugin]);
app.registerPlugin('file-manager', [plugin]);
await app.loadPlugins();
await app.database.sync({
force: true,