30
Streaming downloads proxy service with Node.js ismael celis @ismasan

Node.js streaming csv downloads proxy

Embed Size (px)

DESCRIPTION

Small Node.js proxy to turn a paginated JSON REST API into a CSV streaming download. Examples of code and patterns. Presented at the London Node User Group meetup, April 2014

Citation preview

Page 1: Node.js streaming csv downloads proxy

Streaming downloads proxy service with

Node.js

ismael celis @ismasan

Page 2: Node.js streaming csv downloads proxy

bootic.net - Hosted e-commerce in South America

Page 3: Node.js streaming csv downloads proxy

background job Email

attachment

Previous setup

Page 4: Node.js streaming csv downloads proxy

Previous setup

Page 5: Node.js streaming csv downloads proxy

Previous setup

• Memory limitations • Email deliverability • Code bloat • inflexible

Page 6: Node.js streaming csv downloads proxy

New setup

• Monolithic micro • Leverage existing API

Page 7: Node.js streaming csv downloads proxy

New setup

• Monolithic micro • Leverage existing API

curl -H “Authorization: Bearer xxx” \https://api.bootic.net/v1/orders.json?created_at:gte=2014-02-01&page=2

Page 8: Node.js streaming csv downloads proxy
Page 9: Node.js streaming csv downloads proxy

New setup

Page 10: Node.js streaming csv downloads proxy

API -> CSV Stream

// pipe generated CSV onto the HTTP responsevar writer = csv.createCsvStreamWriter(response)// Turn a series of paginated requests // to the backend API into a stream of datavar stream = apistream.instance(uri, token)// Pipe data stream into CSV writerstream.pipe(writer)

Page 11: Node.js streaming csv downloads proxy

API -> CSV Stream

response.setHeader('Content-Type', ‘text/csv'); response.setHeader('Content-disposition', 'attachment;filename=' + name + '.csv');

Page 12: Node.js streaming csv downloads proxy

API -> mappers -> CSV Stream

{ "code": "123EFCD", "total": 80000, "status": "shipped", "date": "2014-02-03", "items": [ {"product_title": "iPhone 5", "units": 2, "unit_price": 30000}, {"product_title": "Samsung Galaxy S4", "units": 1, "unit_price": 20000} ]}

code, total, date, status, product, units, unit_price, total2 123EFCD, 80000, 2014-02-03, shipped, iPhone 5, 2, 30000, 800003 123EFCD, 80000, 2014-02-03, shipped, Samsung Galaxy S4, 1, 20000, 80000

Page 13: Node.js streaming csv downloads proxy

API -> mappers -> CSV Stream

var OrderMapper = csvmapper.define(function () { this.scope('items', function () { this .map('id', '/id') .map('order', '/code') .map('status', '/status') .map('discount', '/discount_total') .map('shipping price', '/shipping_total') .map('total', '/total') .map('year', '/updated_on', year) .map('month', '/updated_on', month) .map('day', '/updated_on', day) .map('payment method', '/payment_method_type') .map('name', '/contact/name') .map('email', '/contact/email') .map('address', '/address', address) .map('product', 'product_title') .map('variant', 'variant_title') .map('sku', 'variant_sku') .map('unit price', 'unit_price') .map('quantity', 'units')

Page 14: Node.js streaming csv downloads proxy

API -> mappers -> CSV Stream

var writer = csv.createCsvStreamWriter(res);var stream = apistream.instance(uri, token)var mapper = new OrdersMapper()// First line in CSV is the headerswriter.writeRecord(mapper.headers())// mapper.eachRow() turns a single API resource into 1 or more CSV rowsstream.on('item', function (item) { mapper.eachRow(item, function (row) { writer.writeRecord(row) })})

Page 15: Node.js streaming csv downloads proxy

API -> mappers -> CSV Stream

stream.on('item', function (item) { mapper.eachRow(item, function (row) { writer.writeRecord(row) })})

Page 16: Node.js streaming csv downloads proxy

Paremeter definitions

https://api.bootic.net/v1/orders.json? created_at:gte=2014-02-01 & page=2

Page 17: Node.js streaming csv downloads proxy

Paremeter definitions

var OrdersParams = params.define(function () { this .param('sort', 'updated_on:desc') .param('per_page', 20) .param('status', 'closed,pending,invalid,shipped') })

Page 18: Node.js streaming csv downloads proxy

Paremeter definitions

var params = new OrdersParams(request.query)// Compose API url using sanitized / defaulted paramsvar uri = "https://api.com/orders?" + params.query;var stream = apistream.instance(uri, token)

Page 19: Node.js streaming csv downloads proxy

Secure CSV downloads

Page 20: Node.js streaming csv downloads proxy

JSON Web Tokens

headers . claims . signature

http://self-issued.info/docs/draft-ietf-oauth-json-web-token.html

eyJ0eXAiOiJKV1QiLA0KICJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJqb2UiLA0KICJleHAiOjEzMDA4MTkzODAsDQogImh0dHA6Ly9leGFtcGxlLmNvbS9pc19yb290Ijp0cnVlfQ.dBjftJeZ4CVP-mB92K27uhbUJU1p1r_wW1gFWFOEjXk

Page 21: Node.js streaming csv downloads proxy

JSON Web Tokens

headers {“typ":"JWT", "alg":"HS256"}

claims{ “shop_id":"acme", "iat":1300819380, "aud":"orders", "filters": {"status": "shipped"} }

signature

+ Base64

+ Base64

HMAC SHA-256 (headers + claims, secret) + Base64

Page 22: Node.js streaming csv downloads proxy

Rails: Generate token (Ruby)

# controllers/downloads_controller.rbdef create url = Rails.application.config.downloads_host claims = params[:download_options] # Add an issued_at timestamp claims[:iat] = (Time.now.getutc.to_f * 1000).to_i # Scope data on current account claims[“shop_id"] = current_shop.id # generate JWT token = JWT.encode(claims, Rails.application.config.downloads_secret) # Redirect to download URL. Browser will trigger download dialog redirect_to “#{url}?jwt=#{token}" end

Page 23: Node.js streaming csv downloads proxy

Rails: Generate token (Ruby)

claims[:iat] = (Time.now.getutc.to_f * 1000).to_iclaims[“shop_id"] = current_shop.idtoken = JWT.encode(claims, secret)redirect_to "#{url}?jwt=#{token}"

Page 24: Node.js streaming csv downloads proxy

Node: validate JWT

var TTL = 60000;var tokenMiddleware = function(req, res, next){ try{ var decoded = jwt.decode(req.query.jwt, secret); if(decoded.shop_id != req.param(‘shop_id') { res.send(400, ‘JWT and query shop ids do not match'); return } var now = new Date(), utc = getUtcCurrentDate(); if(utc - Number(decoded.iat) > TTL) { res.send(401, "Web token has expired") return } req.query = decoded // all good, carry on next() } catch(e) { res.send(401, 'Unauthorized or invalid web token'); }}

Page 25: Node.js streaming csv downloads proxy

Node: validate JWT

var decoded = jwt.decode(req.query.jwt, secret);

?jwt=eyJ0eXAiOiJKV1QiLA0KICJhbGciOiJIUzI1NiJ.eyJpc3MiOiJqb2UiLA0KICJleHAiOjEzMD.dBjftJeZ4CVP-mB92K27uhbUJU1p1r_wW1gFWFOEjXk

Page 26: Node.js streaming csv downloads proxy

Node: validate JWT

if(decoded.shop_id != req.param(‘shop_id') { res.send(400, ‘JWT and query shop ids do not match'); return}

Page 27: Node.js streaming csv downloads proxy

Node: validate JWT

var now = new Date(), utc = getUtcCurrentDate();if(utc - Number(decoded.iat) > TTL) { res.send(401, "Web token has expired") return}

Page 28: Node.js streaming csv downloads proxy

Node: validate JWT

req.query = decoded // all good, carry onnext()

Page 29: Node.js streaming csv downloads proxy

Node: HTTP handlers

app.get('/:shop_id/orders.csv', tokenMiddleware, handler.create('orders', ...));

app.get('/:shop_id/contacts.csv', tokenMiddleware, handler.create('contacts', ...));

app.get('/:shop_id/products.csv', tokenMiddleware, handler.create('products', ...));

Page 30: Node.js streaming csv downloads proxy

}

goo.gl/nolmRK

ismael celis @ismasan