Augmented Reality in JavaScript

augmented reality.js

San Francisco, 2013.

I’m from brazil

no, I don’t samba

I ♥ HTML5

html5-pro.com/wormz

I ♥ opensource

browserdiet.com

today we’regoing to

talk about augmented

reality...

...and how tointegratedifferent

HTML5 APIs

augumented reality is everywhere,or it will be...

augmented reality is

(or at leastwill be)

everywhere

before it was cool

used Google Glass

what is augmented reality?

how can youdo it usingjavascript?

mzl.la/ilyX7k

1. capturewebcam

dev.w3.org/2011/webrtc/editor/webrtc.html

Access user's camera and microphone

navigator.getUserMedia({ video: true, audio: true}, onSuccess, onFail);

STEP 1

2. play thecapturedvideo

<video>

Play webcam's content into a video element

function onSuccess(stream) { var video = document.querySelector('video'); video.src = window.URL.createObjectURL(stream); video.onloadedmetadata = function(e) { // Ready };}

STEP 2

3. track apattern ofpixels

<video>

fiducialmarkers

github.com/kig/JSARToolKit

libspark.org/wiki/saqoosha/FLARToolKit/en

hitl.washington.edu/artoolkit

soundstep.com/blog/experiments/jstracking/

bit.ly/XR0aGH

bit.ly/124FT3A

FIDUCIALMARKERS

<html><head><script src="../../src/NyAs3Utils.js"></script><script src="../../src/FLARArrayUtil.js"></script><script src="../../src/FLARException.js"></script><script src="../../src/FLARMat.js"></script><script src="../../src/FLARRgbPixelReader.js"></script><script src="../../src/NyARHistogramAnalyzer.js"></script><script src="../../src/NyARPca2d.js"></script><script src="../../src/NyARRasterReader.js"></script><script src="../../src/NyARTypes.js"></script><script src="../../src/FLARRasterFilter.js"></script><script src="../../src/FLARTypes.js"></script><script src="../../src/NyARLabel.js"></script><script src="../../src/FLARLabeling.js"></script><script src="../../src/NyARParam.js"></script><script src="../../src/FLARParam.js"></script><script src="../../src/NyARRaster.js"></script><script src="../../src/FLARRaster.js"></script><script src="../../src/NyARCode.js"></script><script src="../../src/FLARCode.js"></script><script src="../../src/NyARMatch.js"></script><script src="../../src/NyARRasterAnalyzer.js"></script><script src="../../src/FLARRasterAnalyzer.js"></script><script src="../../src/NyARRasterFilter.js"></script><script src="../../src/NyARSquareDetect.js"></script><script src="../../src/FLARSquareDetect.js"></script><script src="../../src/NyARTransMat.js"></script><script src="../../src/FLARTransMat.js"></script><script src="../../src/NyARUtils.js"></script><script src="../../src/NyARIdMarker.js"></script><script src="../../src/NyARPickup.js"></script><script src="../../src/FLARProcessor.js"></script><script src="../../src/NyARDetector.js"></script><script src="../../src/FLARDetector.js"></script><script src="../../src/FLARIdMarkerDetector.js"></script><script src="../../src/NyARSingleMarkerProcesser.js"></script><script src="../../src/NyUtils.js"></script><script src="../magi.js"></script><script>/** CPSQueue serializes a bunch of callbacks, which is useful for e.g. loading images in order.

To use CPSQueue, first create it passing the call function as argument:

var q = new CPSQueue(function(src, callback, onError){ var img = new Image() img.onload = callback img.onerror = onError img.onabort = onError img.src = src })

Then use the queue by calling append:

Rg(1,10).forEach(function(i){ q.append(i+".jpg", function(){ console.log("Loaded "+this.src) }, function(){ console.log("Failed to load "+this.src) }) })

Now you should see that the images load (or fail to load) in the same order as they were appended, even if 1.jpg is 5MB and 2.jpg is 1kB in size. */CPSQueue = Klass({ initialize : function(call, cond, merge) { this.queue = []; if (call) this.call = call; if (cond) this.cond = cond; if (merge) this.merge = merge; },

append : function (args, callback, onError) { if (this.cond.call(this, args)) { if (callback) callback(args); } else { this.queue.push({args: args, callback: callback, onError: onError}); this.process(); } },

abort : function() { this.queue = []; this.aborted = true; },

process : function() { if (this.processing) return; this.aborted = false; this.processing = true; this.queue = this.merge(this.queue); var t = this.queue.shift(); while (t) { if (this.cond.call(this, t.args)) { if (t.callback) t.callback(t.args); t = this.queue.shift(); } else { break; } } if (!t) { this.processing = false; return; } var self = this; this.call(t.args, function(arg) { if (!self.aborted) { if (t.callback) t.callback(t.args, arg); } self.processing = false; self.process(); }, function(arg) { if (t.onError) t.onError(t.args, arg); self.processing = false; self.process(); }) },

merge : function(q) { return q; },

call : function(args, cb) { cb(args); },

cond : function(args) { return false; }});

UploadNotification = Klass({ initialize : function(file, container) { this.file = file; this.progressElem = E.canvas(30, 30); this.elem = DIV({className: 'uploadNotification'}, this.progressElem, H3(file.name), SPAN(this.formatSize(file.size, "B")) ); container.appendChild(this.elem); },

formatSize : function(b, unit) { var mags = ["", "k", "M", "G", "T", "P", "E"]; var i,j; for (i=0, j=1000; i<mags.length && j<b; i++, j*=1000) null; if (i == 0) { return b + unit; } else { var s = parseInt(10 * b / (j/1000)) / 10; return s + mags[i] + unit; } },

onprogress : function(x) { var w = this.progressElem.width, h = this.progressElem.height; var ctx = this.progressElem.getContext('2d'); ctx.clearRect(0,0,w,h); var r = Math.min(w,h) / 2 - 5; ctx.beginPath(); ctx.arc(w/2,h/2, r, 0, Math.PI*2, false); ctx.strokeStyle = "rgba(255,255,255, 0.2)" ctx.lineWidth = 8; ctx.stroke(); ctx.beginPath(); var top = -Math.PI/2; var frac = x.loaded / x.total; if (x.type == 'load') frac = 1; if (x.type == 'loadstart') frac = 0; ctx.arc(w/2,h/2, r, top, top+frac*Math.PI*2, false); ctx.strokeStyle = "rgba(0,192,255, 0.8)" ctx.lineWidth = 9; ctx.stroke(); },

upload : function(callback, onerror) { var self = this; this.xhr = DnDUpload.uploadFile(this.file, {}, function(x) { self.fadeStart = new Date(); self.fadeInterval = setInterval(function() { var elapsed = new Date() - self.fadeStart; var f = elapsed / 500; if (f >= 1) { f = 1; self.elem.parentNode.removeChild(self.elem); clearInterval(self.fadeInterval); } self.elem.opacity = 1 - f; }, 15); callback(); }, function(x) { self.elem.appendChild(H3("Error: " + x.toString())); onerror(); }, function(x){ self.onprogress(x) }); }});

DnDUpload = Klass({ formData : function(name, value, filename, headers) { var CRLF = '\r\n'; var s = 'Content-Disposition: form-data; '; s += 'name="'+name+'"'; if (filename) s += '; filename="'+filename+'"'; if (headers) s += CRLF + headers; s += CRLF + CRLF + value + CRLF; return s; },

generateBoundary : function(parts) { var b; var found = true; while (found) { found = false; b = Math.random() + "---BOUNDARY---" + new Date().getTime(); for (var i=0; i<parts.length; i++) { if (parts[i].indexOf(b) != -1) { found = true; break; } } } return b; },

sendFileUpload : function(xhr, parts, callback, onerror, onprogress) { xhr.open("POST", this.path); if (callback) xhr.addEventListener('load', callback, false); if (onerror) xhr.addEventListener('error', onerror, false); if (onprogress) { xhr.upload.addEventListener('error', function(x) { }, false); xhr.upload.addEventListener('load', onprogress, false); xhr.upload.addEventListener('loadstart', onprogress, false); xhr.upload.addEventListener('progress', onprogress, false); } if (window.FormData && parts instanceof window.FormData) { xhr.send(parts); } else { var CRLF = '\r\n'; var boundary = this.generateBoundary(parts); var req = "--" + boundary + CRLF + parts.join("--" + boundary + CRLF) + "--" + boundary + "--" + CRLF; var ct = "multipart/form-data; boundary=" + boundary; xhr.setRequestHeader("Content-Type", ct); xhr.sendAsBinary(req); } },

uploadFile : function(file, opts, callback, onerror, onprogress) { return this.uploadFiles([file], opts, callback, onerror, onprogress); },

uploadFiles : function(files, opts, callback, onerror, onprogress) { var xhr = new XMLHttpRequest; if (!(window.FormData || xhr.sendAsBinary)) throw("Can't upload files!"); var self = this; var parts; if (window.FormData) { parts = new FormData(); } else { parts = []; parts.append = function(key, value) { if (value.getAsBinary) this.push( self.formData(key, value.getAsBinary(), value.name, "Content-Type: "+value.type) ); else this.push( self.formData(key, value) ); }; } for (var i=0; i<files.length; i++) parts.append("files", files[i]); self.sendFileUpload(xhr, parts, callback, onerror, onprogress); return xhr; },

getUploadQueue : function() { if (!this.uploadQueue) this.uploadQueue = new CPSQueue(function(notification, callback, onerror) { notification.upload(callback, onerror); }); return this.uploadQueue; },

upload : function(files, notificationElement) { var uq = this.getUploadQueue(); toArray(files).forEach(function(file) { var notification = new UploadNotification(file, notificationElement); uq.append(notification); }); },

setupTarget : function(cont, callback) { var self = this;

var hasFiles = function(ev) { return ev.dataTransfer && (ev.dataTransfer.hasOwnProperty('files') || ev.dataTransfer.__lookupGetter__('files')); };

cont.addEventListener('dragenter', function(ev) { if (!hasFiles(ev)) return; this.style.backgroundColor = 'rgba(0, 255, 0, 0.3)'; Event.stop(ev); }, false); cont.addEventListener('dragleave', function(ev) { if (!hasFiles(ev)) return; this.style.backgroundColor = 'transparent'; Event.stop(ev); }, false); cont.addEventListener('dragover', function(ev) { if (!hasFiles(ev)) return; this.style.backgroundColor = 'rgba(0, 255, 0, 0.3)'; Event.stop(ev); }, false); cont.addEventListener('drop', function(ev) { if (!hasFiles(ev)) return; this.style.backgroundColor = 'transparent'; Event.stop(ev); if (callback) callback(ev.dataTransfer.files, this); else self.upload(ev.dataTransfer.files, this); }, false); }});</script><script> threshold = 50; DEBUG = false; var xhr = new XMLHttpRequest; xhr.open("GET", "photos2.xml", false); photos = []; xhr.addEventListener('load', function(){ var ps = this.responseText.match(/http:\/\/[a-z0-9_\/\.]+_m.jpg/g).sort().unique() .map(function(u){ return u.replace(/m\.jpg$/, "z.jpg") }); photos = ps .map(function(u){ return u.replace(/.*\/([^\/]+)$/, "$1") }) .map(Image.load); }, false); xhr.send(null);

var video = document.createElement('video'); video.width = 640; video.height = 480; video.loop = true; video.volume = 0; video.autoplay = true; video.style.display = 'none'; video.controls = true; video.src = "swap_loop.ogg";

window.onload = function() { byId('loading').style.display = 'none'; document.body.appendChild(video);

var canvas = document.createElement('canvas'); canvas.width = 320; canvas.height = 240; canvas.style.display = 'block';

var videoCanvas = document.createElement('canvas'); videoCanvas.width = video.width; videoCanvas.height = video.height;

var raster = new NyARRgbRaster_Canvas2D(canvas); var param = new FLARParam(320,240);

var resultMat = new NyARTransMatResult();

var detector = new FLARMultiIdMarkerDetector(param, 80); detector.setContinueMode(true);

var ctx = canvas.getContext('2d'); ctx.font = "24px URW Gothic L, Arial, Sans-serif";

var glCanvas = document.createElement('canvas'); glCanvas.width = 960; glCanvas.height = 720; var s = glCanvas.style; document.body.appendChild(glCanvas); display = new Magi.Scene(glCanvas); display.drawOnlyWhenChanged = true; param.copyCameraMatrix(display.camera.perspectiveMatrix, 10, 10000); display.camera.useProjectionMatrix = true; var videoTex = new Magi.FlipFilterQuad(); videoTex.material.textures.Texture0 = new Magi.Texture(); videoTex.material.textures.Texture0.image = videoCanvas; videoTex.material.textures.Texture0.generateMipmaps = false; display.scene.appendChild(videoTex);

var times = []; var pastResults = {}; var lastTime = 0; var cubes = {}; var images = [];

setInterval(function(){ if (video.ended) video.play(); if (video.paused) return; if (window.paused) return; if (video.currentTime == video.duration) { video.currentTime = 0; } if (video.currentTime == lastTime) return; lastTime = video.currentTime; videoCanvas.getContext('2d').drawImage(video,0,0); ctx.drawImage(videoCanvas, 0,0,320,240); var dt = new Date().getTime();

videoTex.material.textures.Texture0.changed = true;

canvas.changed = true; display.changed = true;

</head><body><div id="loading">Hang on, I'm loading stuff...</div></body></html>

face detection

webdesign.maratz.com/lab/responsivetypography

RESPONSIVETYPOGRAPHY

github.com/auduno/headtrackr

auduno.github.com/headtrackr/examples/targets.html

HEADTRACKRTARGETS

@eduardolundgren (Liferay)

trackingjs.com

bit.ly/YVPz3e

bit.ly/ZSZuUd

TRACKING.JSSINGLE CONTROLLER

Imports tracking.js core

Imports tracking.js color module

STEP 1SINGLE CONTROLLER

Gets user's camera and renders it

var videoCamera = new tracking .VideoCamera() .render();

Hides video camera and renders a canvas from it

videoCamera = videoCamera .hide() .renderVideoCanvas();

Instantiates tracking by magenta color

videoCamera.track({ type: 'color', color: 'magenta', onFound: function() {}, onNotFound: function() {}});

Paints with magenta all detected pixels

onFound: function(track) { var pixels = track.pixels,

ctx = videoCamera.canvas.context;

for (var i = 0, len = pixels.length; i < len; i += 2) { ctx.fillStyle = "rgb(255,0,255)"; ctx.fillRect(pixels[i], pixels[i+1], 2, 2); }

ctx.fillStyle = "rgb(0,0,0)"; ctx.fillRect(track.x, track.y, 5, 5);}

bit.ly/118nzaL

TRACKING.JSMULTIPLE CONTROLLERS

Gets user's camera and renders it

var videoCamera = new tracking .VideoCamera() .render();

STEP 1MULTIPLE CONTROLLERS

Hides video camera and renders a canvas from it

videoCamera = videoCamera .hide() .renderVideoCanvas();

Instantiates tracking by magenta color

videoCamera.track({ type: 'color', color: 'magenta', onFound: function() {}, onNotFound: function() {}});

Draws a square around tracked area

onFound: function(track) { var size = 60 - track.z; var ctx = videoCamera.canvas.context; ctx.strokeStyle = "rgb(255,0,255)"; ctx.lineWidth = 3; ctx.strokeRect(track.x - size*0.5, track.y - size*0.5, size, size);}

Instantiates tracking by cyan color

videoCamera.track({ type: 'color', color: 'cyan', onFound: function() {}, onNotFound: function() {}});

Draws a square around tracked area

onFound: function(track) { var size = 60 - track.z; var ctx = videoCamera.canvas.context; ctx.strokeStyle = "rgb(0,255,255)"; ctx.lineWidth = 3; ctx.strokeRect(track.x - size*0.5, track.y - size*0.5, size, size);}

bit.ly/10mIS6h

TRACKING.JSDRAW SOMETHING

bit.ly/XR5vha

TRACKING.JSMINECRAFT

bit.ly/X6LuGj

TRACKING.JSGLASSES

IT’S UP TO YOU!

thanks :)

zenorocha.com

Augmented Reality in JavaScript

Technology

State of Augmented Reality, Virtual Reality and Mixed Reality · State of Augmented Reality, Virtual Reality and Mixed Reality [Microsoft Hololen] [Ready Player One] Augmented Reality

The Emergence of Augmented Reality in Orthopaedic Surgery ... · PDF fileThe Emergence of Augmented Reality in Orthopaedic ... Augmented Reality in Orthopaedic Surgery ... augmented

Augmented Reality George McKeague (ABB)...Augmented Reality George Mc Keague, ABB — Virtual Reality vs. Augmented Reality • Virtual Reality(VR) is an artificial, computer-generated

Augmented Reality or Augmented Space?

Augmented experience: Augmented Reality

Virtual- and Augmented Reality in Education · •Augmented Reality in Education . Milgram’s Reality-Virtuality Continuum (1994) Mixed Reality Real Environment Augmented Reality

The Power of Augmented Reality and Virtual Reality for ...€¦ · The Power of Augmented Reality and Virtual Reality for Business ... AUGMENTED REALITY VIRTUAL REALITY ... •Integrating

Augmented Reality Annotation for Social Video Sharing · Augmented Reality Annotation for Social Video Sharing ... cially focus on using Augmented Reality ... Javascript for the front-end

AUGMENTED AND VISUAL REALITY. WHAT IS AUGMENTED AND VISUAL REALITY?

Augmented Reality for Smartphones - 123seminarsonly.com · Augmented Reality for Smartphones TechWatch Report Introduction to Augmented Reality in Smartphones Page 2 Augmented Reality

Mixed Reality: Virtual Reality, Augmented Reality, Augmented Virtuality Luigi Cinque Maggio 2008

Enrich life with Augmented Reality UI of the Future What is Augmented Reality? z Reality Virtual Reality Augmented Reality • By definition, Augmented Reality means augmenting the

Augmented Reality (AR) - Razvoj karijere | Center for ... · PDF fileProširena stvarnost - Augmented Reality (AR) Dr Nenad Gligorić. Šta je Augmented Reality? •Termin Augmented

Augmented Reality

AUGMENTED REALITY Case Study AUGMENTED REALITY

Augmented Reality and Industry - WebHomecampar.in.tum.de/.../klinker2015ismarKeynote.slides.pdf · Augmented Reality and Industry Prof. Gudrun Klinker, Ph.D. ... Augmented Reality

AR - TLC Augmented Reality big2 · Augmented Reality CREATIVE TECHNOLOGY . Augmented Reality Augmented reality allows attendees to view hidden computer-generated content via a marker

Augmented Reality in JavaScript

Augmented Reality Gaming | Augmented Reality Games

VIRTUAL & AUGMENTED REALITY · VIRTUAL REALITY AUGMENTED REALITY Quriuz VIRTUAL AUGMENTED REALITY REALITY Quriuz . Title: PowerPoint-presentatie Author: Niessing, Sander Created Date: