Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 50 additions & 0 deletions examples/handPose-keypoints-offline/index.html
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
<!--
👋 Hello! This is an ml5.js example made and shared with ❤️.
Learn more about the ml5.js project: https://ml5js.org/
ml5.js license and Code of Conduct: https://github.com/ml5js/ml5-next-gen/blob/main/LICENSE.md

This example demonstrates offline-capable hand tracking using ml5.handPose
with the { cache: true } option. On first load the model is downloaded from
the network and saved to IndexedDB. On every subsequent load the model is
served from the local cache — no internet connection required.
-->

<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>ml5.js handPose Offline Example</title>
<script src="https://cdnjs.cloudflare.com/ajax/libs/p5.js/1.11.10/p5.min.js"></script>
<script src="../../dist/ml5.js"></script>
<style>
body {
font-family: system-ui, sans-serif;
}
button {
margin: 5px;
padding: 5px 10px;
}
#status-bar {
margin: 10px 0;
padding: 10px;
background: #f0f0f0;
border-radius: 4px;
display: inline-block;
}
</style>
</head>
<body>
<div id="controls">
<button id="btn-cache">Pre-Download Model</button>
<button id="btn-start">Start HandPose</button>
<button id="btn-clear">Clear Cache</button>
</div>

<div id="status-bar">Checking cache…</div>

<!-- p5.js appends the canvas here -->
<script src="sketch.js"></script>
</body>
</html>
178 changes: 178 additions & 0 deletions examples/handPose-keypoints-offline/sketch.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,178 @@
/*
* 👋 Hello! This is an ml5.js example made and shared with ❤️.
* Learn more about the ml5.js project: https://ml5js.org/
* ml5.js license and Code of Conduct: https://github.com/ml5js/ml5-next-gen/blob/main/LICENSE.md
*
* Example: handPose — Offline / Cached Mode
*
* This sketch demonstrates how to use ml5.handPose with the { cache: true }
* option for offline-capable installations and exhibitions.
*
* How it works:
* - First load (online): the model downloads from the network and is
* automatically saved to IndexedDB in your browser.
* - Later loads (offline): the model loads directly from IndexedDB —
* no internet connection needed.
*
* Three ways to interact with the cache:
* ⬇ Pre-Download — calls ml5.cacheModel("handPose") to cache the model
* *before* running the sketch, ideal for setup day.
* ▶ Start HandPose — calls ml5.handPose({ cache: true }) to load the model
* (from cache if available, otherwise from network) and
* begin live hand tracking.
* 🗑 Clear Cache — calls ml5.clearCache() to wipe all ml5 IndexedDB entries,
* useful for forcing a fresh download.
*/

let handPose;
let video;
let hands = [];

// App state
let isRunning = false;

// DOM references
const statusBar = document.getElementById("status-bar");
const btnCache = document.getElementById("btn-cache");
const btnStart = document.getElementById("btn-start");
const btnClear = document.getElementById("btn-clear");

// Check the initial cache status
async function checkCacheStatus() {
setStatus("Checking IndexedDB cache…");
const cached = await ml5.isCached("handPose");
if (cached) {
setStatus("Model is cached — you can go offline and click Start HandPose.");
} else {
setStatus(
"Model not yet cached. Click Pre-Download to cache it, or Start HandPose to download and cache automatically."
);
}
}

function setup() {
createCanvas(640, 480);

// Create the webcam capture but keep it hidden — we draw it to the canvas.
video = createCapture(VIDEO);
video.size(640, 480);
video.hide();

// Setup button callbacks
btnCache.addEventListener("click", onPreDownload);
btnStart.addEventListener("click", onStart);
btnClear.addEventListener("click", onClearCache);

checkCacheStatus();
}

function draw() {
if (isRunning) {
// Draw the webcam video
image(video, 0, 0, width, height);

// Draw all the tracked hand points
for (let i = 0; i < hands.length; i++) {
let hand = hands[i];
for (let j = 0; j < hand.keypoints.length; j++) {
let keypoint = hand.keypoints[j];
fill(0, 255, 0);
noStroke();
circle(keypoint.x, keypoint.y, 10);
}
}
} else {
// Idle state
background(200);
fill(100);
noStroke();
textSize(18);
textAlign(CENTER, CENTER);
text(
"Webcam will appear here after\nclicking Start HandPose",
width / 2,
height / 2
);
}
}

// Callback function for when handPose outputs data
function gotHands(results) {
hands = results;
}

// ─── Button handlers ──────────────────────────────────────────────────────────

async function onPreDownload() {
btnCache.disabled = true;
btnStart.disabled = true;

setStatus("Downloading HandPose models and saving to IndexedDB…");

try {
await ml5.cacheModel("handPose");
setStatus(
"Models cached successfully! You can now disconnect from the internet and use Start HandPose."
);
} catch (err) {
setStatus(`Download failed: ${err.message}`);
}

btnCache.disabled = false;
btnStart.disabled = false;
}

async function onStart() {
btnCache.disabled = true;
btnStart.disabled = true;

const cached = await ml5.isCached("handPose");
if (cached) {
setStatus("Loading HandPose from local cache…");
} else {
setStatus(
"Downloading HandPose models (first time — this may take a moment)…"
);
}

// Pass { cache: true } to load from cache or save to cache
handPose = await ml5.handPose({ cache: true });

const nowCached = await ml5.isCached("handPose");
setStatus(
nowCached
? "Running — model loaded from cache. Safe to go offline!"
: "Running — model loaded from network (cache unavailable)."
);

isRunning = true;
handPose.detectStart(video, gotHands);

btnStart.textContent = "HandPose Running";
btnStart.disabled = true;
btnCache.disabled = false;
}

async function onClearCache() {
btnClear.disabled = true;

setStatus("Clearing ml5 model cache…");
const count = await ml5.clearCache();

if (count > 0) {
setStatus(
`Removed ${count} cached model file${
count !== 1 ? "s" : ""
} from IndexedDB.`
);
} else {
setStatus("Cache was already empty.");
}

btnClear.disabled = false;
}

function setStatus(message) {
if (statusBar) statusBar.textContent = message;
console.log("[handPose-offline]", message);
}
1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
"devDependencies": {
"@babel/core": "^7.23.2",
"@babel/preset-env": "^7.23.2",
"@tensorflow/tfjs-node": "^4.22.0",
"all-contributors-cli": "^6.26.1",
"babel-jest": "^29.7.0",
"bson-objectid": "^2.0.4",
Expand Down
45 changes: 45 additions & 0 deletions src/BodyPose/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,13 @@ import handleOptions from "../utils/handleOptions";
import { handleModelName } from "../utils/handleOptions";
import objectRenameKey from "../utils/objectRenameKey";
import { isVideo } from "../utils/handleArguments";
import { CachingIOHandler } from "../utils/modelCache";
import {
getBodyPoseMoveNetUrl,
getBodyPoseMoveNetCacheKey,
getBodyPoseBlazePoseUrls,
getBodyPoseBlazePoseCacheKeys,
} from "../utils/modelRegistry";

/**
* User provided options object for BodyPose with MoveNet model.
Expand All @@ -44,6 +51,9 @@ import { isVideo } from "../utils/handleArguments";
* @property {string} [trackerType] - The type of tracker to use.
* @property {object} [trackerConfig] - Advanced tracker configurations.
* @property {string} [modelUrl] - The file path or URL to the MoveNet model.
* @property {boolean} [cache] - Whether to cache the model in IndexedDB for offline
* use. On first load the model is downloaded and saved;
* on subsequent loads it is served from the local cache.
*/

/**
Expand All @@ -64,6 +74,9 @@ import { isVideo } from "../utils/handleArguments";
* model. Only for `tfjs` runtime.
* @property {string} [landmarkModelUrl] - The file path or URL to the BlazePose landmark
* model. Only for `tfjs` runtime.
* @property {boolean} [cache] - Whether to cache the model in IndexedDB for offline
* use. On first load the model is downloaded and saved;
* on subsequent loads it is served from the local cache.
*/

/**
Expand Down Expand Up @@ -227,13 +240,43 @@ class BodyPose {
blazePoseConfigSchema,
"bodyPose"
);

// If cache: true is requested and runtime is tfjs, wrap the model URLs
// with CachingIOHandler for transparent IndexedDB caching.
if (this.userOptions?.cache === true && modelConfig.runtime === "tfjs") {
const defaults = getBodyPoseBlazePoseUrls(modelConfig);
const keys = getBodyPoseBlazePoseCacheKeys(modelConfig);
const detectorUrl = modelConfig.detectorModelUrl ?? defaults.detector;
const landmarkUrl = modelConfig.landmarkModelUrl ?? defaults.landmark;
modelConfig.detectorModelUrl = new CachingIOHandler(
detectorUrl,
keys.detector
);
modelConfig.landmarkModelUrl = new CachingIOHandler(
landmarkUrl,
keys.landmark
);
}
} else {
pipeline = poseDetection.SupportedModels.MoveNet;
modelConfig = handleOptions(
this.userOptions,
MoveNetConfigSchema,
"bodyPose"
);

// If cache: true is requested and no custom modelUrl is provided, wrap
// the default URL with CachingIOHandler for transparent IndexedDB caching.
// We read modelType as a string here, before the switch below replaces it
// with the poseDetection enum value.
if (this.userOptions?.cache === true && !modelConfig.modelUrl) {
const url = getBodyPoseMoveNetUrl({ modelType: modelConfig.modelType });
const key = getBodyPoseMoveNetCacheKey({
modelType: modelConfig.modelType,
});
modelConfig.modelUrl = new CachingIOHandler(url, key);
}

// Map the modelType string to the `movenet.modelType` enum
switch (modelConfig.modelType) {
case "SINGLEPOSE_LIGHTNING":
Expand Down Expand Up @@ -277,6 +320,7 @@ class BodyPose {
);
const { image, callback } = argumentObject;
// Run the detection
await this.ready;
await mediaReady(image, false);
const predictions = await this.model.estimatePoses(image);
let result = predictions;
Expand Down Expand Up @@ -342,6 +386,7 @@ class BodyPose {
* @private
*/
async detectLoop() {
await this.ready;
await mediaReady(this.detectMedia, false);
while (!this.signalStop) {
const predictions = await this.model.estimatePoses(this.detectMedia);
Expand Down
2 changes: 2 additions & 0 deletions src/BodySegmentation/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -208,6 +208,7 @@ class BodySegmentation {
);
const { image, callback } = argumentObject;

await this.ready;
await mediaReady(image, false);

let inputForSegmenter = image;
Expand Down Expand Up @@ -330,6 +331,7 @@ class BodySegmentation {
* @private
*/
async detectLoop() {
await this.ready;
await mediaReady(this.detectMedia, false);
while (!this.signalStop) {
let inputForSegmenter = this.detectMedia;
Expand Down
Loading