-
Notifications
You must be signed in to change notification settings - Fork 41
[WIP] Add MobileNet Feature Extractor #301
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
bf80337
adea7f2
e7f125f
01fa498
91006a6
fb88763
2336f6b
2964258
f1efb33
0b921f3
490ce7e
e119244
a1d3140
17afa84
aa53f36
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,14 @@ | ||
| <!DOCTYPE html> | ||
| <html lang="en"> | ||
| <head> | ||
| <meta charset="UTF-8" /> | ||
| <meta http-equiv="X-UA-Compatible" content="IE=edge" /> | ||
| <meta name="viewport" content="width=device-width, initial-scale=1.0" /> | ||
| <title>ml5.js featureExtractor Example</title> | ||
| <script src="https://cdnjs.cloudflare.com/ajax/libs/p5.js/1.11.10/p5.min.js"></script> | ||
| <script src="../../dist/ml5.js"></script> | ||
| </head> | ||
| <body> | ||
| <script src="sketch.js"></script> | ||
| </body> | ||
| </html> |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,75 @@ | ||
| let feClassifier; | ||
| let video; | ||
| let label = ""; | ||
| let input1, input2; | ||
| let addButton1, addButton2; | ||
| let trainButton; | ||
| let count1 = 0; | ||
| let count2 = 0; | ||
|
|
||
| function modelReady() { | ||
| console.log("Model is ready!"); | ||
| } | ||
|
|
||
| function gotResults(results) { | ||
| label = results[0].label + " (" + nf(results[0].confidence, 0, 2) + ")"; | ||
| } | ||
|
|
||
| function preload() { | ||
| feClassifier = ml5.featureExtractor({ task: 'classification' }, modelReady); | ||
| } | ||
|
|
||
| function setup() { | ||
| createCanvas(640, 480); | ||
| video = createCapture(VIDEO); | ||
| video.hide(); | ||
| background(0); | ||
| // Set the video as the input for the Classifier | ||
| feClassifier.video = video; | ||
|
|
||
|
|
||
| // Create inputs and buttons for adding samples for two classes | ||
| input1 = createInput("", "text"); | ||
| input1.attribute("placeholder", "Class 1"); | ||
| addButton1 = createButton("Add Sample"); | ||
| addButton1.mousePressed(function () { | ||
| let className = input1.value() || "Class 1"; | ||
| feClassifier.addImage(className); | ||
| count1++; | ||
| console.log(className + " samples: " + count1); | ||
| }); | ||
|
|
||
| input2 = createInput("", "text"); | ||
| input2.attribute("placeholder", "Class 2"); | ||
| addButton2 = createButton("Add Sample"); | ||
| addButton2.mousePressed(function () { | ||
| let className = input2.value() || "Class 2"; | ||
| feClassifier.addImage(className); | ||
| count2++; | ||
| console.log(className + " samples: " + count2); | ||
| }); | ||
|
|
||
| trainButton = createButton("train"); | ||
| trainButton.mousePressed(function () { | ||
| feClassifier.train({ epochs: 100, debug: true }, function () { | ||
| console.log("Starting classification..."); | ||
| feClassifier.classifyStart(gotResults); | ||
| }); | ||
| }); | ||
| } | ||
|
|
||
| function draw() { | ||
| background(0); | ||
|
|
||
| // Draw the video (flipped horizontally) | ||
| push(); | ||
| translate(640, 0); | ||
| scale(-1, 1); | ||
| image(video, 0, 0, 640, 450); | ||
| pop(); | ||
|
|
||
| // Draw the label | ||
| fill(255); | ||
| textSize(16); | ||
| text(label, 10, height - 10); | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,23 @@ | ||
| <!DOCTYPE html> | ||
| <html lang="en"> | ||
| <head> | ||
| <meta charset="UTF-8" /> | ||
| <meta http-equiv="X-UA-Compatible" content="IE=edge" /> | ||
| <meta name="viewport" content="width=device-width, initial-scale=1.0" /> | ||
| <title>ml5.js featureExtractor Regression - Distance Control</title> | ||
| <script src="https://cdnjs.cloudflare.com/ajax/libs/p5.js/1.11.10/p5.min.js"></script> | ||
| <script src="../../dist/ml5.js"></script> | ||
| </head> | ||
| <body> | ||
| <h2>Feature Extractor: Regression - Distance Control</h2> | ||
| <p>Control the size of a circle by moving closer to or farther from the camera.</p> | ||
| <ol> | ||
| <li><strong>Lean back</strong> from the camera, set the slider to <strong>0</strong>, and click "Add Sample" several times.</li> | ||
| <li><strong>Lean in close</strong> to the camera, set the slider to <strong>1</strong>, and click "Add Sample" several times.</li> | ||
| <li>Optionally, add samples at <strong>middle distances</strong> with the slider around <strong>0.5</strong>.</li> | ||
| <li>Click <strong>"Train"</strong> and wait for training to complete.</li> | ||
| <li>Move closer or farther from the camera — the yellow circle will grow and shrink in real time!</li> | ||
| </ol> | ||
| <script src="sketch.js"></script> | ||
| </body> | ||
| </html> |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,81 @@ | ||
| let feRegressor; | ||
| let video; | ||
| let slider; | ||
| let addButton; | ||
| let trainButton; | ||
| let sampleCount = 0; | ||
| let predictedValue = 0; | ||
|
|
||
| function modelReady() { | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Small nitpick: defining those function in code in the order that we expect them to be called (e.g. |
||
| console.log("Model is ready!"); | ||
| } | ||
|
|
||
| function gotResults(results) { | ||
| predictedValue = results[0].value; | ||
| } | ||
|
|
||
| function preload() { | ||
| // Initialize the feature extractor for regression | ||
| feRegressor = ml5.featureExtractor({ task: 'regression', version: 2 }, modelReady); | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Is the
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The reason why I passed
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Curious if changing the default to version 2 could be an option? Does this have downsides for the classification task perhaps?
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I believe version 2 also works perfectly for classification tasks. (will change the default version to 2 in next commit) |
||
| } | ||
|
|
||
| function setup() { | ||
| createCanvas(640, 480); | ||
| video = createCapture(VIDEO); | ||
| video.hide(); | ||
| background(0); | ||
| // Set the video as the input for the Regressor | ||
| feRegressor.video = video; | ||
|
|
||
| // Slider: 0 = far from camera, 1 = close to camera | ||
| slider = createSlider(0, 1, 0.5, 0.01); | ||
| slider.style("width", "640px"); | ||
|
|
||
| // Add a sample with the current slider value | ||
| addButton = createButton("Add Sample"); | ||
| addButton.mousePressed(function () { | ||
| feRegressor.addImage(slider.value()); | ||
| sampleCount++; | ||
| console.log("Sample " + sampleCount + " added with value: " + slider.value()); | ||
| }); | ||
|
|
||
| // Train and start predicting | ||
| trainButton = createButton("Train"); | ||
| trainButton.mousePressed(function () { | ||
| feRegressor.train({ epochs: 500, learningRate: 0.0001, debug: true }, function () { | ||
| console.log("Starting regression..."); | ||
| feRegressor.predictStart(gotResults); | ||
| }); | ||
| }); | ||
| } | ||
|
|
||
| function draw() { | ||
| background(0); | ||
|
|
||
| // Draw the video (flipped horizontally) | ||
| push(); | ||
| translate(640, 0); | ||
| scale(-1, 1); | ||
| image(video, 0, 0, 640, 450); | ||
| pop(); | ||
|
|
||
| // Use predicted value if trained, otherwise follow the slider | ||
| let currentValue = feRegressor.isTrained ? predictedValue : slider.value(); | ||
| // Clamp to 0-1 range | ||
| currentValue = constrain(currentValue, 0, 1); | ||
|
|
||
| // Map value to circle size: 0 = small (80), 1 = large (300) | ||
| let circleSize = map(currentValue, 0, 1, 80, 300); | ||
|
|
||
| // Draw the circle in the center of the video area | ||
| noFill(); | ||
| stroke(255, 255, 0); | ||
| strokeWeight(3); | ||
| ellipse(320, 250, circleSize, circleSize); | ||
|
|
||
| // Draw the label | ||
| noStroke(); | ||
| fill(255); | ||
| textSize(16); | ||
| text("Value: " + nf(currentValue, 1, 4), 10, height - 10); | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
It is probably nicer to have a dedicated method to setting the input here (which might be a video, but possibly also an image, canvas...) 🤔 Something to think together with @shiffman at some convenient time.
For context: previously, the video input got passed as an argument to the constructor, but since the feature extractor now gets created in
preload(), we typically don't have the video element yet at this point.Our other models, such as bodyPose, work around this by taking
videoas an argument to e.g.detectStart(). We could do the same, and require video as an argument to bothaddImage()andclassifyStart(). Or, pass the source to the feature extractor at one point in time only - similarly to how @JunhaoZhu0220 is doing here.