基于p5.js和ml5.js库的“音乐可视化+手势交互控制”创意网页制作

最新推荐文章于 2025-01-16 13:44:18 发布

doll ～CJ

最新推荐文章于 2025-01-16 13:44:18 发布

阅读量447

点赞数 7

CC 4.0 BY-SA版权

分类专栏： Python编程应用及网页设计实践文章标签：音乐可视化手势交互控制 p5.js ml5.js

本文链接：https://blog.csdn.net/Jcb1906824038/article/details/143919604

Python编程应用及网页设计实践专栏收录该内容

7 篇文章

订阅专栏

一、效果展示

二、“音乐可视化+手势交互控制”项目简述

“音乐可视化+手势交互控制”项目的设计思路源于本人在公众号上看到的一篇文章，其内容为一位来自武汉大学的同学将地理空间坐标编排形成了一首钢琴曲。在此次启发下，我开始学习相关技术支持，其中，来自bilibili网站的艺或XORLAB视频博主分享的相关技术视频给予我了很大的帮助。在此，对bilibili网站的艺或XORLAB视频博主表示衷心的感谢！

“音乐可视化+手势交互控制”项目，通过导入音乐且利用傅里叶变换将音乐转换为振幅数据，然后映射到一个转换区间，继而将振幅与圆柱高度关联并基于圆柱的高度振动变化可视化音乐状态。目前，该项目的手势交互控制仅添加了“手掌展开则播放音乐”和“手掌握拳则暂停音乐”两个功能，项目代码存在信号较长时间延迟。

三、项目代码

// launch.json

{
    // 使用 IntelliSense 了解相关属性。 
    // 悬停以查看现有属性的描述。
    // 欲了解更多信息，请访问: https://go.microsoft.com/fwlink/?linkid=830387
    "version": "0.2.0",
    "configurations": [
        {
            "type": "chrome",
            "request": "launch",
            "name": "针对 localhost 启动 Chrome",
            "url": "http://127.0.0.1:5500",
            "webRoot": "${workspaceFolder}",
            "resolveSourceMapLocations":[
                "${workspaceFolder}/**",
                "!**/node_modules/**"
            ]
        }
    ]
}

// index.html

<!DOCTYPE html>
<html lang="en">
<head>
	<meta charset="utf-8">
	<title>demo</title>

	<script src="https://cdnjs.cloudflare.com/ajax/libs/p5.js/1.4.0/p5.js"></script>
	<script src="https://cdnjs.cloudflare.com/ajax/libs/p5.js/1.4.0/addons/p5.sound.min.js"></script>
	<script src="https://unpkg.com/ml5@1/dist/ml5.min.js"></script>

</head>

<body>
	<script src="sketch.js"></script>
</body>

</html>

let soundPaths = ["res/music_1.ogg"];
let sound;
let fft,waveform;
let stars = [];
// 手势控制
let handPose;
let video;
let hands = [];
let threshold = 210000; // 该数值需要调试
let isFist;

function preload()
{   
    // prepare all music
    sound = loadSound(soundPaths);
    // Load the handPose model
    handPose = ml5.handPose();
}

function setup()
{
    createCanvas(640,480,WEBGL);  // 创建三维画板
    colorMode(HSB);    // 颜色体系切换
    fft = new p5.FFT();
    waveform = fft.waveform();
    // console.log(waveform);  // 在网页开发者工具的控制台中查看数据日志，waveform包含1024个数据
    // 手势控制部分
    //   // Create the webcam video and hide it
    video = createCapture(VIDEO);
    video.size(width, height);    // 如果video显示大小和幕布（canvas）大小不一致，则会造成显示坐标错误，但仍可完成手势判别效果
    video.hide();
    // start detecting hands from the webcam video
    handPose.detectStart(video, gotHands);
}

function draw()
{
    background(255);
    // Draw the webcam video
    image(video, -320, -240, 100, 100);
    meanSquaredError()
    if(!sound.isPlaying() && (isFist != true))
    {
        sound.play(); 
    }
    else
    {
        sound.pause();
    }
    // Draw all the tracked hand points
    // for (let i = 0; i < hands.length; i++) 
    // {
    //     let hand = hands[i];
    //     for (let j = 0; j < hand.keypoints.length; j++) 
    //     {
    //         let keypoint = hand.keypoints[j];
    //         fill(0, 255, 0);
    //         noStroke();
    //         circle(keypoint.x - 320, keypoint.y - 320, 5);
    //     }
    // }
    orbitControl();
    waveform = fft.waveform(); // 计算每一次刷新的音乐段振幅
    rotateX(PI/3);
    let r = width * 0.3;
    for(let a = 0;a < 2 * PI;a += PI/25)
    {
        let index = int(map(a, 0, 2*PI, 0, 1024));
        let curH = abs(300 * waveform[index])
        // 需要注意图像绘制原点在电脑屏幕正中央
        let x = r * cos(a);
        let y = r * sin(a);
        push();
        translate(x,y,curH/2);
        rotateX(PI/2);
        let c1 = color(150,200,200);
        let c2 = color(200,100,160);
        let rate = map(a, 0, 2*PI, 0, 0.9);
        let col = lerpColor(c1,c2,rate);
        stroke(col);
        cylinder(10, 5 + curH);  // 基于圆柱基础高度5
        pop();

        for(let k = 0; k < 10; k++)
        {
            // 振幅越小，创建粒子的概率就会越小
            // 粒子运动的速度和圆柱的高度大小正相关，即振幅越大，粒子运动速度越快
            if(random(0.01,1) < waveform[index]) 
            {
                // console.log(waveform[index]);
                stars.push(new star(x, y, 5 + curH, col));
            }
        }
    }

    for(let i = 0; i < stars.length; i++)
    {
        stars[i].move();
        stars[i].show();
        // console.log(stars[i].z);
        if (stars[i].z > 500)
        {
            stars.splice(i,1);  // 让粒子到一定时间慢慢被删除
        }
    }
}

function star(x, y, z, col)
{
    this.x = x + random(-2,2);
    this.y = y + random(-2,2);
    this.z = z;
    this.col = col;
    this.life = 500;
    this.speedX = random(-0.3,0.3);
    this.speedY = random(-0.3,0.3);
    this.speedZ = 0.05 + (z - 5) / 15;

    this.move = function()
    {
        this.z += this.speedZ;
        this.x += this.speedX;
        this.y += this.speedY;
        this.life -= 1;
    };

    this.show = function()
    {
        push();
        let a = map(this.life, 0, 500, 0, 1);
        stroke(hue(this.col), saturation(this.col),brightness(this.col));
        strokeWeight(1);
        point(this.x, this.y, this.z);
        pop()
    };

}

// Callback function for when handPose outputs data
function gotHands(results) {
  // save the output to the hands variable
  hands = results;
}

function meanSquaredError()
{
    let totalX = 0;
    let totalY = 0;
    let totalError = 0;
    // 调试
    // console.log(hands.length);
    for (let i = 0; i < hands.length; i++) 
    {
        let hand = hands[i];
        // 调试
        // console.log(hand.keypoints.length);
        for (let j = 0; j < hand.keypoints.length; j++) 
        {
            let keypoint = hand.keypoints[j];
            fill(0, 255, 0);
            noStroke();
            circle(keypoint.x - 320, keypoint.y - 320, 5);
            totalX += keypoint.x;
            totalY += keypoint.y;
        }
        // 计算中心点坐标
        const centerX = totalX / (hands.length * hand.keypoints.length);
        const centerY = totalY / (hands.length * hand.keypoints.length);
        // 计算每个关键点与中心点的均方根误差并添加到总误差中
        for(let j = 0; j < hand.keypoints.length; j++)
        {
            const keypoint = hand.keypoints[j];
            const errorX = keypoint.x - 320 - centerX;
            const errorY = keypoint.y - 320 - centerY;
            const squredError = errorX * errorX + errorY * errorY;
            totalError += squredError;
        }
        // 计算均方误差
        const meanSquaredError = totalError / hand.keypoints.length;
        if(meanSquaredError > threshold)
        {
            isFist = false;
        }
        else
        {
            isFist = true;
        }
        console.log("均方误差：" + meanSquaredError);
    }
}