go并发请求url_go 请求url-CSDN博客

本文链接：https://blog.csdn.net/weixin_42592326/article/details/137385181

文章详细描述了如何在Go语言中使用sync.WaitGroup和channel来管理并发HTTP请求，确保每个请求完成后将结果写入数据库，同时设置3秒超时以控制请求时间。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

sync.WaitGroup写法

package main

import (
	"database/sql"
	"fmt"
	"net/http"
	"sync"
	"time"

	_ "github.com/go-sql-driver/mysql"
)

func main() {
	//开始计时
	start := time.Now()

	//链接数据库，用户名：密码@tcp(地址:端口)/数据库名
	dsn := "gin:gin@tcp(42.1.25.18:3306)/gin"
	// 连接到MySQL数据库
	db, err := sql.Open("mysql", dsn)
	if err != nil {
		fmt.Println("数据库连接错误: ", err)
		return
	}
	defer db.Close()

	urls := []string{
		"https://www.baidu.com",
		"https://www.taobao.com",
		"https://www.tmall.com",
		"https://www.jd.com",
		"https://www.sina.com.cn",
		"https://www.sohu.com",
		"https://www.qq.com",
		"https://www.163.com",
		"https://www.zhihu.com",
		"https://www.weibo.com",
		"https://www.bilibili.com",
		"https://www.youku.com",
		"https://www.iqiyi.com",
		"https://www.alipay.com",
		"https://www.dingtalk.com",
		"https://www.wechat.com",
		"https://www.360.cn",
		"https://www.58.com",
		"https://www.liepin.com",
		"https://www.zhaopin.com",
		"https://www.douban.com",
		"https://www.ifeng.com",
		"https://www.ctrip.com",
		"https://www.qunar.com",
		"https://www.meituan.com",
		"https://www.dianping.com",
		"https://www.ele.me",
		"https://www.vip.com",
		"https://www.suning.com",
		"https://www.mi.com",
		"https://www.huawei.com",
		"https://www.vivo.com.cn",
		"https://www.oppo.com",
		"https://www.le.com",
		"https://www.ganji.com",
		"https://www.kuaidi100.com",
		"https://www.17173.com",
		"https://www.4399.com",
		"https://www.tgbus.com",
		"https://www.10010.com",
		"https://www.10086.cn",
		"https://www.189.cn",
		"https://www.21cn.com",
		"https://www.chinadaily.com.cn",
		"https://www.xinhuanet.com",
		"https://www.people.com.cn",
		"https://www.thepaper.cn",
		"https://www.guokr.com",
		"https://www.ftchinese.com",
		"https://www.cbnweek.com",
		"https://www.163.com",
		"https://www.sina.com.cn",
		"https://www.sohu.com",
		"https://www.qq.com",
		"https://e.baidu.com",
		"https://hezuo.baidu.com",
		"https://yiyan.baidu.com",
	}

	//设置3s超时，避免请求时间过长
	client := http.Client{
		Timeout: 3 * time.Second,
	}

	//定义chan结构体，url，status
	type Web struct {
		url    string
		status string
	}

	//并发请求，定义一个WaitGroup，用于等待所有请求完成
	var wg sync.WaitGroup

	for _, url := range urls {
		//增加一个等待
		wg.Add(1)
		//启动一个goroutine
		go func(url string) {
			//协程结束时调用Done通知main函数工作已经完成
			defer wg.Done()
			//请求
			resp, err := client.Get(url)
			if err != nil {
				fmt.Println("请求失败", err)
				_, err := db.Exec("INSERT INTO web (url, status) VALUES (?, ?)", url, "请求失败")
				if err != nil {
					return
				}
			} else {
				// 写入数据库
				_, err1 := db.Exec("INSERT INTO web (url, status) VALUES (?, ?)", url, resp.StatusCode)
				if err1 != nil {
					fmt.Println("写入数据库错误:", err1)
					return
				}
			}
		}(url)
	}

	//等待所有请求完成
	wg.Wait()

	//输出耗时
	elapsed := time.Since(start)
	fmt.Println("耗时：", elapsed)
}

sync.WaitGroup+channel

package main

import (
	"database/sql"
	"fmt"
	"net/http"
	"strconv"
	"sync"
	"time"

	_ "github.com/go-sql-driver/mysql"
)

func main() {
	//开始计时
	start := time.Now()

	//链接数据库，用户名：密码@tcp(地址:端口)/数据库名
	dsn := "gin:gin@tcp(4.19.21.17:3306)/gin"
	// 连接到MySQL数据库
	db, err := sql.Open("mysql", dsn)
	if err != nil {
		fmt.Println("数据库连接错误: ", err)
		return
	}
	//关闭数据库
	defer db.Close()

	urls := []string{
		"https://www.baidu.com",
		"https://www.taobao.com",
		"https://www.tmall.com",
		"https://www.jd.com",
		"https://www.sina.com.cn",
		"https://www.sohu.com",
		"https://www.qq.com",
		"https://www.163.com",
		"https://www.zhihu.com",
		"https://www.weibo.com",
		"https://www.bilibili.com",
		"https://www.youku.com",
		"https://www.iqiyi.com",
		"https://www.alipay.com",
		"https://www.dingtalk.com",
		"https://www.wechat.com",
		"https://www.360.cn",
		"https://www.58.com",
		"https://www.liepin.com",
		"https://www.zhaopin.com",
		"https://www.douban.com",
		"https://www.ifeng.com",
		"https://www.ctrip.com",
		"https://www.qunar.com",
		"https://www.meituan.com",
		"https://www.dianping.com",
		"https://www.ele.me",
		"https://www.vip.com",
		"https://www.suning.com",
		"https://www.mi.com",
		"https://www.huawei.com",
		"https://www.vivo.com.cn",
		"https://www.oppo.com",
		"https://www.le.com",
		"https://www.ganji.com",
		"https://www.kuaidi100.com",
		"https://www.17173.com",
		"https://www.4399.com",
		"https://www.tgbus.com",
		"https://www.10010.com",
		"https://www.10086.cn",
		"https://www.189.cn",
		"https://www.21cn.com",
		"https://www.chinadaily.com.cn",
		"https://www.xinhuanet.com",
		"https://www.people.com.cn",
		"https://www.thepaper.cn",
		"https://www.guokr.com",
		"https://www.ftchinese.com",
		"https://www.cbnweek.com",
		"https://www.163.com",
		"https://www.sina.com.cn",
		"https://www.sohu.com",
		"https://www.qq.com",
		"https://e.baidu.com",
		"https://hezuo.baidu.com",
		"https://yiyan.baidu.com",
	}

	//设置3s超时，避免请求时间过长
	client := http.Client{
		Timeout: 3 * time.Second,
	}

	//定义chan结构体，url，status
	type Web struct {
		url    string
		status string
	}

	//并发请求
	var wg sync.WaitGroup
	//创建一个并发请求的channel,缓冲区大小为urls的长度
	results := make(chan *Web, len(urls))

	//遍历urls，k为索引，url为值
	for k, url := range urls {
		//增加一个等待
		wg.Add(1)
		fmt.Println("开始请求：", k, url)
		//启动一个goroutine，传入k和url
		go func(k int, url string) {
			//协程结束时调用Done通知main函数工作已经完成
			defer wg.Done()
			//请求
			resp, err := client.Get(url)
			if err != nil {
				fmt.Println("请求失败", err)
				//设置状态为请求失败
				results <- &Web{url: url, status: "请求失败"}
				return
			}
			//关闭请求
			defer resp.Body.Close()
			fmt.Println("请求完成：", k, url)
			//写入channel
			results <- &Web{url: url, status: strconv.Itoa(resp.StatusCode)}
		}(k, url)
	}

	//统计有多少个请求完成了
	fmt.Println("等待所有请求完成")
	//等待所有请求完成
	wg.Wait()
	//关闭channel
	close(results)
	//统计results的长度
	//fmt.Println("results长度：", len(results))

	fmt.Println("开始写入数据库")
	//遍历channel
	for resp := range results {
		// 写入数据库
		_, err := db.Exec("INSERT INTO web (url, status) VALUES (?, ?)", resp.url, resp.status)
		if err != nil {
			fmt.Println("写入数据库错误:", err)
		}
	}
	fmt.Println("写入数据库完成")

	//输出耗时
	elapsed := time.Since(start)
	fmt.Println("耗时：", elapsed)
}