企业🤖AI智能体构建引擎,智能编排和调试,一键部署,支持私有化部署方案 广告
``` package main import ( "fmt" "io/ioutil" "net/http" "os" "strconv" "time" ) func SpiderPage(i int, page chan int) { //抓取数据 url := "https://tieba.baidu.com/f?kw=%E7%BB%9D%E5%9C%B0%E6%B1%82%E7%94%9F&ie=utf-8&pn="+ strconv.Itoa((i-1)*50) data, err := http_get1(url) if err != nil { fmt.Println("http get err : ", err) return } //保存到文件 f, err := os.Create("第"+strconv.Itoa(i)+"页.html") if err != nil { fmt.Println("写入文件失败 : ", i, err) return } f.WriteString(data) f.Close() page <- i } func work1(start int , end int) { time.Sleep(time.Second * 1) page := make(chan int) for i:=start; i<=end; i++{ go SpiderPage(i, page) } for i:=start; i<=end; i++ { fmt.Printf("第 %d 个页面爬取完成\n", <-page) } } func http_get1(url string)(result string, err error) { resp, err := http.Get(url) if err != nil { return } data, err := ioutil.ReadAll(resp.Body) if err != nil { return } result = string(data) return } func main() { var start, end int fmt.Print("请输入起始页:") fmt.Scan(&start) fmt.Print("请输入终止页:") fmt.Scan(&end) //开始爬 work1(start, end) } ```