免费代理IP到底能不能用？一套自动化方案搞定从找到IP资源到可用

#推荐用Python脚本实现动态抓取，以66免费代理为例，核心代码如下，可快速爬取IP与端口并整理成列表：#
import requests
from bs4 import BeautifulSoup

def scrape_free_proxies():
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
    }
    url = 'https://www.66ip.cn/'
    response = requests.get(url, headers=headers)
    soup = BeautifulSoup(response.text, 'html.parser')
    
    proxies = []
    for tr in soup.find_all('tr')[1:]:  # 跳过表头
        tds = tr.find_all('td')
        if len(tds) >= 2:
            ip = tds[0].text.strip()
            port = tds[1].text.strip()
            proxies.append(f'{ip}:{port}')
    return proxies

#核心代码实现（整合验证、存储与更新功能）：#
import pymysql
import redis
import requests
from concurrent.futures import ThreadPoolExecutor

class ProxyPool:
    def __init__(self):
        # MySQL配置（存储原始IP）
        self.db = pymysql.connect(
            host='localhost', user='root', password='password', database='proxy_pool'
        )
        # Redis配置（存储可用IP，便于快速调度）
        self.r = redis.Redis(host='localhost', port=6379, db=0)
        
    def validate_proxy(self, proxy):
        # 验证IP连通性与可用性
        try:
            proxies = {'http': f'http://{proxy}', 'https': f'https://{proxy}'}
            response = requests.get(
                'http://httpbin.org/ip', proxies=proxies, timeout=5,
                headers={'User-Agent': 'Mozilla/5.0'}
            )
            return response.status_code == 200
        except:
            return False
    
    def refresh_pool(self):
        # 从MySQL取待验证IP，多线程高效校验
        with self.db.cursor() as cursor:
            cursor.execute("SELECT proxy FROM raw_proxies WHERE validated=0 LIMIT 1000")
            raw_proxies = [row[0] for row in cursor.fetchall()]
        
        # 50线程并发验证，提升效率
        with ThreadPoolExecutor(max_workers=50) as executor:
            results = executor.map(self.validate_proxy, raw_proxies)
            valid_proxies = [p for p, valid in zip(raw_proxies, results) if valid]
        
        # 更新可用IP到Redis和MySQL状态
        if valid_proxies:
            self.r.rpush('proxies:active', *valid_proxies)
            with self.db.cursor() as cursor:
                for proxy in valid_proxies:
                    cursor.execute("UPDATE raw_proxies SET validated=1 WHERE proxy=%s", (proxy,))
                self.db.commit()

import time
import statistics
import requests
from concurrent.futures import ThreadPoolExecutor

class ProxyTester:
    def __init__(self):
        self.test_url = 'http://httpbin.org/ip'
        self.success_threshold = 0.95  # 95%成功率为合格标准
    
    def test_single_proxy(self, proxy):
        proxies = {'http': f'http://{proxy}', 'https': f'https://{proxy}'}
        latencies = []
        success_count = 0
        
        # 连续100次请求测试稳定性
        for _ in range(100):
            try:
                start_time = time.time()
                response = requests.get(self.test_url, proxies=proxies, timeout=5)
                if response.status_code == 200:
                    success_count += 1
                    latencies.append((time.time() - start_time) * 1000)  # 转换为毫秒
            except:
                continue
        
        # 计算测试指标，返回结果
        if success_count == 0:
            return None
        success_rate = success_count / 100
        avg_latency = statistics.mean(latencies) if latencies else float('inf')
        return {
            'proxy': proxy, 'success_rate': success_rate,
            'avg_latency': avg_latency, 'is_valid': success_rate >= self.success_threshold
        }
    
    def batch_test(self, proxies):
        # 批量测试IP，20线程并发提升效率
        results = []
        with ThreadPoolExecutor(max_workers=20) as executor:
            futures = [executor.submit(self.test_single_proxy, p) for p in proxies]
            for future in futures:
                if result := future.result():
                    results.append(result)
        return results