py_facebook/spider/proxy_valid.py

import os

import requests
import time
from typing import Optional, Tuple, Dict, Any


class ProxyChecker:
    def __init__(self, timeout: int = 5):
        """
        初始化代理检查器

        Args:
            api_url: 获取代理的API地址
            timeout: 请求超时时间（秒）
        """
        self.timeout = timeout

    def get_proxies_from_api(self) -> list:
        """
        从API获取代理列表

        Returns:
            list: 代理列表，格式为 [{"ip": "x.x.x.x", "port": xxxx}, ...]
        """
        try:
            url = 'https://api.911proxy.com/web_v1/ip/get-ip-v3?app_key=67645499c25418a2669fe1470307fd56&pt=9&num=10&ep=&cc=&state=&city=&life=30&protocol=1&format=json'
            response = requests.get(url=url, timeout=self.timeout, verify=False)
            response.raise_for_status()

            data = response.json()
            # {'code': 200, 'msg': 'SUCCESS', 'data': {'list': ['5.78.16.61:24878', '5.78.16.61:24879', '5.78.16.61:24880', '5.78.16.61:24881', '5.78.16.61:24882', '5.78.16.61:24883', '5.78.16.61:24884', '5.78.16.61:24885', '5.78.16.61:24886', '5.78.16.61:24887', '5.78.16.61:24888', '5.78.16.61:24889', '5.78.16.61:24890', '5.78.16.61:24891', '5.78.16.61:24892', '5.78.16.61:24893', '5.78.16.61:24894', '5.78.16.61:24895', '5.78.16.61:24896', '5.78.16.61:24897', '5.78.16.61:24898', '5.78.16.61:24899', '5.78.16.61:24900', '5.78.16.61:24901', '5.78.16.61:24902', '5.78.16.61:24903', '5.78.16.61:24904', '5.78.16.61:24905', '5.78.16.61:24906', '5.78.16.61:24907', '5.78.16.61:24908', '5.78.16.61:24909', '5.78.16.61:24910', '5.78.16.61:24911', '5.78.16.61:24912', '5.78.16.61:24913', '5.78.16.61:24914', '5.78.16.61:24915', '5.78.16.61:24916', '5.78.16.61:24917', '5.78.16.61:24918', '5.78.16.61:24919', '5.78.16.61:24920', '5.78.16.61:24921', '5.78.16.61:24922', '5.78.16.61:24923', '5.78.16.61:24924', '5.78.16.61:24925', '5.78.16.61:24926', '5.78.16.61:24927', '5.78.16.61:24928', '5.78.16.61:24929', '5.78.16.61:24930', '5.78.16.61:24931', '5.78.16.61:24932', '5.78.16.61:24933', '5.78.16.61:24934', '5.78.16.61:24935', '5.78.16.61:24936', '5.78.16.61:24937', '5.78.16.61:24938', '5.78.16.61:24939', '5.78.16.61:24940', '5.78.16.61:24941', '5.78.16.61:24942', '5.78.16.61:24943', '5.78.16.61:24944', '5.78.16.61:24945', '5.78.16.61:24946', '5.78.16.61:24947', '5.78.16.61:24948', '5.78.16.61:24949', '5.78.16.61:24950', '5.78.16.61:24951', '5.78.16.61:24952', '5.78.16.61:24953', '5.78.16.61:24954', '5.78.16.61:24955', '5.78.16.61:24956', '5.78.16.61:24957', '5.78.16.61:24958', '5.78.16.61:24959', '5.78.16.61:24960', '5.78.16.61:24961', '5.78.16.61:24962', '5.78.16.61:24963', '5.78.16.61:24964', '5.78.16.61:24965', '5.78.16.61:24966', '5.78.16.61:24967', '5.78.16.61:24968', '5.78.16.61:24969', '5.78.16.61:24970', '5.78.16.61:24971', '5.78.16.61:24972', '5.78.16.61:24973', '5.78.16.61:24974', '5.78.16.61:24975', '5.78.16.61:24976', '5.78.16.61:24977', '5.78.16.61:24978', '5.78.16.61:24979', '5.78.16.61:24980', '5.78.16.61:24981', '5.78.16.61:24982', '5.78.16.61:24983', '5.78.16.61:24984', '5.78.16.61:24985', '5.78.16.61:24986', '5.78.16.61:24987', '5.78.16.61:24988', '5.78.16.61:24989', '5.78.16.61:24990', '5.78.16.61:24991', '5.78.16.61:24992', '5.78.16.61:24993', '5.78.16.61:24994', '5.78.16.61:24995', '5.78.16.61:24996', '5.78.16.61:24997', '5.78.16.61:24998', '5.78.16.61:24999', '5.78.16.61:25000', '5.78.16.61:25001', '5.78.16.61:25002', '5.78.16.61:25003', '5.78.16.61:25004', '5.78.16.61:25005', '5.78.16.61:25006', '5.78.16.61:25007', '5.78.16.61:25008', '5.78.16.61:25009', '5.78.16.61:25010', '5.78.16.61:25011', '5.78.16.61:25012', '5.78.16.61:25013', '5.78.16.61:25014', '5.78.16.61:25015', '5.78.16.61:25016', '5.78.16.61:25017', '5.78.16.61:25018', '5.78.16.61:25019', '5.78.16.61:25020', '5.78.16.61:25021', '5.78.16.61:25022', '5.78.16.61:25023', '5.78.16.61:25024', '5.78.16.61:25025', '5.78.16.61:25026', '5.78.16.61:25027', '5.78.16.61:25028', '5.78.16.61:25029', '5.78.16.61:25030', '5.78.16.61:25031', '5.78.16.61:25032', '5.78.16.61:25033', '5.78.16.61:25034', '5.78.16.61:25035', '5.78.16.61:25036', '5.78.16.61:25037', '5.78.16.61:25038', '5.78.16.61:25039', '5.78.16.61:25040', '5.78.16.61:25041', '5.78.16.61:25042', '5.78.16.61:25043', '5.78.16.61:25044', '5.78.16.61:25045', '5.78.16.61:25046', '5.78.16.61:25047', '5.78.16.61:25048', '5.78.16.61:25049', '5.78.16.61:25050', '5.78.16.61:25051', '5.78.16.61:25052', '5.78.16.61:25053', '5.78.16.61:25054', '5.78.16.61:25055', '5.78.16.61:25056', '5.78.16.61:25057', '5.78.16.61:25058', '5.78.16.61:25059', '5.78.16.61:25060', '5.78.16.61:25061', '5.78.16.61:25062', '5.78.16.61:25063', '5.78.16.61:25064', '5.78.16.61:25065', '5.78.16.61:25066', '5.78.16.61:25067', '5.78.16.61:25068', '5.78.16.61:25069', '5.78.16.61:25070', '5.78.16.61:25071', '5.78.16.61:25072', '5.78.16.61:25073', '5.78.16.61:25074', '5.78.16.61:25075', '5.78.16.61:25076', '5.78.16.61:25077', '5.78.16.61:25078', '5.78.16.61:25079', '5.78.16.61:25080', '5.78.16.61:25081', '5.78.16.61:25082', '5.78.16.61:25083', '5.78.16.61:25084', '5.78.16.61:25085', '5.78.16.61:25086', '5.78.16.61:25087', '5.78.16.61:25088', '5.78.16.61:25089', '5.78.16.61:25090', '5.78.16.61:25091', '5.78.16.61:25092', '5.78.16.61:25093', '5.78.16.61:25094', '5.78.16.61:25095', '5.78.16.61:25096', '5.78.16.61:25097', '5.78.16.61:25098', '5.78.16.61:25099', '5.78.16.61:25100', '5.78.16.61:25101', '5.78.16.61:25102', '5.78.16.61:25103', '5.78.16.61:25104', '5.78.16.61:25105', '5.78.16.61:25106', '5.78.16.61:25107', '5.78.16.61:25108', '5.78.16.61:25109', '5.78.16.61:25110', '5.78.16.61:25111', '5.78.16.61:25112', '5.78.16.61:25113', '5.78.16.61:25114', '5.78.16.61:25115', '5.78.16.61:25116', '5.78.16.61:25117', '5.78.16.61:25118', '5.78.16.61:25119', '5.78.16.61:25120', '5.78.16.61:25121', '5.78.16.61:25122', '5.78.16.61:25123', '5.78.16.61:25124', '5.78.16.61:25125', '5.78.16.61:25126', '5.78.16.61:25127', '5.78.16.61:25128', '5.78.16.61:25129', '5.78.16.61:25130', '5.78.16.61:25131', '5.78.16.61:25132', '5.78.16.61:25133', '5.78.16.61:25134', '5.78.16.61:25135', '5.78.16.61:25136', '5.78.16.61:25137', '5.78.16.61:25138', '5.78.16.61:25139', '5.78.16.61:25140', '5.78.16.61:25141', '5.78.16.61:25142', '5.78.16.61:25143', '5.78.16.61:25144', '5.78.16.61:25145', '5.78.16.61:25146', '5.78.16.61:25147', '5.78.16.61:25148', '5.78.16.61:25149', '5.78.16.61:25150', '5.78.16.61:25151', '5.78.16.61:25152', '5.78.16.61:25153', '5.78.16.61:25154', '5.78.16.61:25155', '5.78.16.61:25156', '5.78.16.61:25157', '5.78.16.61:25158', '5.78.16.61:25159', '5.78.16.61:25160', '5.78.16.61:25161', '5.78.16.61:25162', '5.78.16.61:25163', '5.78.16.61:25164', '5.78.16.61:25165', '5.78.16.61:25166', '5.78.16.61:25167', '5.78.16.61:25168', '5.78.16.61:25169', '5.78.16.61:25170', '5.78.16.61:25171', '5.78.16.61:25172', '5.78.16.61:25173', '5.78.16.61:25174', '5.78.16.61:25175', '5.78.16.61:25176', '5.78.16.61:25177', '5.78.16.61:25178', '5.78.16.61:25179', '5.78.16.61:25180', '5.78.16.61:25181', '5.78.16.61:25182', '5.78.16.61:25183', '5.78.16.61:25184', '5.78.16.61:25185', '5.78.16.61:25186', '5.78.16.61:25187', '5.78.16.61:25188', '5.78.16.61:25189', '5.78.16.61:25190', '5.78.16.61:25191', '5.78.16.61:25192', '5.78.16.61:25193', '5.78.16.61:25194', '5.78.16.61:25195', '5.78.16.61:25196', '5.78.16.61:25197', '5.78.16.61:25198', '5.78.16.61:25199', '5.78.16.61:25200', '5.78.16.61:25201', '5.78.16.61:25202', '5.78.16.61:25203', '5.78.16.61:25204', '5.78.16.61:25205', '5.78.16.61:25206', '5.78.16.61:25207', '5.78.16.61:25208', '5.78.16.61:25209', '5.78.16.61:25210', '5.78.16.61:25211', '5.78.16.61:25212', '5.78.16.61:25213', '5.78.16.61:25214', '5.78.16.61:25215', '5.78.16.61:25216', '5.78.16.61:25217', '5.78.16.61:25218', '5.78.16.61:25219', '5.78.16.61:25220', '5.78.16.61:25221', '5.78.16.61:25222', '5.78.16.61:25223', '5.78.16.61:25224', '5.78.16.61:25225', '5.78.16.61:25226', '5.78.16.61:25227', '5.78.16.61:25228', '5.78.16.61:25229', '5.78.16.61:25230', '5.78.16.61:25231', '5.78.16.61:25232', '5.78.16.61:25233', '5.78.16.61:25234', '5.78.16.61:25235', '5.78.16.61:25236', '5.78.16.61:25237', '5.78.16.61:25238', '5.78.16.61:25239', '5.78.16.61:25240', '5.78.16.61:25241', '5.78.16.61:25242', '5.78.16.61:25243', '5.78.16.61:25244', '5.78.16.61:25245', '5.78.16.61:25246', '5.78.16.61:25247', '5.78.16.61:25248', '5.78.16.61:25249', '5.78.16.61:25250', '5.78.16.61:25251', '5.78.16.61:25252', '5.78.16.61:25253', '5.78.16.61:25254', '5.78.16.61:25255', '5.78.16.61:25256', '5.78.16.61:25257', '5.78.16.61:25258', '5.78.16.61:25259', '5.78.16.61:25260', '5.78.16.61:25261', '5.78.16.61:25262', '5.78.16.61:25263', '5.78.16.61:25264', '5.78.16.61:25265', '5.78.16.61:25266', '5.78.16.61:25267', '5.78.16.61:25268', '5.78.16.61:25269', '5.78.16.61:25270', '5.78.16.61:25271', '5.78.16.61:25272', '5.78.16.61:25273', '5.78.16.61:25274', '5.78.16.61:25275', '5.78.16.61:25276', '5.78.16.61:25277', '5.78.16.61:25278', '5.78.16.61:25279', '5.78.16.61:25280', '5.78.16.61:25281', '5.78.16.61:25282', '5.78.16.61:25283', '5.78.16.61:25284', '5.78.16.61:25285', '5.78.16.61:25286', '5.78.16.61:25287', '5.78.16.61:25288', '5.78.16.61:25289', '5.78.16.61:25290', '5.78.16.61:25291', '5.78.16.61:25292', '5.78.16.61:25293', '5.78.16.61:25294', '5.78.16.61:25295', '5.78.16.61:25296', '5.78.16.61:25297', '5.78.16.61:25298', '5.78.16.61:25299', '5.78.16.61:25300', '5.78.16.61:25301', '5.78.16.61:25302', '5.78.16.61:25303', '5.78.16.61:25304', '5.78.16.61:25305', '5.78.16.61:25306', '5.78.16.61:25307', '5.78.16.61:25308', '5.78.16.61:25309', '5.78.16.61:25310', '5.78.16.61:25311', '5.78.16.61:25312', '5.78.16.61:25313', '5.78.16.61:25314', '5.78.16.61:25315', '5.78.16.61:25316', '5.78.16.61:25317', '5.78.16.61:25318', '5.78.16.61:25319', '5.78.16.61:25320', '5.78.16.61:25321', '5.78.16.61:25322', '5.78.16.61:25323', '5.78.16.61:25324', '5.78.16.61:25325', '5.78.16.61:25326', '5.78.16.61:25327', '5.78.16.61:25328', '5.78.16.61:25329', '5.78.16.61:25330', '5.78.16.61:25331', '5.78.16.61:25332', '5.78.16.61:25333', '5.78.16.61:25334', '5.78.16.61:25335', '5.78.16.61:25336', '5.78.16.61:25337', '5.78.16.61:25338', '5.78.16.61:25339', '5.78.16.61:25340', '5.78.16.61:25341', '5.78.16.61:25342', '5.78.16.61:25343', '5.78.16.61:25344', '5.78.16.61:25345', '5.78.16.61:25346', '5.78.16.61:25347', '5.78.16.61:25348', '5.78.16.61:25349', '5.78.16.61:25350', '5.78.16.61:25351', '5.78.16.61:25352', '5.78.16.61:25353', '5.78.16.61:25354', '5.78.16.61:25355', '5.78.16.61:25356', '5.78.16.61:25357', '5.78.16.61:25358', '5.78.16.61:25359', '5.78.16.61:25360', '5.78.16.61:25361', '5.78.16.61:25362', '5.78.16.61:25363', '5.78.16.61:25364', '5.78.16.61:25365', '5.78.16.61:25366', '5.78.16.61:25367', '5.78.16.61:25368', '5.78.16.61:25369', '5.78.16.61:25370', '5.78.16.61:25371', '5.78.16.61:25372', '5.78.16.61:25373', '5.78.16.61:25374', '5.78.16.61:25375', '5.78.16.61:25376', '5.78.16.61:25377']}, 'request_id': 'lVJGmO2Z3AKQkr5tzZEJ2xcHQ_91O7bR'}
            if data.get("code") == 200 and data.get("msg"):
                return [{"ip": ip_port.split(":")[0], "port": int(ip_port.split(":")[1])} for ip_port in data.get("data").get("list", [])]
            else:
                print(f"API返回错误: {data.get('msg')}")
                return []

        except requests.exceptions.RequestException as e:
            print(f"获取代理失败: {e}")
            return []
        except ValueError as e:
            print(f"解析JSON失败: {e}")
            return []

    def check_proxy(self, proxy: Dict[str, Any]) -> bool:
        """
        检查单个代理是否有效

        Args:
            proxy: 代理字典，包含ip和port

        Returns:
            bool: 代理是否有效
        """
        if proxy["port"] != '':
            proxy_url = f"http://{proxy['ip']}:{proxy['port']}"
        else:
            proxy_url = f"http://{proxy['ip']}"
        proxies = {
            "http": proxy_url,
            "https": proxy_url
        }

        test_urls = [
            "https://www.facebook.com"
        ]

        for test_url in test_urls:
            try:
                start_time = time.time()
                response = requests.get(
                    test_url,
                    proxies=proxies,
                    timeout=self.timeout,
                    headers={
                        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
                    }
                )
                response_time = time.time() - start_time

                if response.status_code == 200:
                    print(f"代理 {proxy_url} 有效，响应时间: {response_time:.2f}s，测试URL: {test_url}")
                    return True

            except (requests.exceptions.RequestException, requests.exceptions.ProxyError):
                continue

        return False

    def get_valid_proxy(self) -> Optional[Tuple[str, int]]:
        """
        获取并验证代理，返回第一个有效的代理

        Returns:
            tuple: (ip, port) 或 None（如果没有有效代理）
        """
        print("正在从API获取代理列表...")
        # 本地开发模式
        if os.getenv("dev"):
            return "127.0.0.1", 1080

        proxies = self.get_proxies_from_api()

        if not proxies:
            print("未获取到代理列表")
            return None

        print(f"获取到 {len(proxies)} 个代理，开始验证...")

        for proxy in proxies:
            print(f"正在验证代理: {proxy['ip']}:{proxy['port']}")
            if self.check_proxy(proxy):
                print(f"找到有效代理: {proxy['ip']}:{proxy['port']}")
                return (proxy['ip'], proxy['port'])

        print("所有代理均无效")
        return None

    def get_valid_proxy_dict(self) -> Optional[Dict[str, str]]:
        """
        获取有效的代理字典格式

        Returns:
            dict: {"http": "http://ip:port", "https": "http://ip:port"} 或 None
        """
        result = self.get_valid_proxy()
        if result:
            ip, port = result
            proxy_url = f"http://{ip}:{port}"
            return {
                "http": proxy_url,
                "https": proxy_url
            }
        return None

    def get_valid_proxy_url(self) -> Optional[str]:
        """
        获取有效的代理URL格式 (http://ip:port)

        Returns:
            str: "http://ip:port" 或 None
        """
        result = self.get_valid_proxy()
        if result:
            ip, port = result
            return f"http://{ip}:{port}"
        return None


# 使用示例
if __name__ == "__main__":
    # 你的API地址
    API_URL = "你的API地址"  # 替换为实际的API地址

    checker = ProxyChecker(timeout=8)

    # 获取有效代理（返回元组格式）
    valid_proxy = checker.get_valid_proxy()
    if valid_proxy:
        ip, port = valid_proxy
        print(f"\n最终选择的代理: {ip}:{port}")

        # 使用代理示例
        proxies = {
            "http": f"http://{ip}:{port}",
            "https": f"http://{ip}:{port}"
        }

        try:
            response = requests.get("http://httpbin.org/ip", proxies=proxies, timeout=10)
            print(f"使用代理请求测试: {response.text}")
        except Exception as e:
            print(f"测试请求失败: {e}")
    else:
        print("未找到有效代理")

    # 或者直接获取代理字典格式
    # proxy_dict = checker.get_valid_proxy_dict()
    # if proxy_dict:
    #     print(f"代理字典: {proxy_dict}")