--- imagedl/modules/sources/duckduckgo.py.orig +++ imagedl/modules/sources/duckduckgo.py @@ -7,7 +7,6 @@ Charles的皮卡丘 ''' import math -import primp import random import requests import json_repair @@ -15,6 +14,7 @@ from .base import BaseImageClient from urllib.parse import urlencode from fake_useragent import UserAgent +from curl_cffi import requests as curl_requests from ..utils import Filter, ImageInfo @@ -51,8 +51,8 @@ return search_results '''_initsession''' def _initsession(self): - self.session = primp.Client(proxy=None, timeout=30, impersonate="random", impersonate_os="random", verify=True) - self.session.headers_update(self.default_headers) + self.session = curl_requests.Session(impersonate="chrome", timeout=30, verify=True) + self.session.headers.update(self.default_headers) '''_parsesearchresult''' def _parsesearchresult(self, search_result: str) -> list[ImageInfo]: # parse json text in safety @@ -129,7 +129,9 @@ for _ in range(self.max_retries): if not self.maintain_session: self._initsession(); self.random_update_ua and self.session.headers.update({'User-Agent': UserAgent().random}) proxies, resp = kwargs.pop('proxies', None) or self._autosetproxies(), None - if proxies: self.session.proxy = random.choice(list(proxies.values())) if isinstance(proxies, dict) else proxies + if proxies: + proxy_url = random.choice(list(proxies.values())) if isinstance(proxies, dict) else proxies + self.session.proxies = {"http": proxy_url, "https": proxy_url} try: (resp := self.session.request(method, url, **kwargs)).raise_for_status() except Exception as err: self.logger_handle.error(f'{self.source}.request >>> {url} (Error: {err}; status={getattr(locals().get("resp"), "status_code", None)})', disable_print=self.disable_print); continue return resp --- imagedl/modules/sources/wikipedia.py.orig +++ imagedl/modules/sources/wikipedia.py @@ -7,12 +7,12 @@ Charles的皮卡丘 ''' import math -import primp import random import json_repair from ..utils import ImageInfo from .base import BaseImageClient from fake_useragent import UserAgent +from curl_cffi import requests from urllib.parse import quote, urlencode @@ -27,8 +27,8 @@ self._initsession() '''_initsession''' def _initsession(self): - self.session = primp.Client(proxy=None, timeout=30, impersonate="random", impersonate_os="random", verify=True) - self.session.headers_update(self.default_headers) + self.session = requests.Session(impersonate="chrome", timeout=30, verify=True) + self.session.headers.update(self.default_headers) '''_parsesearchresult''' def _parsesearchresult(self, search_result: str) -> list[ImageInfo]: # parse json text in safety @@ -57,7 +57,9 @@ for _ in range(self.max_retries): if not self.maintain_session: self._initsession(); self.random_update_ua and self.session.headers.update({'User-Agent': UserAgent().random}) proxies, resp = kwargs.pop('proxies', None) or self._autosetproxies(), None - if proxies: self.session.proxy = random.choice(list(proxies.values())) if isinstance(proxies, dict) else proxies + if proxies: + proxy_url = random.choice(list(proxies.values())) if isinstance(proxies, dict) else proxies + self.session.proxies = {"http": proxy_url, "https": proxy_url} try: (resp := self.session.request(method, url, **kwargs)).raise_for_status() except Exception as err: self.logger_handle.error(f'{self.source}.request >>> {url} (Error: {err}; status={getattr(locals().get("resp"), "status_code", None)})', disable_print=self.disable_print); continue return resp --- imagedl/modules/sources/yahoo.py.orig +++ imagedl/modules/sources/yahoo.py @@ -7,13 +7,13 @@ Charles的皮卡丘 ''' import math -import primp import random import json_repair from bs4 import BeautifulSoup from urllib.parse import quote from .base import BaseImageClient from fake_useragent import UserAgent +from curl_cffi import requests from ..utils import Filter, ImageInfo @@ -33,8 +33,8 @@ self._initsession() '''_initsession''' def _initsession(self): - self.session = primp.Client(proxy=None, timeout=30, impersonate="random", impersonate_os="random", verify=True) - self.session.headers_update(self.default_headers) + self.session = requests.Session(impersonate="chrome", timeout=30, verify=True) + self.session.headers.update(self.default_headers) '''_parsesearchresult''' def _parsesearchresult(self, search_result: str) -> list[ImageInfo]: soup, image_infos, seen = BeautifulSoup(search_result, "lxml"), [], set() @@ -88,7 +88,9 @@ for _ in range(self.max_retries): if not self.maintain_session: self._initsession(); self.random_update_ua and self.session.headers.update({'User-Agent': UserAgent().random}) proxies, resp = kwargs.pop('proxies', None) or self._autosetproxies(), None - if proxies: self.session.proxy = random.choice(list(proxies.values())) if isinstance(proxies, dict) else proxies + if proxies: + proxy_url = random.choice(list(proxies.values())) if isinstance(proxies, dict) else proxies + self.session.proxies = {"http": proxy_url, "https": proxy_url} try: (resp := self.session.request(method, url, **kwargs)).raise_for_status() except Exception as err: self.logger_handle.error(f'{self.source}.request >>> {url} (Error: {err}; status={getattr(locals().get("resp"), "status_code", None)})', disable_print=self.disable_print); continue return resp