diff --git a/llarp/constants/platform.hpp b/llarp/constants/platform.hpp index 7e4f21a..20efb47 100644 --- a/llarp/constants/platform.hpp +++ b/llarp/constants/platform.hpp @@ -21,6 +21,15 @@ namespace llarp::platform #endif ; + /// are we an apple platform ? + inline constexpr bool is_apple = +#ifdef __APPLE__ + true +#else + false +#endif + ; + /// are we freebsd ? inline constexpr bool is_freebsd = #ifdef __FreeBSD__ diff --git a/llarp/dns/apple_platform.hpp b/llarp/dns/apple_platform.hpp new file mode 100644 index 0000000..d77ecee --- /dev/null +++ b/llarp/dns/apple_platform.hpp @@ -0,0 +1,180 @@ +#pragma once +#include "platform.hpp" + +#include + +// The implementation must only be parsed on Apple platforms: it uses popen(3) +// and two-argument mkdir(2), neither of which exists on Windows toolchains, +// and dns/server.cpp includes this header unconditionally. +#ifdef __APPLE__ + +#include +#include +#include +#include +#include +#include + +namespace llarp::dns +{ + namespace apple + { + /// DNS platform for a native (non NetworkExtension) macOS daemon, running as + /// root. Split-tunnel DNS (.loki/.snode) is done with /etc/resolver files, + /// which the macOS resolver picks up automatically and which require no + /// cleanup beyond deleting the files. Global DNS override (exit mode) is + /// done by rewriting the primary network service's DNS entry in the + /// SystemConfiguration dynamic store via scutil, saving the old servers so + /// they can be restored when exit mode is turned off. + class Platform : public I_Platform + { + std::optional m_SavedServiceID; + std::vector m_SavedServers; + bool m_GlobalSet{false}; + + static std::string + RunAndCollect(const std::string& cmd) + { + std::string out; + if (FILE* f = ::popen(cmd.c_str(), "r")) + { + char buf[512]; + while (size_t n = ::fread(buf, 1, sizeof(buf), f)) + out.append(buf, n); + ::pclose(f); + } + return out; + } + + /// ask configd which network service currently owns the default route + static std::optional + PrimaryServiceID() + { + const auto out = + RunAndCollect("echo 'show State:/Network/Global/IPv4' | /usr/sbin/scutil"); + std::istringstream lines{out}; + for (std::string line; std::getline(lines, line);) + { + if (auto pos = line.find("PrimaryService"); pos != std::string::npos) + { + if (auto colon = line.find(':', pos); colon != std::string::npos) + { + auto id = line.substr(colon + 1); + // trim whitespace + id.erase(0, id.find_first_not_of(" \t")); + id.erase(id.find_last_not_of(" \t\r\n") + 1); + if (not id.empty()) + return id; + } + } + } + return std::nullopt; + } + + static std::vector + ServiceDNSServers(const std::string& service_id) + { + std::vector servers; + const auto out = RunAndCollect( + "echo 'show State:/Network/Service/" + service_id + "/DNS' | /usr/sbin/scutil"); + // ServerAddresses entries look like " 0 : 192.168.1.1" + std::istringstream lines{out}; + bool in_servers = false; + for (std::string line; std::getline(lines, line);) + { + if (line.find("ServerAddresses") != std::string::npos) + { + in_servers = true; + continue; + } + if (not in_servers) + continue; + if (line.find('}') != std::string::npos) + break; + if (auto colon = line.find(':'); colon != std::string::npos) + { + auto addr = line.substr(colon + 1); + addr.erase(0, addr.find_first_not_of(" \t")); + addr.erase(addr.find_last_not_of(" \t\r\n") + 1); + if (not addr.empty()) + servers.push_back(std::move(addr)); + } + } + return servers; + } + + static void + SetServiceDNSServers(const std::string& service_id, const std::vector& servers) + { + std::string script = "d.init\nd.add ServerAddresses *"; + for (const auto& s : servers) + script += " " + s; + script += "\nset State:/Network/Service/" + service_id + "/DNS\nquit\n"; + + if (FILE* f = ::popen("/usr/sbin/scutil", "w")) + { + ::fwrite(script.data(), 1, script.size(), f); + if (::pclose(f) != 0) + throw std::runtime_error{"scutil failed setting DNS for service " + service_id}; + } + else + throw std::runtime_error{"cannot spawn scutil"}; + } + + static void + WriteResolverFile(const std::string& tld, const std::string& ip, uint16_t port) + { + ::mkdir("/etc/resolver", 0755); // may already exist + const auto path = "/etc/resolver/" + tld; + std::ofstream f{path, std::ios::trunc}; + if (not f) + throw std::runtime_error{"cannot write " + path}; + f << "nameserver " << ip << "\n"; + if (port != 53) + f << "port " << port << "\n"; + } + + public: + virtual ~Platform() = default; + + void + set_resolver(unsigned int, llarp::SockAddr dns, bool global) override + { + const auto ip = dns.hostString(false); + const auto port = dns.getPort(); + + // split-horizon resolution for our TLDs always works via resolver files: + for (const auto* tld : {"loki", "snode"}) + WriteResolverFile(tld, ip, port); + + if (global and not m_GlobalSet) + { + auto service = PrimaryServiceID(); + if (not service) + throw std::runtime_error{"cannot determine primary network service"}; + m_SavedServiceID = service; + m_SavedServers = ServiceDNSServers(*service); + SetServiceDNSServers(*service, {ip}); + m_GlobalSet = true; + } + else if (not global and m_GlobalSet) + { + if (m_SavedServiceID and not m_SavedServers.empty()) + SetServiceDNSServers(*m_SavedServiceID, m_SavedServers); + m_GlobalSet = false; + } + } + }; + } // namespace apple + + using Apple_Platform_t = apple::Platform; +} // namespace llarp::dns + +#else // !__APPLE__ + +namespace llarp::dns +{ + using Apple_Platform_t = Null_Platform; +} // namespace llarp::dns + +#endif // __APPLE__ diff --git a/llarp/dns/server.cpp b/llarp/dns/server.cpp index bf7c1a5..05b3635 100644 --- a/llarp/dns/server.cpp +++ b/llarp/dns/server.cpp @@ -20,6 +20,7 @@ #include #include "sd_platform.hpp" #include "nm_platform.hpp" +#include "apple_platform.hpp" namespace llarp::dns { @@ -662,6 +663,8 @@ namespace llarp::dns plat->add_impl(std::make_unique()); plat->add_impl(std::make_unique()); } + if constexpr (llarp::platform::is_apple) + plat->add_impl(std::make_unique()); return plat; } diff --git a/llarp/vpn/apple.hpp b/llarp/vpn/apple.hpp index 2bfb3bd..30dc6b8 100644 --- a/llarp/vpn/apple.hpp +++ b/llarp/vpn/apple.hpp @@ -3,7 +3,9 @@ #include "platform.hpp" #include "common.hpp" #include +#include #include +#include #include #include @@ -13,6 +15,7 @@ #include #include #include +#include #include #include @@ -32,20 +35,67 @@ #include #include #include +#include #include +#include namespace llarp::vpn { + // The lo0 host route for the tun address must be removed when the daemon + // exits, but the AppleInterface object is retained by event-loop handles + // beyond context teardown, so its destructor does not reliably run before + // process exit. Track added routes and flush them via atexit(3); the + // handler must not log, as logging may already be torn down by then. + namespace apple_route_cleanup + { + inline std::vector& + Routes() + { + // Deliberately heap-allocated and never freed: atexit handlers and + // static destructors run interleaved in reverse registration order, so + // a plain static vector could be destroyed before Flush() runs and it + // would then iterate a dead object. A leaked vector has no destructor + // to register, making Flush() safe no matter when it fires. + static auto* routes = new std::vector(); + return *routes; + } + + inline void + Flush() + { + for (const auto& ip : Routes()) + ::system(("/sbin/route -n delete -host " + ip + " -interface lo0").c_str()); + Routes().clear(); + } + + inline void + Track(std::string ip) + { + static bool registered = (::atexit(&Flush), true); + (void)registered; + Routes().push_back(std::move(ip)); + } + } // namespace apple_route_cleanup + + // UTUN_OPT_IFNAME from , hardcoded so we don't depend on that + // header existing in older SDKs. + inline constexpr int apple_utun_opt_ifname = 2; + class AppleInterface : public NetworkInterface { std::unique_ptr m_FD; - static int - Exec(const std::string& cmd) + static void + Exec(const std::string& cmd, bool must_succeed = true) { - return system(cmd.c_str()); + static auto logcat = log::Cat("vpn.apple"); + log::info(logcat, "exec: {}", cmd); + if (int ret = ::system(cmd.c_str()); ret != 0 and must_succeed) + throw std::runtime_error{"command failed (" + std::to_string(ret) + "): " + cmd}; } + friend class AppleRouteManager; + public: AppleInterface(InterfaceInfo info) : NetworkInterface{std::move(info)} @@ -68,7 +118,7 @@ namespace llarp::vpn addr.sc_len = sizeof(addr); addr.sc_family = AF_SYSTEM; addr.ss_sysaddr = AF_SYS_CONTROL; - addr.sc_unit = 0; + addr.sc_unit = 0; // kernel picks the first free utunN if (::connect(m_FD->fd(), (sockaddr*)&addr, sizeof(addr)) < 0) { @@ -78,7 +128,8 @@ namespace llarp::vpn } uint32_t namesz = IFNAMSIZ; std::array name{}; - if (::getsockopt(m_FD->fd(), SYSPROTO_CONTROL, 2, name.data(), &namesz) < 0) + if (::getsockopt(m_FD->fd(), SYSPROTO_CONTROL, apple_utun_opt_ifname, name.data(), &namesz) + < 0) { m_FD.reset(); throw std::runtime_error{ @@ -90,6 +141,7 @@ namespace llarp::vpn auto& m_IfName = m_Info.ifname; m_IfName = name.data(); + m_Info.index = if_nametoindex(m_IfName.c_str()); for (const auto& ifaddr : m_Info.addrs) { if (ifaddr.fam == AF_INET) @@ -97,22 +149,49 @@ namespace llarp::vpn const huint32_t addr = net::TruncateV6(ifaddr.range.addr); const huint32_t netmask = net::TruncateV6(ifaddr.range.netmask_bits); const huint32_t daddr = addr & netmask; + // utun is point-to-point only; use the OpenVPN-style "topology subnet" + // trick: /32 p2p pair to the network base address, then route the whole + // range at the interface. Exec( "/sbin/ifconfig " + m_IfName + " " + addr.ToString() + " " + daddr.ToString() - + " mtu 1500 netmask 255.255.255.255 up"); + + " mtu " + std::to_string(m_Info.mtu) + " netmask 255.255.255.255 up"); Exec( - "/sbin/route add " + daddr.ToString() + " -netmask " + netmask.ToString() + "/sbin/route -n add -net " + daddr.ToString() + " -netmask " + netmask.ToString() + " -interface " + m_IfName); - Exec("/sbin/route add " + addr.ToString() + " -interface lo0"); + // our own tun address is local, macOS wants it via loopback: + Exec("/sbin/route -n add -host " + addr.ToString() + " -interface lo0"); + apple_route_cleanup::Track(addr.ToString()); } else if (ifaddr.fam == AF_INET6) { - Exec("/sbin/ifconfig " + m_IfName + " inet6 " + ifaddr.range.ToString()); + const auto prefixlen = bits::count_bits(ifaddr.range.netmask_bits); + Exec( + "/sbin/ifconfig " + m_IfName + " inet6 " + ifaddr.range.addr.ToString() + + " prefixlen " + std::to_string(prefixlen) + " up"); + Exec( + "/sbin/route -n add -inet6 -net " + ifaddr.range.addr.ToString() + " -prefixlen " + + std::to_string(prefixlen) + " -interface " + m_IfName, + false); } } } - ~AppleInterface() override = default; + ~AppleInterface() override + { + // Normally unreachable before process exit (see apple_route_cleanup); + // when it does run, clean up here and de-register so the atexit flush + // does not delete the same routes twice. + auto& routes = apple_route_cleanup::Routes(); + for (const auto& ifaddr : m_Info.addrs) + { + if (ifaddr.fam == AF_INET) + { + const auto ip = net::TruncateV6(ifaddr.range.addr).ToString(); + Exec("/sbin/route -n delete -host " + ip + " -interface lo0", false); + routes.erase(std::remove(routes.begin(), routes.end(), ip), routes.end()); + } + } + } int PollFD() const override @@ -126,22 +205,27 @@ namespace llarp::vpn constexpr int uintsize = sizeof(unsigned int); net::IPPacket pkt{net::IPPacket::MaxSize}; - // Prepare storage for header + max-size packet. + // Each utun datagram is prefixed with a 4-byte address family header. unsigned int pktinfo = 0; std::array vecs = {iovec{&pktinfo, uintsize}, iovec{pkt.data(), pkt.size()}}; - int sz = ::readv(m_FD->fd(), vecs.data(), vecs.size()); + auto sz = ::readv(m_FD->fd(), vecs.data(), vecs.size()); if (sz >= uintsize) { pkt.truncate(sz - uintsize); // shrink to actual size } - else if (errno == EAGAIN || errno == EWOULDBLOCK) + else if (sz < 0) { - pkt.truncate(0); - errno = 0; + if (errno == EAGAIN || errno == EWOULDBLOCK) + { + pkt.truncate(0); + errno = 0; + } + else + throw std::error_code{errno, std::system_category()}; } - else + else // 0 <= sz < uintsize: short read of the AF header; drop it { - throw std::error_code{errno, std::system_category()}; + pkt.truncate(0); } return pkt; } @@ -170,85 +254,178 @@ namespace llarp::vpn class AppleRouteManager : public IRouteManager { + static void + Exec(const std::string& cmd, bool must_succeed = true) + { + AppleInterface::Exec(cmd, must_succeed); + } + + static std::string + FamilyFlag(const net::ipaddr_t& ip) + { + if (std::holds_alternative(ip)) + return "-inet6 "; + return ""; + } + public: AppleRouteManager() = default; ~AppleRouteManager() override = default; - // Add a route to a specific IP via a gateway + // Add a first-hop hole-poke route to a specific IP via a physical gateway void AddRoute(net::ipaddr_t ip, net::ipaddr_t gateway) override { - std::string cmd = - "/sbin/route add -host " + llarp::net::ToString(ip) + " " + llarp::net::ToString(gateway); - int ret = std::system(cmd.c_str()); - if (ret != 0) - throw std::runtime_error("AddRoute failed: " + cmd); + Exec( + "/sbin/route -n add " + FamilyFlag(ip) + "-host " + llarp::net::ToString(ip) + " " + + llarp::net::ToString(gateway)); } void DelRoute(net::ipaddr_t ip, net::ipaddr_t gateway) override { - std::string cmd = "/sbin/route delete -host " + llarp::net::ToString(ip) + " " - + llarp::net::ToString(gateway); - int ret = std::system(cmd.c_str()); - if (ret != 0) - throw std::runtime_error("DelRoute failed: " + cmd); + Exec( + "/sbin/route -n delete " + FamilyFlag(ip) + "-host " + llarp::net::ToString(ip) + " " + + llarp::net::ToString(gateway), + false); } - // Add a default route via the VPN interface's first IPv4 address + // Capture all traffic at the tun interface WITHOUT touching the existing + // default route: two /1 routes (plus four /2s for IPv6) are more specific + // than "default", so the original gateway stays in the table for the + // hole-poked first-hop routes added above. void AddDefaultRouteViaInterface(NetworkInterface& vpn) override { - const auto& info = vpn.Info(); - if (info.addrs.empty()) - throw std::runtime_error("No interface addresses found"); - - std::string gateway = info.addrs[0].range.addr.ToString(); - std::string cmd = "/sbin/route add default " + gateway; - int ret = std::system(cmd.c_str()); - if (ret != 0) - throw std::runtime_error("AddDefaultRouteViaInterface failed: " + cmd); + const auto& ifname = vpn.Info().ifname; + for (const auto* range : {"0.0.0.0/1", "128.0.0.0/1"}) + Exec(std::string{"/sbin/route -n add -net "} + range + " -interface " + ifname); + + bool have_v6 = false; + for (const auto& addr : vpn.Info().addrs) + have_v6 |= (addr.fam == AF_INET6); + if (have_v6) + for (const auto* base : {"::", "4000::", "8000::", "c000::"}) + Exec( + std::string{"/sbin/route -n add -inet6 -net "} + base + " -prefixlen 2 -interface " + + ifname, + false); } void DelDefaultRouteViaInterface(NetworkInterface& vpn) override { - const auto& info = vpn.Info(); - if (info.addrs.empty()) - throw std::runtime_error("No interface addresses found"); - - std::string gateway = info.addrs[0].range.addr.ToString(); - std::string cmd = "/sbin/route delete default " + gateway; - int ret = std::system(cmd.c_str()); - if (ret != 0) - throw std::runtime_error("DelDefaultRouteViaInterface failed: " + cmd); + const auto& ifname = vpn.Info().ifname; + for (const auto* range : {"0.0.0.0/1", "128.0.0.0/1"}) + Exec(std::string{"/sbin/route -n delete -net "} + range + " -interface " + ifname, false); + + for (const auto* base : {"::", "4000::", "8000::", "c000::"}) + Exec( + std::string{"/sbin/route -n delete -inet6 -net "} + base + " -prefixlen 2 -interface " + + ifname, + false); } // Add a route for a subnet via the VPN interface void AddRouteViaInterface(NetworkInterface& vpn, IPRange range) override { - std::string cmd = "/sbin/route add -net " + range.addr.ToString() + " -netmask " - + range.NetmaskString() + " -interface " + vpn.Info().ifname; - int ret = std::system(cmd.c_str()); - if (ret != 0) - throw std::runtime_error("AddRouteViaInterface failed: " + cmd); + const auto& ifname = vpn.Info().ifname; + if (range.IsV4()) + Exec( + "/sbin/route -n add -net " + net::TruncateV6(range.addr).ToString() + " -netmask " + + range.NetmaskString() + " -interface " + ifname); + else + Exec( + "/sbin/route -n add -inet6 -net " + range.addr.ToString() + " -prefixlen " + + std::to_string(bits::count_bits(range.netmask_bits)) + " -interface " + ifname); } void DelRouteViaInterface(NetworkInterface& vpn, IPRange range) override { - std::string cmd = "/sbin/route delete -net " + range.addr.ToString() + " -netmask " - + range.NetmaskString() + " -interface " + vpn.Info().ifname; - int ret = std::system(cmd.c_str()); - if (ret != 0) - throw std::runtime_error("DelRouteViaInterface failed: " + cmd); + const auto& ifname = vpn.Info().ifname; + if (range.IsV4()) + Exec( + "/sbin/route -n delete -net " + net::TruncateV6(range.addr).ToString() + " -netmask " + + range.NetmaskString() + " -interface " + ifname, + false); + else + Exec( + "/sbin/route -n delete -inet6 -net " + range.addr.ToString() + " -prefixlen " + + std::to_string(bits::count_bits(range.netmask_bits)) + " -interface " + ifname, + false); } + // Enumerate default-route gateways that do NOT live on the tun interface, + // by dumping the routing table via the classic BSD sysctl interface. The + // route poker uses this to find the physical gateway for hole-poking. std::vector - GetGatewaysNotOnInterface(NetworkInterface&) override + GetGatewaysNotOnInterface(NetworkInterface& vpn) override { - return {}; + std::vector gateways; + const unsigned int tun_index = if_nametoindex(vpn.Info().ifname.c_str()); + + int mib[6] = {CTL_NET, PF_ROUTE, 0, 0 /* all families */, NET_RT_FLAGS, RTF_GATEWAY}; + size_t needed = 0; + if (sysctl(mib, 6, nullptr, &needed, nullptr, 0) != 0) + return gateways; + std::vector buf; + buf.resize(needed); + if (sysctl(mib, 6, buf.data(), &needed, nullptr, 0) != 0) + return gateways; + + // routing socket sockaddrs are packed with 4-byte alignment; sa_len == 0 + // still occupies one alignment unit + constexpr auto align = sizeof(uint32_t); + const auto sa_size = [](const sockaddr* sa) { + return sa->sa_len ? ((sa->sa_len + align - 1) & ~(align - 1)) : align; + }; + + for (char* ptr = buf.data(); ptr + sizeof(rt_msghdr) <= buf.data() + needed;) + { + auto* rtm = reinterpret_cast(ptr); + if (rtm->rtm_msglen == 0) + break; + char* addrs = ptr + sizeof(rt_msghdr); + ptr += rtm->rtm_msglen; + + if (rtm->rtm_version != RTM_VERSION) + continue; + if ((rtm->rtm_flags & (RTF_UP | RTF_GATEWAY)) != (RTF_UP | RTF_GATEWAY)) + continue; + if (rtm->rtm_index == tun_index) + continue; // route lives on our own interface + if ((rtm->rtm_addrs & (RTA_DST | RTA_GATEWAY)) != (RTA_DST | RTA_GATEWAY)) + continue; + + const sockaddr* dst = nullptr; + const sockaddr* gw = nullptr; + char* sa_ptr = addrs; + for (int i = 0; i < RTAX_MAX; i++) + { + if (not(rtm->rtm_addrs & (1 << i))) + continue; + auto* sa = reinterpret_cast(sa_ptr); + if (i == RTAX_DST) + dst = sa; + else if (i == RTAX_GATEWAY) + gw = sa; + sa_ptr += sa_size(sa); + } + if (not dst or not gw) + continue; + + // only default routes (dst 0.0.0.0); the poker only consumes IPv4 + if (dst->sa_family != AF_INET or gw->sa_family != AF_INET) + continue; + if (reinterpret_cast(dst)->sin_addr.s_addr != 0) + continue; + + gateways.emplace_back( + net::ipv4addr_t{reinterpret_cast(gw)->sin_addr.s_addr}); + } + return gateways; } };